源码剖析go基于信号的抢占式调度

794 阅读2分钟

go version: 1.17
system: macos
信号的在程序中无非就是注册,发送信号以及信号的处理这三个流程,下面按照这个流程一个一个的进行分析

信号的注册

func mstart1() {
   ...
   if _g_.m == &m0 {
      mstartm0()
   }
   ...
}
func mstartm0() {
   ...
   initsig(false)
}
func initsig(preinit bool) {
   if !preinit {
      // It's now OK for signal handlers to run.
      signalsOK = true
   }

   // For c-archive/c-shared this is called by libpreinit with
   // preinit == true.
   if (isarchive || islibrary) && !preinit {
      return
   }

   for i := uint32(0); i < _NSIG; i++ {
      t := &sigtable[i]
      if t.flags == 0 || t.flags&_SigDefault != 0 {
         continue
      }

      // We don't need to use atomic operations here because
      // there shouldn't be any other goroutines running yet.
      fwdSig[i] = getsig(i)
      // 检这个信号是否需要设置signal handler
      if !sigInstallGoHandler(i) {
         // Even if we are not installing a signal handler,
         // set SA_ONSTACK if necessary.
         if fwdSig[i] != _SIG_DFL && fwdSig[i] != _SIG_IGN {
            setsigstack(i)
         } else if fwdSig[i] == _SIG_IGN {
            sigInitIgnored(i)
         }
         continue
      }

      handlingSig[i] = 1
      setsig(i, funcPC(sighandler))
   }
}

通过setsig来进行对某一个信号设置sighandler

func setsig(i uint32, fn uintptr) {
   var sa usigactiont
   sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK | _SA_RESTART
   sa.sa_mask = ^uint32(0)
   if fn == funcPC(sighandler) { // funcPC(sighandler) matches the callers in signal_unix.go
      if iscgo {
         fn = abi.FuncPCABI0(cgoSigtramp)
      } else {
         // sighandler被替换为sigtramp
         fn = abi.FuncPCABI0(sigtramp)
      }
   }
   *(*uintptr)(unsafe.Pointer(&sa.__sigaction_u)) = fn
   // 在这里进行信号的注册,当收到信号时候,会调用sigtramp
   sigaction(i, &sa, nil)
}

到这里,完成信号的注册,需要注意,当fnsighandler时,会被替换成sigtramp

收到信号后的处理

// This is the function registered during sigaction and is invoked when
// a signal is received. It just redirects to the Go function sigtrampgo.
// Called using C ABI.
TEXT runtime·sigtramp(SB),NOSPLIT,$0
   // Transition from C ABI to Go ABI.
   PUSH_REGS_HOST_TO_ABI0()

   // Call into the Go signal handler
   NOP    SP    // disable vet stack checking
   ADJSP  $24
   MOVL   DI, 0(SP)  // sig
   MOVQ   SI, 8(SP)  // info
   MOVQ   DX, 16(SP) // ctx
   CALL   ·sigtrampgo(SB)
   ADJSP  $-24

   POP_REGS_HOST_TO_ABI0()
   RET

收到信号后,会调用之前注册的函数sigtramp,并继续调用sigtrampgo

func sigtrampgo(sig uint32, info *siginfo, ctx unsafe.Pointer) {
   if sigfwdgo(sig, info, ctx) {
      return
   }
   c := &sigctxt{info, ctx}
   g := sigFetchG(c)
   setg(g)
   ...
   sighandler(sig, info, ctx, g)
   setg(g)
   ...
func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) {
   _g_ := getg()
   c := &sigctxt{info, ctxt}

   if sig == _SIGPROF {
      sigprof(c.sigpc(), c.sigsp(), c.siglr(), gp, _g_.m)
      return
   }

   if sig == _SIGTRAP && testSigtrap != nil && testSigtrap(info, (*sigctxt)(noescape(unsafe.Pointer(c))), gp) {
      return
   }

   if sig == _SIGUSR1 && testSigusr1 != nil && testSigusr1(gp) {
      return
   }
   // 代表是一个抢占信号
   if sig == sigPreempt && debug.asyncpreemptoff == 0 {
      // Might be a preemption signal.
      doSigPreempt(gp, c)
      // Even if this was definitely a preemption signal, it
      // may have been coalesced with another signal, so we
      // still let it through to the application.
   }
}

收到的信号是抢占信号,则调用doSigPreempt

// doSigPreempt handles a preemption signal on gp.
func doSigPreempt(gp *g, ctxt *sigctxt) {
   // Check if this G wants to be preempted and is safe to
   // preempt.
   // 检查g是否要被抢占,并且能被安全的抢占
   if wantAsyncPreempt(gp) {
      if ok, newpc := isAsyncSafePoint(gp, ctxt.sigpc(), ctxt.sigsp(), ctxt.siglr()); ok {
         // Adjust the PC and inject a call to asyncPreempt.
         ctxt.pushCall(funcPC(asyncPreempt), newpc)
      }
   }

   // Acknowledge the preemption.
   atomic.Xadd(&gp.m.preemptGen, 1)
   atomic.Store(&gp.m.signalPending, 0)

   if GOOS == "darwin" || GOOS == "ios" {
      atomic.Xadd(&pendingPreemptSignals, -1)
   }
}
func (c *sigctxt) pushCall(targetPC, resumePC uintptr) {
   // Make it look like we called target at resumePC.
   sp := uintptr(c.rsp())
   sp -= sys.PtrSize
   *(*uintptr)(unsafe.Pointer(sp)) = resumePC
   c.set_rsp(uint64(sp))
   c.set_rip(uint64(targetPC))
}

image.png pushCall这个函数很关键,他通过改变PC寄存器,从而执行抢占调用

TEXT ·asyncPreempt<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-0
   PUSHQ BP
   MOVQ SP, BP
   // Save flags before clobbering them
   PUSHFQ
   // obj doesn't understand ADD/SUB on SP, but does understand ADJSP
   ADJSP $368
   // But vet doesn't know ADJSP, so suppress vet stack checking
   NOP SP
   MOVQ AX, 0(SP)
   MOVQ CX, 8(SP)
   ...保存现场,其实就是把所有寄存器的值存起来
   MOVQ R14, 96(SP)
   MOVQ R15, 104(SP)
   #ifdef GOOS_darwin
   CMPB internal∕cpu·X86+const_offsetX86HasAVX(SB), $0
   JE 2(PC)
   VZEROUPPER
   #endif
   MOVUPS X0, 112(SP)
   MOVUPS X1, 128(SP)
   MOVUPS X2, 144(SP)
   MOVUPS X3, 160(SP)
   MOVUPS X4, 176(SP)
   MOVUPS X5, 192(SP)
   MOVUPS X6, 208(SP)
   MOVUPS X7, 224(SP)
   MOVUPS X8, 240(SP)
   MOVUPS X9, 256(SP)
   MOVUPS X10, 272(SP)
   MOVUPS X11, 288(SP)
   MOVUPS X12, 304(SP)
   MOVUPS X13, 320(SP)
   MOVUPS X14, 336(SP)
   MOVUPS X15, 352(SP)
   CALL ·asyncPreempt2(SB)
   MOVUPS 352(SP), X15
   MOVUPS 336(SP), X14
   ...恢复之前保存的所有寄存器的值
   MOVQ 8(SP), CX
   MOVQ 0(SP), AX
   ADJSP $-368
   POPFQ
   POPQ BP
   RET

asyncPreempt做的事就是保存现场,调用asyncPreempt2,再恢复现场

//go:nosplit
func asyncPreempt2() {
   gp := getg()
   gp.asyncSafePoint = true
   // preemptStop 在 GC 的栈扫描中会设置为 true
   if gp.preemptStop {
      mcall(preemptPark)
   } else { // 除了栈扫描,其它抢占全部走这条分支
      mcall(gopreempt_m)
   }
   gp.asyncSafePoint = false
}

asyncPreempt2有两个分支,需要注意下
分支1:

// preemptPark parks gp and puts it in _Gpreempted.
//
//go:systemstack
func preemptPark(gp *g) {
   ...

   // Transition from _Grunning to _Gscan|_Gpreempted. We can't
   // be in _Grunning when we dropg because then we'd be running
   // without an M, but the moment we're in _Gpreempted,
   // something could claim this G before we've fully cleaned it
   // up. Hence, we set the scan bit to lock down further
   // transitions until we can dropg.
   casGToPreemptScan(gp, _Grunning, _Gscan|_Gpreempted)
   // 解除m和curg的关系, curg.m = nil   m.curg = nil
   dropg()
   // 修改g的状态为_Gpreempted
   casfrom_Gscanstatus(gp, _Gscan|_Gpreempted, _Gpreempted)
   schedule()
}

分支2:

func gopreempt_m(gp *g) {
   if trace.enabled {
      traceGoPreempt()
   }
   goschedImpl(gp)
}
func goschedImpl(gp *g) {
   status := readgstatus(gp)
   if status&^_Gscan != _Grunning {
      dumpgstatus(gp)
      throw("bad g status")
   }
   // 修改g的状态为 _Grunnable
   casgstatus(gp, _Grunning, _Grunnable)
   // 解除m和curg的关系, curg.m = nil   m.curg = nil
   dropg()
   lock(&sched.lock)
   //将g放入全局队列的尾部
   globrunqput(gp)
   unlock(&sched.lock)

   schedule()
}

发送信号

栈扫描流程

func markroot(gcw *gcWork, i uint32) {
   // Note: if you add a case here, please also update heapdump.go:dumproots.
   switch {
      ...
   default:
      // 获取需要扫描的g
      var gp *g
      if work.baseStacks <= i && i < work.baseEnd {
         gp = allgs[i-work.baseStacks]
      } else {
         throw("markroot: bad index")
      }
      ...
      // scanstack must be done on the system stack in case
      // we're trying to scan our own stack
      // 切换到g0进行扫描
      systemstack(func() {
         // 挂起g,让g停止运行
         stopped := suspendG(gp)
         if stopped.dead {
            gp.gcscandone = true
            return
         }
         if gp.gcscandone {
            throw("g already scanned")
         }
         // 扫描g的栈
         scanstack(gp, gcw)
         gp.gcscandone = true
         // 恢复g的运行
         resumeG(stopped)
         ...
      })
   }
}
func suspendG(gp *g) suspendGState {
   ...
   // Drive the goroutine to a preemption point.
   stopped := false
   var asyncM *m
   var asyncGen uint32
   var nextPreemptM int64
   for i := 0; ; i++ {
      switch s := readgstatus(gp); s {
      // ===②===
      case _Gpreempted:
         // We (or someone else) suspended the G. Claim
         // ownership of it by transitioning it to
         // _Gwaiting.
         if !casGFromPreempted(gp, _Gpreempted, _Gwaiting) {
            break
         }

         // We stopped the G, so we have to ready it later.
         stopped = true

         s = _Gwaiting
         fallthrough
        // ===③===
      case _Grunnable, _Gsyscall, _Gwaiting:
         // Claim goroutine by setting scan bit.
         // This may race with execution or readying of gp.
         // The scan bit keeps it from transition state.
         if !castogscanstatus(gp, s, s|_Gscan) {
            break
         }

         // Clear the preemption request. It's safe to
         // reset the stack guard because we hold the
         // _Gscan bit and thus own the stack.
         gp.preemptStop = false
         gp.preempt = false
         gp.stackguard0 = gp.stack.lo + _StackGuard
         return suspendGState{g: gp, stopped: stopped}

      // ===①===
      case _Grunning:
         // Optimization: if there is already a pending preemption request
         // (from the previous loop iteration), don't bother with the atomics.
         if gp.preemptStop && gp.preempt && gp.stackguard0 == stackPreempt && asyncM == gp.m && atomic.Load(&asyncM.preemptGen) == asyncGen {
            break
         }

         // Temporarily block state transitions.
         if !castogscanstatus(gp, _Grunning, _Gscanrunning) {
            break
         }

         // Request synchronous preemption.
         // 设置抢占字段
         gp.preemptStop = true
         gp.preempt = true
         gp.stackguard0 = stackPreempt

         // Prepare for asynchronous preemption.
         asyncM2 := gp.m
         asyncGen2 := atomic.Load(&asyncM2.preemptGen)
         needAsync := asyncM != asyncM2 || asyncGen != asyncGen2
         asyncM = asyncM2
         asyncGen = asyncGen2

         casfrom_Gscanstatus(gp, _Gscanrunning, _Grunning)

         // Send asynchronous preemption. We do this
         // after CASing the G back to _Grunning
         // because preemptM may be synchronous and we
         // don't want to catch the G just spinning on
         // its status.
         if preemptMSupported && debug.asyncpreemptoff == 0 && needAsync {
            // Rate limit preemptM calls. This is
            // particularly important on Windows
            // where preemptM is actually
            // synchronous and the spin loop here
            // can lead to live-lock.
            now := nanotime()
            // 限制抢占频率
            if now >= nextPreemptM {
               nextPreemptM = now + yieldDelay/2
               // 发送抢占信号
               preemptM(asyncM)
            }
         }
      }
      ...
   }
}
const sigPreempt = _SIGURG

func preemptM(mp *m) {
   ...
   if atomic.Cas(&mp.signalPending, 0, 1) {
      signalM(mp, sigPreempt)
   }
    ...
}
func signalM(mp *m, sig int) {
   pthread_kill(pthread(mp.procid), uint32(sig))
}

: suspendG刚进来的时候,g为_Grunning,执行preemptM发送抢占信号_SIGURG
: 下一次循环会走到这里,将stop设置为true,执行fallthrough继续执行到下个case代码块
: 将preemptStop设置为false,并且返回 到此,suspendG函数执行结束,执行scanstack开始栈扫描,再之后会执行resumeG恢复g

func resumeG(state suspendGState) {
   ...
   // 前面将stopped设置为true,所以会进入这里
   if state.stopped {
      // We stopped it, so we need to re-schedule it.
      ready(gp, 0, true)
   }
}
func ready(gp *g, traceskip int, next bool) {
   // status is Gwaiting or Gscanwaiting, make Grunnable and put on runq
   casgstatus(gp, _Gwaiting, _Grunnable)
   // next为true,放入当前p的 runnext
   runqput(_g_.m.p.ptr(), gp, next)
   wakep()
   releasem(mp)
}

其他流程

sysmon -> retake -> preemptone

func preemptone(_p_ *p) bool {
   mp := _p_.m.ptr()
   if mp == nil || mp == getg().m {
      return false
   }
   // 获取正在执行的g
   gp := mp.curg
   if gp == nil || gp == mp.g0 {
      return false
   }

   gp.preempt = true

   // Every call in a goroutine checks for stack overflow by
   // comparing the current stack pointer to gp->stackguard0.
   // Setting gp->stackguard0 to StackPreempt folds
   // preemption into the normal stack overflow check.
   gp.stackguard0 = stackPreempt
    // 栈扩容时候检测是否被抢占
   // Request an async preemption of this P.
   if preemptMSupported && debug.asyncpreemptoff == 0 {
      _p_.preempt = true
      preemptM(mp)
   }

   return true
}
func preemptM(mp *m) {
   // On Darwin, don't try to preempt threads during exec.
   // Issue #41702.
   if GOOS == "darwin" || GOOS == "ios" {
      execLock.rlock()
   }

   if atomic.Cas(&mp.signalPending, 0, 1) {
      if GOOS == "darwin" || GOOS == "ios" {
         atomic.Xadd(&pendingPreemptSignals, 1)
      }

      // If multiple threads are preempting the same M, it may send many
      // signals to the same M such that it hardly make progress, causing
      // live-lock problem. Apparently this could happen on darwin. See
      // issue #37741.
      // Only send a signal if there isn't already one pending.
      // 发送抢占信号
      signalM(mp, sigPreempt)
   }

   if GOOS == "darwin" || GOOS == "ios" {
      execLock.runlock()
   }
}

preemptall -> preemptone

func preemptall() bool {
   res := false
   for _, _p_ := range allp {
      if _p_.status != _Prunning {
         continue
      }
      // 只抢占正在运行的P的g
      if preemptone(_p_) {
         res = true
      }
   }
   return res
}

下图简单画出了信号的发送以及处理

image.png