源码分析go调度器一: 调度器初始化

595 阅读3分钟

找到程序入口

package main

func main() {
}

go build main.go 后,用gdb,sudo gdb main (我自己是macos,不加sudo的话,gdb会卡死),并在gdb中执行info files找到可执行程序入口

sudo gdb main (我自己是macos,不加sudo的话,gdb会卡死)

 $  sudo gdb main
Password:
GNU gdb (GDB) 11.1
Copyright (C) 2021 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Type "show copying" and "show warranty" for details.
This GDB was configured as "x86_64-apple-darwin20.4.0".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<https://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
    <http://www.gnu.org/software/gdb/documentation/>.

For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from main...
(No debugging symbols found in main)
Loading Go Runtime support.
(gdb) info files
Symbols from "goroutine/main".
Local exec file:
	`goroutine/main', file type mach-o-x86-64.
	Entry point: 0x105c660
	0x0000000001001000 - 0x000000000105e270 is .text
	0x000000000105e280 - 0x000000000105e35e is __TEXT.__symbol_stub1
	0x000000000105e360 - 0x000000000108b0eb is __TEXT.__rodata
	0x000000000108b100 - 0x000000000108b59c is __TEXT.__typelink
	0x000000000108b5a0 - 0x000000000108b5a8 is __TEXT.__itablink
	0x000000000108b5a8 - 0x000000000108b5a8 is __TEXT.__gosymtab
	0x000000000108b5c0 - 0x00000000010c7658 is __TEXT.__gopclntab
	0x00000000010c8000 - 0x00000000010c8020 is __DATA.__go_buildinfo
	0x00000000010c8020 - 0x00000000010c8148 is __DATA.__nl_symbol_ptr
	0x00000000010c8160 - 0x00000000010c9360 is __DATA.__noptrdata
	0x00000000010c9360 - 0x00000000010cb150 is .data
	0x00000000010cb160 - 0x00000000010f8470 is .bss
	0x00000000010f8480 - 0x00000000010fd570 is __DATA.__noptrbss

找到了入口点为0x105c660,那么在这下断点

(gdb) b *0x105c660
Breakpoint 1 at 0x105c660
(gdb) r
Starting program: goroutine/main
[New Thread 0x1f03 of process 10002]
[New Thread 0x2303 of process 10002]
warning: unhandled dyld version (17)

Thread 2 hit Breakpoint 1, 0x000000000105c660 in _rt0_amd64_darwin ()
(gdb)

找到了入口函数为rt0_amd64_darwin

rt0_darwin_amd64.s:7

TEXT _rt0_amd64_darwin(SB),NOSPLIT,$-8
   JMP    _rt0_amd64(SB)

这个函数没干啥,只是JMP到了_rt0_amd64,继续往下分析

asm_amd64.s:15

TEXT _rt0_amd64(SB),NOSPLIT,$-8
   MOVQ   0(SP), DI  // DI=argc
   LEAQ   8(SP), SI  // SI=&argv
   JMP    runtime·rt0_go(SB)

将argc放到了DI里,argv放到了SI里,接下来JMP到runtime·rt0_go

asm_amd64.s:81

TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
   // copy arguments forward on an even stack
   MOVQ   DI, AX    // argc
   MOVQ   SI, BX    // &argv
   SUBQ   $(4*8+7), SP      // 2args 2auto
   // 目的是让SP按16字节对齐
   ANDQ   $~15, SP
   MOVQ   AX, 16(SP)
   MOVQ   BX, 24(SP)

这几条指令做的是:

  1. SP16字节对齐
  2. argcargv拷贝到新的位置 此时栈到分布如下

初始化g0

asm_amd64.s:92

// create istack out of the given (operating system) stack.
// _cgo_init may update stackguard.
MOVQ   $runtime·g0(SB), DI        // DI = g0
LEAQ   (-64*1024+104)(SP), BX     // BX=SP-64*1024 + 104
MOVQ   BX, g_stackguard0(DI)      // g0.stackguard0 = SP-64*1024 + 104
MOVQ   BX, g_stackguard1(DI)      // g0.stackguard1 = SP-64*1024 + 104
MOVQ   BX, (g_stack+stack_lo)(DI) // g0.stack.lo = SP-64*1024 + 104
MOVQ   SP, (g_stack+stack_hi)(DI) // g0.stack.hi = SP

这里初始化g0的栈,大小约为64K, 此时g0和栈空间如下所示:

image.png

主线程与m0绑定

对于macos,会直接执行到ok这里

asm_amd64.s:181

ok:
   // set the per-goroutine and per-mach "registers"
   // #define	get_tls(r)	MOVQ TLS, r
   get_tls(BX)                  // BX = TLS
   LEAQ   runtime·g0(SB), CX    // CX=&g0
   MOVQ   CX, g(BX)             // TLS.g=&g0
   LEAQ   runtime·m0(SB), AX    //AX=&m0

   // save m->g0 = g0
   MOVQ   CX, m_g0(AX)          // m0.g0 = &g0 
   // save m0 to g0->m 
   MOVQ   AX, g_m(CX)           // g0.m = &m0

此时m0、g0、g0的栈关系如下:

image.png

初始化m0和P

asm_amd64.s:206

MOVL    16(SP), AX    // copy argc
MOVL   AX, 0(SP)
MOVQ   24(SP), AX    // copy argv
MOVQ   AX, 8(SP)
CALL   runtime·args(SB)    //处理下参数
CALL   runtime·osinit(SB)  //这里仅仅是获取cpu核数
CALL   runtime·schedinit(SB)

&argvargc移动下,做为接下来调用函数的参数, 此时栈分布如下 image.png 接下来初始化调度器

proc.go:654

// The bootstrap sequence is:
//
// call osinit
// call schedinit
// make & queue new G
// call runtime·mstart
//
// The new G calls runtime·main.
func schedinit() {
   (...)
   // raceinit must be the first call to race detector.
   // In particular, it must be done before mallocinit below calls racemapshadow.
   _g_ := getg()                //g_=g0
   if raceenabled {
      _g_.racectx, raceprocctx0 = raceinit()
   }

   sched.maxmcount = 10000      // m最多为10000个 

   (...)
   mcommoninit(_g_.m, -1)       // 初始化m0, _g_.m 其实就是m0
   
   (...)
   

   lock(&sched.lock)
   sched.lastpoll = uint64(nanotime())
   procs := ncpu
   if n, ok := atoi32(gogetenv("GOMAXPROCS")); ok && n > 0 {
      procs = n
   }
   if procresize(procs) != nil {
      throw("unknown runnable goroutine during bootstrap")
   }
   (...)
}

这里schedinit大体做了两件事:

  1. 初始化m0
  2. 初始化P 先看下m0怎么初始化的
// Pre-allocated ID may be passed as 'id', or omitted by passing -1.
func mcommoninit(mp *m, id int64) {
   _g_ := getg()                  // g=g0

   // g0 stack won't make sense for user (and is not necessary unwindable).
   if _g_ != _g_.m.g0 {           // 这里肯定相等
      callers(1, mp.createstack[:])
   }

   lock(&sched.lock)            

   if id >= 0 {
      mp.id = id
   } else {
      mp.id = mReserveID()       // 这里面会检查m的数量是否大于10000,大于就抛出异常
   }

   mp.fastrand[0] = uint32(int64Hash(uint64(mp.id), fastrandseed))
   mp.fastrand[1] = uint32(int64Hash(uint64(cputicks()), ^fastrandseed))
   if mp.fastrand[0]|mp.fastrand[1] == 0 {
      mp.fastrand[1] = 1
   }

   mpreinit(mp)
   if mp.gsignal != nil {
      mp.gsignal.stackguard1 = mp.gsignal.stack.lo + _StackGuard
   }

   // Add to allm so garbage collector doesn't free g->m
   // when it is just in a register or thread-local storage.
   mp.alllink = allm            // 把m加到allm中

   // NumCgoCall() iterates over allm w/o schedlock,
   // so we need to publish it safely.
   atomicstorep(unsafe.Pointer(&allm), unsafe.Pointer(mp))  // allm的地址设置为m
   unlock(&sched.lock)

   
}

这个函数主要就是将m0与allm关联,具体关联关系如下:

image.png

proc.go:4994

// Change number of processors.
//
// sched.lock must be held, and the world must be stopped.
//
// gcworkbufs must not be being modified by either the GC or the write barrier
// code, so the GC must not be running if the number of Ps actually changes.
//
// Returns list of Ps with local work, they need to be scheduled by the caller.
func procresize(nprocs int32) *p {
   assertLockHeld(&sched.lock)
   assertWorldStopped()

   old := gomaxprocs   //初始化阶段 old=0
   if old < 0 || nprocs <= 0 {
      throw("procresize: invalid arg")
   }
   
   (...)

   // Grow allp if necessary.
   // 初始化阶段 len(allp)=0
   if nprocs > int32(len(allp)) {
      // Synchronize with retake, which could be running
      // concurrently since it doesn't run on a P.
      lock(&allpLock)
      if nprocs <= int32(cap(allp)) {
         allp = allp[:nprocs]
      } else {
         // 初始化阶段,会执行到这里
         nallp := make([]*p, nprocs)
         // Copy everything up to allp's cap so we
         // never lose old allocated Ps.
         copy(nallp, allp[:cap(allp)])
         allp = nallp
      }

      (...)
      
      unlock(&allpLock)
   }

   // initialize new P's
   for i := old; i < nprocs; i++ {
      pp := allp[i]
      if pp == nil {
         pp = new(p)
      }
      pp.init(i)  // 初始化pp,pp.id = id; pp.status = _Pgcstop
      atomicstorep(unsafe.Pointer(&allp[i]), unsafe.Pointer(pp))
   }

   _g_ := getg()     // _g_=g0
   if _g_.m.p != 0 && _g_.m.p.ptr().id < nprocs {  //初始化阶段, m.p为空
      // continue to use the current P
      _g_.m.p.ptr().status = _Prunning
      _g_.m.p.ptr().mcache.prepareForSweep()
   } else {
      // release the current P and acquire allp[0].
      //
      // We must do this before destroying our current P
      // because p.destroy itself has write barriers, so we
      // need to do that from a valid P.
      if _g_.m.p != 0 {       //初始化阶段, m.p为空,不走这里
         if trace.enabled {
            // Pretend that we were descheduled
            // and then scheduled again to keep
            // the trace sane.
            traceGoSched()
            traceProcStop(_g_.m.p.ptr())
         }
         _g_.m.p.ptr().m = 0
      }
      _g_.m.p = 0
      p := allp[0]
      p.m = 0
      p.status = _Pidle
      acquirep(p)  // 关联m0和allp[0], m0.p = allp[0]; allp[0].m = m0
      if trace.enabled {  
         traceGoStart()   
      }
   }

   // g.m.p is now set, so we no longer need mcache0 for bootstrapping.
   mcache0 = nil

   // 初始化阶段不会走这里,以后的逻辑可能会
   // 逻辑为将多余的P所关联的资源转移,比如P拥有的g,并且最终将P放到sched.gFree里,方便以后复用
   for i := nprocs; i < old; i++ {
      p := allp[i]
      p.destroy()
      // can't free P itself because it can be referenced by an M in syscall
   }
   
   // 再次截断allp,保证长度和预期相符
   // Trim allp.
   if int32(len(allp)) != nprocs {
      lock(&allpLock)
      allp = allp[:nprocs]
      idlepMask = idlepMask[:maskWords]
      timerpMask = timerpMask[:maskWords]
      unlock(&allpLock)
   }

   var runnablePs *p
   for i := nprocs - 1; i >= 0; i-- {
      p := allp[i]
      if _g_.m.p.ptr() == p {   // 忽略当前被关联的p
         continue
      }
      p.status = _Pidle
      if runqempty(p) { // 初始化阶段,所有的P都没有任务,所以都放入sched.pidle
         pidleput(p) 
      } else {         //初始化走不到这里,所以这里的返回值也没有被使用
         p.m.set(mget())
         p.link.set(runnablePs)
         runnablePs = p
      }
   }
   stealOrder.reset(uint32(nprocs))
   var int32p *int32 = &gomaxprocs // make compiler check that gomaxprocs is an int32
   atomic.Store((*uint32)(unsafe.Pointer(int32p)), uint32(nprocs))
   return runnablePs
}

此时m0、g0、g0的栈空间、allp[0]的关系如下:

image.png 此时调度器已经初始化完毕,下一节分析main函数的运行以及goroutine的创建