...
Run Format

Source file src/runtime/proc.go

     1	// Copyright 2014 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	package runtime
     6	
     7	import (
     8		"runtime/internal/atomic"
     9		"runtime/internal/sys"
    10		"unsafe"
    11	)
    12	
    13	var buildVersion = sys.TheVersion
    14	
    15	// Goroutine scheduler
    16	// The scheduler's job is to distribute ready-to-run goroutines over worker threads.
    17	//
    18	// The main concepts are:
    19	// G - goroutine.
    20	// M - worker thread, or machine.
    21	// P - processor, a resource that is required to execute Go code.
    22	//     M must have an associated P to execute Go code, however it can be
    23	//     blocked or in a syscall w/o an associated P.
    24	//
    25	// Design doc at https://golang.org/s/go11sched.
    26	
    27	// Worker thread parking/unparking.
    28	// We need to balance between keeping enough running worker threads to utilize
    29	// available hardware parallelism and parking excessive running worker threads
    30	// to conserve CPU resources and power. This is not simple for two reasons:
    31	// (1) scheduler state is intentionally distributed (in particular, per-P work
    32	// queues), so it is not possible to compute global predicates on fast paths;
    33	// (2) for optimal thread management we would need to know the future (don't park
    34	// a worker thread when a new goroutine will be readied in near future).
    35	//
    36	// Three rejected approaches that would work badly:
    37	// 1. Centralize all scheduler state (would inhibit scalability).
    38	// 2. Direct goroutine handoff. That is, when we ready a new goroutine and there
    39	//    is a spare P, unpark a thread and handoff it the thread and the goroutine.
    40	//    This would lead to thread state thrashing, as the thread that readied the
    41	//    goroutine can be out of work the very next moment, we will need to park it.
    42	//    Also, it would destroy locality of computation as we want to preserve
    43	//    dependent goroutines on the same thread; and introduce additional latency.
    44	// 3. Unpark an additional thread whenever we ready a goroutine and there is an
    45	//    idle P, but don't do handoff. This would lead to excessive thread parking/
    46	//    unparking as the additional threads will instantly park without discovering
    47	//    any work to do.
    48	//
    49	// The current approach:
    50	// We unpark an additional thread when we ready a goroutine if (1) there is an
    51	// idle P and there are no "spinning" worker threads. A worker thread is considered
    52	// spinning if it is out of local work and did not find work in global run queue/
    53	// netpoller; the spinning state is denoted in m.spinning and in sched.nmspinning.
    54	// Threads unparked this way are also considered spinning; we don't do goroutine
    55	// handoff so such threads are out of work initially. Spinning threads do some
    56	// spinning looking for work in per-P run queues before parking. If a spinning
    57	// thread finds work it takes itself out of the spinning state and proceeds to
    58	// execution. If it does not find work it takes itself out of the spinning state
    59	// and then parks.
    60	// If there is at least one spinning thread (sched.nmspinning>1), we don't unpark
    61	// new threads when readying goroutines. To compensate for that, if the last spinning
    62	// thread finds work and stops spinning, it must unpark a new spinning thread.
    63	// This approach smooths out unjustified spikes of thread unparking,
    64	// but at the same time guarantees eventual maximal CPU parallelism utilization.
    65	//
    66	// The main implementation complication is that we need to be very careful during
    67	// spinning->non-spinning thread transition. This transition can race with submission
    68	// of a new goroutine, and either one part or another needs to unpark another worker
    69	// thread. If they both fail to do that, we can end up with semi-persistent CPU
    70	// underutilization. The general pattern for goroutine readying is: submit a goroutine
    71	// to local work queue, #StoreLoad-style memory barrier, check sched.nmspinning.
    72	// The general pattern for spinning->non-spinning transition is: decrement nmspinning,
    73	// #StoreLoad-style memory barrier, check all per-P work queues for new work.
    74	// Note that all this complexity does not apply to global run queue as we are not
    75	// sloppy about thread unparking when submitting to global queue. Also see comments
    76	// for nmspinning manipulation.
    77	
    78	var (
    79		m0           m
    80		g0           g
    81		raceprocctx0 uintptr
    82	)
    83	
    84	//go:linkname runtime_init runtime.init
    85	func runtime_init()
    86	
    87	//go:linkname main_init main.init
    88	func main_init()
    89	
    90	// main_init_done is a signal used by cgocallbackg that initialization
    91	// has been completed. It is made before _cgo_notify_runtime_init_done,
    92	// so all cgo calls can rely on it existing. When main_init is complete,
    93	// it is closed, meaning cgocallbackg can reliably receive from it.
    94	var main_init_done chan bool
    95	
    96	//go:linkname main_main main.main
    97	func main_main()
    98	
    99	// runtimeInitTime is the nanotime() at which the runtime started.
   100	var runtimeInitTime int64
   101	
   102	// Value to use for signal mask for newly created M's.
   103	var initSigmask sigset
   104	
   105	// The main goroutine.
   106	func main() {
   107		g := getg()
   108	
   109		// Racectx of m0->g0 is used only as the parent of the main goroutine.
   110		// It must not be used for anything else.
   111		g.m.g0.racectx = 0
   112	
   113		// Max stack size is 1 GB on 64-bit, 250 MB on 32-bit.
   114		// Using decimal instead of binary GB and MB because
   115		// they look nicer in the stack overflow failure message.
   116		if sys.PtrSize == 8 {
   117			maxstacksize = 1000000000
   118		} else {
   119			maxstacksize = 250000000
   120		}
   121	
   122		// Record when the world started.
   123		runtimeInitTime = nanotime()
   124	
   125		systemstack(func() {
   126			newm(sysmon, nil)
   127		})
   128	
   129		// Lock the main goroutine onto this, the main OS thread,
   130		// during initialization. Most programs won't care, but a few
   131		// do require certain calls to be made by the main thread.
   132		// Those can arrange for main.main to run in the main thread
   133		// by calling runtime.LockOSThread during initialization
   134		// to preserve the lock.
   135		lockOSThread()
   136	
   137		if g.m != &m0 {
   138			throw("runtime.main not on m0")
   139		}
   140	
   141		runtime_init() // must be before defer
   142	
   143		// Defer unlock so that runtime.Goexit during init does the unlock too.
   144		needUnlock := true
   145		defer func() {
   146			if needUnlock {
   147				unlockOSThread()
   148			}
   149		}()
   150	
   151		gcenable()
   152	
   153		main_init_done = make(chan bool)
   154		if iscgo {
   155			if _cgo_thread_start == nil {
   156				throw("_cgo_thread_start missing")
   157			}
   158			if GOOS != "windows" {
   159				if _cgo_setenv == nil {
   160					throw("_cgo_setenv missing")
   161				}
   162				if _cgo_unsetenv == nil {
   163					throw("_cgo_unsetenv missing")
   164				}
   165			}
   166			if _cgo_notify_runtime_init_done == nil {
   167				throw("_cgo_notify_runtime_init_done missing")
   168			}
   169			cgocall(_cgo_notify_runtime_init_done, nil)
   170		}
   171	
   172		fn := main_init // make an indirect call, as the linker doesn't know the address of the main package when laying down the runtime
   173		fn()
   174		close(main_init_done)
   175	
   176		needUnlock = false
   177		unlockOSThread()
   178	
   179		if isarchive || islibrary {
   180			// A program compiled with -buildmode=c-archive or c-shared
   181			// has a main, but it is not executed.
   182			return
   183		}
   184		fn = main_main // make an indirect call, as the linker doesn't know the address of the main package when laying down the runtime
   185		fn()
   186		if raceenabled {
   187			racefini()
   188		}
   189	
   190		// Make racy client program work: if panicking on
   191		// another goroutine at the same time as main returns,
   192		// let the other goroutine finish printing the panic trace.
   193		// Once it does, it will exit. See issue 3934.
   194		if panicking != 0 {
   195			gopark(nil, nil, "panicwait", traceEvGoStop, 1)
   196		}
   197	
   198		exit(0)
   199		for {
   200			var x *int32
   201			*x = 0
   202		}
   203	}
   204	
   205	// os_beforeExit is called from os.Exit(0).
   206	//go:linkname os_beforeExit os.runtime_beforeExit
   207	func os_beforeExit() {
   208		if raceenabled {
   209			racefini()
   210		}
   211	}
   212	
   213	// start forcegc helper goroutine
   214	func init() {
   215		go forcegchelper()
   216	}
   217	
   218	func forcegchelper() {
   219		forcegc.g = getg()
   220		for {
   221			lock(&forcegc.lock)
   222			if forcegc.idle != 0 {
   223				throw("forcegc: phase error")
   224			}
   225			atomic.Store(&forcegc.idle, 1)
   226			goparkunlock(&forcegc.lock, "force gc (idle)", traceEvGoBlock, 1)
   227			// this goroutine is explicitly resumed by sysmon
   228			if debug.gctrace > 0 {
   229				println("GC forced")
   230			}
   231			gcStart(gcBackgroundMode, true)
   232		}
   233	}
   234	
   235	//go:nosplit
   236	
   237	// Gosched yields the processor, allowing other goroutines to run. It does not
   238	// suspend the current goroutine, so execution resumes automatically.
   239	func Gosched() {
   240		mcall(gosched_m)
   241	}
   242	
   243	var alwaysFalse bool
   244	
   245	// goschedguarded does nothing, but is written in a way that guarantees a preemption check in its prologue.
   246	// Calls to this function are inserted by the compiler in otherwise uninterruptible loops (see insertLoopReschedChecks).
   247	func goschedguarded() {
   248		if alwaysFalse {
   249			goschedguarded()
   250		}
   251	}
   252	
   253	// Puts the current goroutine into a waiting state and calls unlockf.
   254	// If unlockf returns false, the goroutine is resumed.
   255	// unlockf must not access this G's stack, as it may be moved between
   256	// the call to gopark and the call to unlockf.
   257	func gopark(unlockf func(*g, unsafe.Pointer) bool, lock unsafe.Pointer, reason string, traceEv byte, traceskip int) {
   258		mp := acquirem()
   259		gp := mp.curg
   260		status := readgstatus(gp)
   261		if status != _Grunning && status != _Gscanrunning {
   262			throw("gopark: bad g status")
   263		}
   264		mp.waitlock = lock
   265		mp.waitunlockf = *(*unsafe.Pointer)(unsafe.Pointer(&unlockf))
   266		gp.waitreason = reason
   267		mp.waittraceev = traceEv
   268		mp.waittraceskip = traceskip
   269		releasem(mp)
   270		// can't do anything that might move the G between Ms here.
   271		mcall(park_m)
   272	}
   273	
   274	// Puts the current goroutine into a waiting state and unlocks the lock.
   275	// The goroutine can be made runnable again by calling goready(gp).
   276	func goparkunlock(lock *mutex, reason string, traceEv byte, traceskip int) {
   277		gopark(parkunlock_c, unsafe.Pointer(lock), reason, traceEv, traceskip)
   278	}
   279	
   280	func goready(gp *g, traceskip int) {
   281		systemstack(func() {
   282			ready(gp, traceskip, true)
   283		})
   284	}
   285	
   286	//go:nosplit
   287	func acquireSudog() *sudog {
   288		// Delicate dance: the semaphore implementation calls
   289		// acquireSudog, acquireSudog calls new(sudog),
   290		// new calls malloc, malloc can call the garbage collector,
   291		// and the garbage collector calls the semaphore implementation
   292		// in stopTheWorld.
   293		// Break the cycle by doing acquirem/releasem around new(sudog).
   294		// The acquirem/releasem increments m.locks during new(sudog),
   295		// which keeps the garbage collector from being invoked.
   296		mp := acquirem()
   297		pp := mp.p.ptr()
   298		if len(pp.sudogcache) == 0 {
   299			lock(&sched.sudoglock)
   300			// First, try to grab a batch from central cache.
   301			for len(pp.sudogcache) < cap(pp.sudogcache)/2 && sched.sudogcache != nil {
   302				s := sched.sudogcache
   303				sched.sudogcache = s.next
   304				s.next = nil
   305				pp.sudogcache = append(pp.sudogcache, s)
   306			}
   307			unlock(&sched.sudoglock)
   308			// If the central cache is empty, allocate a new one.
   309			if len(pp.sudogcache) == 0 {
   310				pp.sudogcache = append(pp.sudogcache, new(sudog))
   311			}
   312		}
   313		n := len(pp.sudogcache)
   314		s := pp.sudogcache[n-1]
   315		pp.sudogcache[n-1] = nil
   316		pp.sudogcache = pp.sudogcache[:n-1]
   317		if s.elem != nil {
   318			throw("acquireSudog: found s.elem != nil in cache")
   319		}
   320		releasem(mp)
   321		return s
   322	}
   323	
   324	//go:nosplit
   325	func releaseSudog(s *sudog) {
   326		if s.elem != nil {
   327			throw("runtime: sudog with non-nil elem")
   328		}
   329		if s.selectdone != nil {
   330			throw("runtime: sudog with non-nil selectdone")
   331		}
   332		if s.next != nil {
   333			throw("runtime: sudog with non-nil next")
   334		}
   335		if s.prev != nil {
   336			throw("runtime: sudog with non-nil prev")
   337		}
   338		if s.waitlink != nil {
   339			throw("runtime: sudog with non-nil waitlink")
   340		}
   341		if s.c != nil {
   342			throw("runtime: sudog with non-nil c")
   343		}
   344		gp := getg()
   345		if gp.param != nil {
   346			throw("runtime: releaseSudog with non-nil gp.param")
   347		}
   348		mp := acquirem() // avoid rescheduling to another P
   349		pp := mp.p.ptr()
   350		if len(pp.sudogcache) == cap(pp.sudogcache) {
   351			// Transfer half of local cache to the central cache.
   352			var first, last *sudog
   353			for len(pp.sudogcache) > cap(pp.sudogcache)/2 {
   354				n := len(pp.sudogcache)
   355				p := pp.sudogcache[n-1]
   356				pp.sudogcache[n-1] = nil
   357				pp.sudogcache = pp.sudogcache[:n-1]
   358				if first == nil {
   359					first = p
   360				} else {
   361					last.next = p
   362				}
   363				last = p
   364			}
   365			lock(&sched.sudoglock)
   366			last.next = sched.sudogcache
   367			sched.sudogcache = first
   368			unlock(&sched.sudoglock)
   369		}
   370		pp.sudogcache = append(pp.sudogcache, s)
   371		releasem(mp)
   372	}
   373	
   374	// funcPC returns the entry PC of the function f.
   375	// It assumes that f is a func value. Otherwise the behavior is undefined.
   376	//go:nosplit
   377	func funcPC(f interface{}) uintptr {
   378		return **(**uintptr)(add(unsafe.Pointer(&f), sys.PtrSize))
   379	}
   380	
   381	// called from assembly
   382	func badmcall(fn func(*g)) {
   383		throw("runtime: mcall called on m->g0 stack")
   384	}
   385	
   386	func badmcall2(fn func(*g)) {
   387		throw("runtime: mcall function returned")
   388	}
   389	
   390	func badreflectcall() {
   391		panic(plainError("arg size to reflect.call more than 1GB"))
   392	}
   393	
   394	var badmorestackg0Msg = "fatal: morestack on g0\n"
   395	
   396	//go:nosplit
   397	//go:nowritebarrierrec
   398	func badmorestackg0() {
   399		sp := stringStructOf(&badmorestackg0Msg)
   400		write(2, sp.str, int32(sp.len))
   401	}
   402	
   403	var badmorestackgsignalMsg = "fatal: morestack on gsignal\n"
   404	
   405	//go:nosplit
   406	//go:nowritebarrierrec
   407	func badmorestackgsignal() {
   408		sp := stringStructOf(&badmorestackgsignalMsg)
   409		write(2, sp.str, int32(sp.len))
   410	}
   411	
   412	//go:nosplit
   413	func badctxt() {
   414		throw("ctxt != 0")
   415	}
   416	
   417	func lockedOSThread() bool {
   418		gp := getg()
   419		return gp.lockedm != nil && gp.m.lockedg != nil
   420	}
   421	
   422	var (
   423		allgs    []*g
   424		allglock mutex
   425	)
   426	
   427	func allgadd(gp *g) {
   428		if readgstatus(gp) == _Gidle {
   429			throw("allgadd: bad status Gidle")
   430		}
   431	
   432		lock(&allglock)
   433		allgs = append(allgs, gp)
   434		allglen = uintptr(len(allgs))
   435	
   436		// Grow GC rescan list if necessary.
   437		if len(allgs) > cap(work.rescan.list) {
   438			lock(&work.rescan.lock)
   439			l := work.rescan.list
   440			// Let append do the heavy lifting, but keep the
   441			// length the same.
   442			work.rescan.list = append(l[:cap(l)], 0)[:len(l)]
   443			unlock(&work.rescan.lock)
   444		}
   445		unlock(&allglock)
   446	}
   447	
   448	const (
   449		// Number of goroutine ids to grab from sched.goidgen to local per-P cache at once.
   450		// 16 seems to provide enough amortization, but other than that it's mostly arbitrary number.
   451		_GoidCacheBatch = 16
   452	)
   453	
   454	// The bootstrap sequence is:
   455	//
   456	//	call osinit
   457	//	call schedinit
   458	//	make & queue new G
   459	//	call runtime·mstart
   460	//
   461	// The new G calls runtime·main.
   462	func schedinit() {
   463		// raceinit must be the first call to race detector.
   464		// In particular, it must be done before mallocinit below calls racemapshadow.
   465		_g_ := getg()
   466		if raceenabled {
   467			_g_.racectx, raceprocctx0 = raceinit()
   468		}
   469	
   470		sched.maxmcount = 10000
   471	
   472		tracebackinit()
   473		moduledataverify()
   474		stackinit()
   475		mallocinit()
   476		mcommoninit(_g_.m)
   477		alginit()       // maps must not be used before this call
   478		modulesinit()   // provides activeModules
   479		typelinksinit() // uses maps, activeModules
   480		itabsinit()     // uses activeModules
   481	
   482		msigsave(_g_.m)
   483		initSigmask = _g_.m.sigmask
   484	
   485		goargs()
   486		goenvs()
   487		parsedebugvars()
   488		gcinit()
   489	
   490		sched.lastpoll = uint64(nanotime())
   491		procs := ncpu
   492		if n, ok := atoi32(gogetenv("GOMAXPROCS")); ok && n > 0 {
   493			procs = n
   494		}
   495		if procs > _MaxGomaxprocs {
   496			procs = _MaxGomaxprocs
   497		}
   498		if procresize(procs) != nil {
   499			throw("unknown runnable goroutine during bootstrap")
   500		}
   501	
   502		if buildVersion == "" {
   503			// Condition should never trigger. This code just serves
   504			// to ensure runtime·buildVersion is kept in the resulting binary.
   505			buildVersion = "unknown"
   506		}
   507	}
   508	
   509	func dumpgstatus(gp *g) {
   510		_g_ := getg()
   511		print("runtime: gp: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n")
   512		print("runtime:  g:  g=", _g_, ", goid=", _g_.goid, ",  g->atomicstatus=", readgstatus(_g_), "\n")
   513	}
   514	
   515	func checkmcount() {
   516		// sched lock is held
   517		if sched.mcount > sched.maxmcount {
   518			print("runtime: program exceeds ", sched.maxmcount, "-thread limit\n")
   519			throw("thread exhaustion")
   520		}
   521	}
   522	
   523	func mcommoninit(mp *m) {
   524		_g_ := getg()
   525	
   526		// g0 stack won't make sense for user (and is not necessary unwindable).
   527		if _g_ != _g_.m.g0 {
   528			callers(1, mp.createstack[:])
   529		}
   530	
   531		mp.fastrand = 0x49f6428a + uint32(mp.id) + uint32(cputicks())
   532		if mp.fastrand == 0 {
   533			mp.fastrand = 0x49f6428a
   534		}
   535	
   536		lock(&sched.lock)
   537		mp.id = sched.mcount
   538		sched.mcount++
   539		checkmcount()
   540		mpreinit(mp)
   541		if mp.gsignal != nil {
   542			mp.gsignal.stackguard1 = mp.gsignal.stack.lo + _StackGuard
   543		}
   544	
   545		// Add to allm so garbage collector doesn't free g->m
   546		// when it is just in a register or thread-local storage.
   547		mp.alllink = allm
   548	
   549		// NumCgoCall() iterates over allm w/o schedlock,
   550		// so we need to publish it safely.
   551		atomicstorep(unsafe.Pointer(&allm), unsafe.Pointer(mp))
   552		unlock(&sched.lock)
   553	
   554		// Allocate memory to hold a cgo traceback if the cgo call crashes.
   555		if iscgo || GOOS == "solaris" || GOOS == "windows" {
   556			mp.cgoCallers = new(cgoCallers)
   557		}
   558	}
   559	
   560	// Mark gp ready to run.
   561	func ready(gp *g, traceskip int, next bool) {
   562		if trace.enabled {
   563			traceGoUnpark(gp, traceskip)
   564		}
   565	
   566		status := readgstatus(gp)
   567	
   568		// Mark runnable.
   569		_g_ := getg()
   570		_g_.m.locks++ // disable preemption because it can be holding p in a local var
   571		if status&^_Gscan != _Gwaiting {
   572			dumpgstatus(gp)
   573			throw("bad g->status in ready")
   574		}
   575	
   576		// status is Gwaiting or Gscanwaiting, make Grunnable and put on runq
   577		casgstatus(gp, _Gwaiting, _Grunnable)
   578		runqput(_g_.m.p.ptr(), gp, next)
   579		if atomic.Load(&sched.npidle) != 0 && atomic.Load(&sched.nmspinning) == 0 {
   580			wakep()
   581		}
   582		_g_.m.locks--
   583		if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in Case we've cleared it in newstack
   584			_g_.stackguard0 = stackPreempt
   585		}
   586	}
   587	
   588	func gcprocs() int32 {
   589		// Figure out how many CPUs to use during GC.
   590		// Limited by gomaxprocs, number of actual CPUs, and MaxGcproc.
   591		lock(&sched.lock)
   592		n := gomaxprocs
   593		if n > ncpu {
   594			n = ncpu
   595		}
   596		if n > _MaxGcproc {
   597			n = _MaxGcproc
   598		}
   599		if n > sched.nmidle+1 { // one M is currently running
   600			n = sched.nmidle + 1
   601		}
   602		unlock(&sched.lock)
   603		return n
   604	}
   605	
   606	func needaddgcproc() bool {
   607		lock(&sched.lock)
   608		n := gomaxprocs
   609		if n > ncpu {
   610			n = ncpu
   611		}
   612		if n > _MaxGcproc {
   613			n = _MaxGcproc
   614		}
   615		n -= sched.nmidle + 1 // one M is currently running
   616		unlock(&sched.lock)
   617		return n > 0
   618	}
   619	
   620	func helpgc(nproc int32) {
   621		_g_ := getg()
   622		lock(&sched.lock)
   623		pos := 0
   624		for n := int32(1); n < nproc; n++ { // one M is currently running
   625			if allp[pos].mcache == _g_.m.mcache {
   626				pos++
   627			}
   628			mp := mget()
   629			if mp == nil {
   630				throw("gcprocs inconsistency")
   631			}
   632			mp.helpgc = n
   633			mp.p.set(allp[pos])
   634			mp.mcache = allp[pos].mcache
   635			pos++
   636			notewakeup(&mp.park)
   637		}
   638		unlock(&sched.lock)
   639	}
   640	
   641	// freezeStopWait is a large value that freezetheworld sets
   642	// sched.stopwait to in order to request that all Gs permanently stop.
   643	const freezeStopWait = 0x7fffffff
   644	
   645	// freezing is set to non-zero if the runtime is trying to freeze the
   646	// world.
   647	var freezing uint32
   648	
   649	// Similar to stopTheWorld but best-effort and can be called several times.
   650	// There is no reverse operation, used during crashing.
   651	// This function must not lock any mutexes.
   652	func freezetheworld() {
   653		atomic.Store(&freezing, 1)
   654		// stopwait and preemption requests can be lost
   655		// due to races with concurrently executing threads,
   656		// so try several times
   657		for i := 0; i < 5; i++ {
   658			// this should tell the scheduler to not start any new goroutines
   659			sched.stopwait = freezeStopWait
   660			atomic.Store(&sched.gcwaiting, 1)
   661			// this should stop running goroutines
   662			if !preemptall() {
   663				break // no running goroutines
   664			}
   665			usleep(1000)
   666		}
   667		// to be sure
   668		usleep(1000)
   669		preemptall()
   670		usleep(1000)
   671	}
   672	
   673	func isscanstatus(status uint32) bool {
   674		if status == _Gscan {
   675			throw("isscanstatus: Bad status Gscan")
   676		}
   677		return status&_Gscan == _Gscan
   678	}
   679	
   680	// All reads and writes of g's status go through readgstatus, casgstatus
   681	// castogscanstatus, casfrom_Gscanstatus.
   682	//go:nosplit
   683	func readgstatus(gp *g) uint32 {
   684		return atomic.Load(&gp.atomicstatus)
   685	}
   686	
   687	// Ownership of gcscanvalid:
   688	//
   689	// If gp is running (meaning status == _Grunning or _Grunning|_Gscan),
   690	// then gp owns gp.gcscanvalid, and other goroutines must not modify it.
   691	//
   692	// Otherwise, a second goroutine can lock the scan state by setting _Gscan
   693	// in the status bit and then modify gcscanvalid, and then unlock the scan state.
   694	//
   695	// Note that the first condition implies an exception to the second:
   696	// if a second goroutine changes gp's status to _Grunning|_Gscan,
   697	// that second goroutine still does not have the right to modify gcscanvalid.
   698	
   699	// The Gscanstatuses are acting like locks and this releases them.
   700	// If it proves to be a performance hit we should be able to make these
   701	// simple atomic stores but for now we are going to throw if
   702	// we see an inconsistent state.
   703	func casfrom_Gscanstatus(gp *g, oldval, newval uint32) {
   704		success := false
   705	
   706		// Check that transition is valid.
   707		switch oldval {
   708		default:
   709			print("runtime: casfrom_Gscanstatus bad oldval gp=", gp, ", oldval=", hex(oldval), ", newval=", hex(newval), "\n")
   710			dumpgstatus(gp)
   711			throw("casfrom_Gscanstatus:top gp->status is not in scan state")
   712		case _Gscanrunnable,
   713			_Gscanwaiting,
   714			_Gscanrunning,
   715			_Gscansyscall:
   716			if newval == oldval&^_Gscan {
   717				success = atomic.Cas(&gp.atomicstatus, oldval, newval)
   718			}
   719		}
   720		if !success {
   721			print("runtime: casfrom_Gscanstatus failed gp=", gp, ", oldval=", hex(oldval), ", newval=", hex(newval), "\n")
   722			dumpgstatus(gp)
   723			throw("casfrom_Gscanstatus: gp->status is not in scan state")
   724		}
   725	}
   726	
   727	// This will return false if the gp is not in the expected status and the cas fails.
   728	// This acts like a lock acquire while the casfromgstatus acts like a lock release.
   729	func castogscanstatus(gp *g, oldval, newval uint32) bool {
   730		switch oldval {
   731		case _Grunnable,
   732			_Grunning,
   733			_Gwaiting,
   734			_Gsyscall:
   735			if newval == oldval|_Gscan {
   736				return atomic.Cas(&gp.atomicstatus, oldval, newval)
   737			}
   738		}
   739		print("runtime: castogscanstatus oldval=", hex(oldval), " newval=", hex(newval), "\n")
   740		throw("castogscanstatus")
   741		panic("not reached")
   742	}
   743	
   744	// If asked to move to or from a Gscanstatus this will throw. Use the castogscanstatus
   745	// and casfrom_Gscanstatus instead.
   746	// casgstatus will loop if the g->atomicstatus is in a Gscan status until the routine that
   747	// put it in the Gscan state is finished.
   748	//go:nosplit
   749	func casgstatus(gp *g, oldval, newval uint32) {
   750		if (oldval&_Gscan != 0) || (newval&_Gscan != 0) || oldval == newval {
   751			systemstack(func() {
   752				print("runtime: casgstatus: oldval=", hex(oldval), " newval=", hex(newval), "\n")
   753				throw("casgstatus: bad incoming values")
   754			})
   755		}
   756	
   757		if oldval == _Grunning && gp.gcscanvalid {
   758			// If oldvall == _Grunning, then the actual status must be
   759			// _Grunning or _Grunning|_Gscan; either way,
   760			// we own gp.gcscanvalid, so it's safe to read.
   761			// gp.gcscanvalid must not be true when we are running.
   762			print("runtime: casgstatus ", hex(oldval), "->", hex(newval), " gp.status=", hex(gp.atomicstatus), " gp.gcscanvalid=true\n")
   763			throw("casgstatus")
   764		}
   765	
   766		// See http://golang.org/cl/21503 for justification of the yield delay.
   767		const yieldDelay = 5 * 1000
   768		var nextYield int64
   769	
   770		// loop if gp->atomicstatus is in a scan state giving
   771		// GC time to finish and change the state to oldval.
   772		for i := 0; !atomic.Cas(&gp.atomicstatus, oldval, newval); i++ {
   773			if oldval == _Gwaiting && gp.atomicstatus == _Grunnable {
   774				systemstack(func() {
   775					throw("casgstatus: waiting for Gwaiting but is Grunnable")
   776				})
   777			}
   778			// Help GC if needed.
   779			// if gp.preemptscan && !gp.gcworkdone && (oldval == _Grunning || oldval == _Gsyscall) {
   780			// 	gp.preemptscan = false
   781			// 	systemstack(func() {
   782			// 		gcphasework(gp)
   783			// 	})
   784			// }
   785			// But meanwhile just yield.
   786			if i == 0 {
   787				nextYield = nanotime() + yieldDelay
   788			}
   789			if nanotime() < nextYield {
   790				for x := 0; x < 10 && gp.atomicstatus != oldval; x++ {
   791					procyield(1)
   792				}
   793			} else {
   794				osyield()
   795				nextYield = nanotime() + yieldDelay/2
   796			}
   797		}
   798		if newval == _Grunning && gp.gcscanvalid {
   799			// Run queueRescan on the system stack so it has more space.
   800			systemstack(func() { queueRescan(gp) })
   801		}
   802	}
   803	
   804	// casgstatus(gp, oldstatus, Gcopystack), assuming oldstatus is Gwaiting or Grunnable.
   805	// Returns old status. Cannot call casgstatus directly, because we are racing with an
   806	// async wakeup that might come in from netpoll. If we see Gwaiting from the readgstatus,
   807	// it might have become Grunnable by the time we get to the cas. If we called casgstatus,
   808	// it would loop waiting for the status to go back to Gwaiting, which it never will.
   809	//go:nosplit
   810	func casgcopystack(gp *g) uint32 {
   811		for {
   812			oldstatus := readgstatus(gp) &^ _Gscan
   813			if oldstatus != _Gwaiting && oldstatus != _Grunnable {
   814				throw("copystack: bad status, not Gwaiting or Grunnable")
   815			}
   816			if atomic.Cas(&gp.atomicstatus, oldstatus, _Gcopystack) {
   817				return oldstatus
   818			}
   819		}
   820	}
   821	
   822	// scang blocks until gp's stack has been scanned.
   823	// It might be scanned by scang or it might be scanned by the goroutine itself.
   824	// Either way, the stack scan has completed when scang returns.
   825	func scang(gp *g, gcw *gcWork) {
   826		// Invariant; we (the caller, markroot for a specific goroutine) own gp.gcscandone.
   827		// Nothing is racing with us now, but gcscandone might be set to true left over
   828		// from an earlier round of stack scanning (we scan twice per GC).
   829		// We use gcscandone to record whether the scan has been done during this round.
   830		// It is important that the scan happens exactly once: if called twice,
   831		// the installation of stack barriers will detect the double scan and die.
   832	
   833		gp.gcscandone = false
   834	
   835		// See http://golang.org/cl/21503 for justification of the yield delay.
   836		const yieldDelay = 10 * 1000
   837		var nextYield int64
   838	
   839		// Endeavor to get gcscandone set to true,
   840		// either by doing the stack scan ourselves or by coercing gp to scan itself.
   841		// gp.gcscandone can transition from false to true when we're not looking
   842		// (if we asked for preemption), so any time we lock the status using
   843		// castogscanstatus we have to double-check that the scan is still not done.
   844	loop:
   845		for i := 0; !gp.gcscandone; i++ {
   846			switch s := readgstatus(gp); s {
   847			default:
   848				dumpgstatus(gp)
   849				throw("stopg: invalid status")
   850	
   851			case _Gdead:
   852				// No stack.
   853				gp.gcscandone = true
   854				break loop
   855	
   856			case _Gcopystack:
   857			// Stack being switched. Go around again.
   858	
   859			case _Grunnable, _Gsyscall, _Gwaiting:
   860				// Claim goroutine by setting scan bit.
   861				// Racing with execution or readying of gp.
   862				// The scan bit keeps them from running
   863				// the goroutine until we're done.
   864				if castogscanstatus(gp, s, s|_Gscan) {
   865					if !gp.gcscandone {
   866						scanstack(gp, gcw)
   867						gp.gcscandone = true
   868					}
   869					restartg(gp)
   870					break loop
   871				}
   872	
   873			case _Gscanwaiting:
   874			// newstack is doing a scan for us right now. Wait.
   875	
   876			case _Grunning:
   877				// Goroutine running. Try to preempt execution so it can scan itself.
   878				// The preemption handler (in newstack) does the actual scan.
   879	
   880				// Optimization: if there is already a pending preemption request
   881				// (from the previous loop iteration), don't bother with the atomics.
   882				if gp.preemptscan && gp.preempt && gp.stackguard0 == stackPreempt {
   883					break
   884				}
   885	
   886				// Ask for preemption and self scan.
   887				if castogscanstatus(gp, _Grunning, _Gscanrunning) {
   888					if !gp.gcscandone {
   889						gp.preemptscan = true
   890						gp.preempt = true
   891						gp.stackguard0 = stackPreempt
   892					}
   893					casfrom_Gscanstatus(gp, _Gscanrunning, _Grunning)
   894				}
   895			}
   896	
   897			if i == 0 {
   898				nextYield = nanotime() + yieldDelay
   899			}
   900			if nanotime() < nextYield {
   901				procyield(10)
   902			} else {
   903				osyield()
   904				nextYield = nanotime() + yieldDelay/2
   905			}
   906		}
   907	
   908		gp.preemptscan = false // cancel scan request if no longer needed
   909	}
   910	
   911	// The GC requests that this routine be moved from a scanmumble state to a mumble state.
   912	func restartg(gp *g) {
   913		s := readgstatus(gp)
   914		switch s {
   915		default:
   916			dumpgstatus(gp)
   917			throw("restartg: unexpected status")
   918	
   919		case _Gdead:
   920		// ok
   921	
   922		case _Gscanrunnable,
   923			_Gscanwaiting,
   924			_Gscansyscall:
   925			casfrom_Gscanstatus(gp, s, s&^_Gscan)
   926		}
   927	}
   928	
   929	// stopTheWorld stops all P's from executing goroutines, interrupting
   930	// all goroutines at GC safe points and records reason as the reason
   931	// for the stop. On return, only the current goroutine's P is running.
   932	// stopTheWorld must not be called from a system stack and the caller
   933	// must not hold worldsema. The caller must call startTheWorld when
   934	// other P's should resume execution.
   935	//
   936	// stopTheWorld is safe for multiple goroutines to call at the
   937	// same time. Each will execute its own stop, and the stops will
   938	// be serialized.
   939	//
   940	// This is also used by routines that do stack dumps. If the system is
   941	// in panic or being exited, this may not reliably stop all
   942	// goroutines.
   943	func stopTheWorld(reason string) {
   944		semacquire(&worldsema, 0)
   945		getg().m.preemptoff = reason
   946		systemstack(stopTheWorldWithSema)
   947	}
   948	
   949	// startTheWorld undoes the effects of stopTheWorld.
   950	func startTheWorld() {
   951		systemstack(startTheWorldWithSema)
   952		// worldsema must be held over startTheWorldWithSema to ensure
   953		// gomaxprocs cannot change while worldsema is held.
   954		semrelease(&worldsema)
   955		getg().m.preemptoff = ""
   956	}
   957	
   958	// Holding worldsema grants an M the right to try to stop the world
   959	// and prevents gomaxprocs from changing concurrently.
   960	var worldsema uint32 = 1
   961	
   962	// stopTheWorldWithSema is the core implementation of stopTheWorld.
   963	// The caller is responsible for acquiring worldsema and disabling
   964	// preemption first and then should stopTheWorldWithSema on the system
   965	// stack:
   966	//
   967	//	semacquire(&worldsema, 0)
   968	//	m.preemptoff = "reason"
   969	//	systemstack(stopTheWorldWithSema)
   970	//
   971	// When finished, the caller must either call startTheWorld or undo
   972	// these three operations separately:
   973	//
   974	//	m.preemptoff = ""
   975	//	systemstack(startTheWorldWithSema)
   976	//	semrelease(&worldsema)
   977	//
   978	// It is allowed to acquire worldsema once and then execute multiple
   979	// startTheWorldWithSema/stopTheWorldWithSema pairs.
   980	// Other P's are able to execute between successive calls to
   981	// startTheWorldWithSema and stopTheWorldWithSema.
   982	// Holding worldsema causes any other goroutines invoking
   983	// stopTheWorld to block.
   984	func stopTheWorldWithSema() {
   985		_g_ := getg()
   986	
   987		// If we hold a lock, then we won't be able to stop another M
   988		// that is blocked trying to acquire the lock.
   989		if _g_.m.locks > 0 {
   990			throw("stopTheWorld: holding locks")
   991		}
   992	
   993		lock(&sched.lock)
   994		sched.stopwait = gomaxprocs
   995		atomic.Store(&sched.gcwaiting, 1)
   996		preemptall()
   997		// stop current P
   998		_g_.m.p.ptr().status = _Pgcstop // Pgcstop is only diagnostic.
   999		sched.stopwait--
  1000		// try to retake all P's in Psyscall status
  1001		for i := 0; i < int(gomaxprocs); i++ {
  1002			p := allp[i]
  1003			s := p.status
  1004			if s == _Psyscall && atomic.Cas(&p.status, s, _Pgcstop) {
  1005				if trace.enabled {
  1006					traceGoSysBlock(p)
  1007					traceProcStop(p)
  1008				}
  1009				p.syscalltick++
  1010				sched.stopwait--
  1011			}
  1012		}
  1013		// stop idle P's
  1014		for {
  1015			p := pidleget()
  1016			if p == nil {
  1017				break
  1018			}
  1019			p.status = _Pgcstop
  1020			sched.stopwait--
  1021		}
  1022		wait := sched.stopwait > 0
  1023		unlock(&sched.lock)
  1024	
  1025		// wait for remaining P's to stop voluntarily
  1026		if wait {
  1027			for {
  1028				// wait for 100us, then try to re-preempt in case of any races
  1029				if notetsleep(&sched.stopnote, 100*1000) {
  1030					noteclear(&sched.stopnote)
  1031					break
  1032				}
  1033				preemptall()
  1034			}
  1035		}
  1036	
  1037		// sanity checks
  1038		bad := ""
  1039		if sched.stopwait != 0 {
  1040			bad = "stopTheWorld: not stopped (stopwait != 0)"
  1041		} else {
  1042			for i := 0; i < int(gomaxprocs); i++ {
  1043				p := allp[i]
  1044				if p.status != _Pgcstop {
  1045					bad = "stopTheWorld: not stopped (status != _Pgcstop)"
  1046				}
  1047			}
  1048		}
  1049		if atomic.Load(&freezing) != 0 {
  1050			// Some other thread is panicking. This can cause the
  1051			// sanity checks above to fail if the panic happens in
  1052			// the signal handler on a stopped thread. Either way,
  1053			// we should halt this thread.
  1054			lock(&deadlock)
  1055			lock(&deadlock)
  1056		}
  1057		if bad != "" {
  1058			throw(bad)
  1059		}
  1060	}
  1061	
  1062	func mhelpgc() {
  1063		_g_ := getg()
  1064		_g_.m.helpgc = -1
  1065	}
  1066	
  1067	func startTheWorldWithSema() {
  1068		_g_ := getg()
  1069	
  1070		_g_.m.locks++        // disable preemption because it can be holding p in a local var
  1071		gp := netpoll(false) // non-blocking
  1072		injectglist(gp)
  1073		add := needaddgcproc()
  1074		lock(&sched.lock)
  1075	
  1076		procs := gomaxprocs
  1077		if newprocs != 0 {
  1078			procs = newprocs
  1079			newprocs = 0
  1080		}
  1081		p1 := procresize(procs)
  1082		sched.gcwaiting = 0
  1083		if sched.sysmonwait != 0 {
  1084			sched.sysmonwait = 0
  1085			notewakeup(&sched.sysmonnote)
  1086		}
  1087		unlock(&sched.lock)
  1088	
  1089		for p1 != nil {
  1090			p := p1
  1091			p1 = p1.link.ptr()
  1092			if p.m != 0 {
  1093				mp := p.m.ptr()
  1094				p.m = 0
  1095				if mp.nextp != 0 {
  1096					throw("startTheWorld: inconsistent mp->nextp")
  1097				}
  1098				mp.nextp.set(p)
  1099				notewakeup(&mp.park)
  1100			} else {
  1101				// Start M to run P.  Do not start another M below.
  1102				newm(nil, p)
  1103				add = false
  1104			}
  1105		}
  1106	
  1107		// Wakeup an additional proc in case we have excessive runnable goroutines
  1108		// in local queues or in the global queue. If we don't, the proc will park itself.
  1109		// If we have lots of excessive work, resetspinning will unpark additional procs as necessary.
  1110		if atomic.Load(&sched.npidle) != 0 && atomic.Load(&sched.nmspinning) == 0 {
  1111			wakep()
  1112		}
  1113	
  1114		if add {
  1115			// If GC could have used another helper proc, start one now,
  1116			// in the hope that it will be available next time.
  1117			// It would have been even better to start it before the collection,
  1118			// but doing so requires allocating memory, so it's tricky to
  1119			// coordinate. This lazy approach works out in practice:
  1120			// we don't mind if the first couple gc rounds don't have quite
  1121			// the maximum number of procs.
  1122			newm(mhelpgc, nil)
  1123		}
  1124		_g_.m.locks--
  1125		if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
  1126			_g_.stackguard0 = stackPreempt
  1127		}
  1128	}
  1129	
  1130	// Called to start an M.
  1131	//go:nosplit
  1132	func mstart() {
  1133		_g_ := getg()
  1134	
  1135		if _g_.stack.lo == 0 {
  1136			// Initialize stack bounds from system stack.
  1137			// Cgo may have left stack size in stack.hi.
  1138			size := _g_.stack.hi
  1139			if size == 0 {
  1140				size = 8192 * sys.StackGuardMultiplier
  1141			}
  1142			_g_.stack.hi = uintptr(noescape(unsafe.Pointer(&size)))
  1143			_g_.stack.lo = _g_.stack.hi - size + 1024
  1144		}
  1145		// Initialize stack guards so that we can start calling
  1146		// both Go and C functions with stack growth prologues.
  1147		_g_.stackguard0 = _g_.stack.lo + _StackGuard
  1148		_g_.stackguard1 = _g_.stackguard0
  1149		mstart1()
  1150	}
  1151	
  1152	func mstart1() {
  1153		_g_ := getg()
  1154	
  1155		if _g_ != _g_.m.g0 {
  1156			throw("bad runtime·mstart")
  1157		}
  1158	
  1159		// Record top of stack for use by mcall.
  1160		// Once we call schedule we're never coming back,
  1161		// so other calls can reuse this stack space.
  1162		gosave(&_g_.m.g0.sched)
  1163		_g_.m.g0.sched.pc = ^uintptr(0) // make sure it is never used
  1164		asminit()
  1165		minit()
  1166	
  1167		// Install signal handlers; after minit so that minit can
  1168		// prepare the thread to be able to handle the signals.
  1169		if _g_.m == &m0 {
  1170			// Create an extra M for callbacks on threads not created by Go.
  1171			if iscgo && !cgoHasExtraM {
  1172				cgoHasExtraM = true
  1173				newextram()
  1174			}
  1175			initsig(false)
  1176		}
  1177	
  1178		if fn := _g_.m.mstartfn; fn != nil {
  1179			fn()
  1180		}
  1181	
  1182		if _g_.m.helpgc != 0 {
  1183			_g_.m.helpgc = 0
  1184			stopm()
  1185		} else if _g_.m != &m0 {
  1186			acquirep(_g_.m.nextp.ptr())
  1187			_g_.m.nextp = 0
  1188		}
  1189		schedule()
  1190	}
  1191	
  1192	// forEachP calls fn(p) for every P p when p reaches a GC safe point.
  1193	// If a P is currently executing code, this will bring the P to a GC
  1194	// safe point and execute fn on that P. If the P is not executing code
  1195	// (it is idle or in a syscall), this will call fn(p) directly while
  1196	// preventing the P from exiting its state. This does not ensure that
  1197	// fn will run on every CPU executing Go code, but it acts as a global
  1198	// memory barrier. GC uses this as a "ragged barrier."
  1199	//
  1200	// The caller must hold worldsema.
  1201	//
  1202	//go:systemstack
  1203	func forEachP(fn func(*p)) {
  1204		mp := acquirem()
  1205		_p_ := getg().m.p.ptr()
  1206	
  1207		lock(&sched.lock)
  1208		if sched.safePointWait != 0 {
  1209			throw("forEachP: sched.safePointWait != 0")
  1210		}
  1211		sched.safePointWait = gomaxprocs - 1
  1212		sched.safePointFn = fn
  1213	
  1214		// Ask all Ps to run the safe point function.
  1215		for _, p := range allp[:gomaxprocs] {
  1216			if p != _p_ {
  1217				atomic.Store(&p.runSafePointFn, 1)
  1218			}
  1219		}
  1220		preemptall()
  1221	
  1222		// Any P entering _Pidle or _Psyscall from now on will observe
  1223		// p.runSafePointFn == 1 and will call runSafePointFn when
  1224		// changing its status to _Pidle/_Psyscall.
  1225	
  1226		// Run safe point function for all idle Ps. sched.pidle will
  1227		// not change because we hold sched.lock.
  1228		for p := sched.pidle.ptr(); p != nil; p = p.link.ptr() {
  1229			if atomic.Cas(&p.runSafePointFn, 1, 0) {
  1230				fn(p)
  1231				sched.safePointWait--
  1232			}
  1233		}
  1234	
  1235		wait := sched.safePointWait > 0
  1236		unlock(&sched.lock)
  1237	
  1238		// Run fn for the current P.
  1239		fn(_p_)
  1240	
  1241		// Force Ps currently in _Psyscall into _Pidle and hand them
  1242		// off to induce safe point function execution.
  1243		for i := 0; i < int(gomaxprocs); i++ {
  1244			p := allp[i]
  1245			s := p.status
  1246			if s == _Psyscall && p.runSafePointFn == 1 && atomic.Cas(&p.status, s, _Pidle) {
  1247				if trace.enabled {
  1248					traceGoSysBlock(p)
  1249					traceProcStop(p)
  1250				}
  1251				p.syscalltick++
  1252				handoffp(p)
  1253			}
  1254		}
  1255	
  1256		// Wait for remaining Ps to run fn.
  1257		if wait {
  1258			for {
  1259				// Wait for 100us, then try to re-preempt in
  1260				// case of any races.
  1261				//
  1262				// Requires system stack.
  1263				if notetsleep(&sched.safePointNote, 100*1000) {
  1264					noteclear(&sched.safePointNote)
  1265					break
  1266				}
  1267				preemptall()
  1268			}
  1269		}
  1270		if sched.safePointWait != 0 {
  1271			throw("forEachP: not done")
  1272		}
  1273		for i := 0; i < int(gomaxprocs); i++ {
  1274			p := allp[i]
  1275			if p.runSafePointFn != 0 {
  1276				throw("forEachP: P did not run fn")
  1277			}
  1278		}
  1279	
  1280		lock(&sched.lock)
  1281		sched.safePointFn = nil
  1282		unlock(&sched.lock)
  1283		releasem(mp)
  1284	}
  1285	
  1286	// runSafePointFn runs the safe point function, if any, for this P.
  1287	// This should be called like
  1288	//
  1289	//     if getg().m.p.runSafePointFn != 0 {
  1290	//         runSafePointFn()
  1291	//     }
  1292	//
  1293	// runSafePointFn must be checked on any transition in to _Pidle or
  1294	// _Psyscall to avoid a race where forEachP sees that the P is running
  1295	// just before the P goes into _Pidle/_Psyscall and neither forEachP
  1296	// nor the P run the safe-point function.
  1297	func runSafePointFn() {
  1298		p := getg().m.p.ptr()
  1299		// Resolve the race between forEachP running the safe-point
  1300		// function on this P's behalf and this P running the
  1301		// safe-point function directly.
  1302		if !atomic.Cas(&p.runSafePointFn, 1, 0) {
  1303			return
  1304		}
  1305		sched.safePointFn(p)
  1306		lock(&sched.lock)
  1307		sched.safePointWait--
  1308		if sched.safePointWait == 0 {
  1309			notewakeup(&sched.safePointNote)
  1310		}
  1311		unlock(&sched.lock)
  1312	}
  1313	
  1314	// When running with cgo, we call _cgo_thread_start
  1315	// to start threads for us so that we can play nicely with
  1316	// foreign code.
  1317	var cgoThreadStart unsafe.Pointer
  1318	
  1319	type cgothreadstart struct {
  1320		g   guintptr
  1321		tls *uint64
  1322		fn  unsafe.Pointer
  1323	}
  1324	
  1325	// Allocate a new m unassociated with any thread.
  1326	// Can use p for allocation context if needed.
  1327	// fn is recorded as the new m's m.mstartfn.
  1328	//
  1329	// This function is allowed to have write barriers even if the caller
  1330	// isn't because it borrows _p_.
  1331	//
  1332	//go:yeswritebarrierrec
  1333	func allocm(_p_ *p, fn func()) *m {
  1334		_g_ := getg()
  1335		_g_.m.locks++ // disable GC because it can be called from sysmon
  1336		if _g_.m.p == 0 {
  1337			acquirep(_p_) // temporarily borrow p for mallocs in this function
  1338		}
  1339		mp := new(m)
  1340		mp.mstartfn = fn
  1341		mcommoninit(mp)
  1342	
  1343		// In case of cgo or Solaris, pthread_create will make us a stack.
  1344		// Windows and Plan 9 will layout sched stack on OS stack.
  1345		if iscgo || GOOS == "solaris" || GOOS == "windows" || GOOS == "plan9" {
  1346			mp.g0 = malg(-1)
  1347		} else {
  1348			mp.g0 = malg(8192 * sys.StackGuardMultiplier)
  1349		}
  1350		mp.g0.m = mp
  1351	
  1352		if _p_ == _g_.m.p.ptr() {
  1353			releasep()
  1354		}
  1355		_g_.m.locks--
  1356		if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
  1357			_g_.stackguard0 = stackPreempt
  1358		}
  1359	
  1360		return mp
  1361	}
  1362	
  1363	// needm is called when a cgo callback happens on a
  1364	// thread without an m (a thread not created by Go).
  1365	// In this case, needm is expected to find an m to use
  1366	// and return with m, g initialized correctly.
  1367	// Since m and g are not set now (likely nil, but see below)
  1368	// needm is limited in what routines it can call. In particular
  1369	// it can only call nosplit functions (textflag 7) and cannot
  1370	// do any scheduling that requires an m.
  1371	//
  1372	// In order to avoid needing heavy lifting here, we adopt
  1373	// the following strategy: there is a stack of available m's
  1374	// that can be stolen. Using compare-and-swap
  1375	// to pop from the stack has ABA races, so we simulate
  1376	// a lock by doing an exchange (via casp) to steal the stack
  1377	// head and replace the top pointer with MLOCKED (1).
  1378	// This serves as a simple spin lock that we can use even
  1379	// without an m. The thread that locks the stack in this way
  1380	// unlocks the stack by storing a valid stack head pointer.
  1381	//
  1382	// In order to make sure that there is always an m structure
  1383	// available to be stolen, we maintain the invariant that there
  1384	// is always one more than needed. At the beginning of the
  1385	// program (if cgo is in use) the list is seeded with a single m.
  1386	// If needm finds that it has taken the last m off the list, its job
  1387	// is - once it has installed its own m so that it can do things like
  1388	// allocate memory - to create a spare m and put it on the list.
  1389	//
  1390	// Each of these extra m's also has a g0 and a curg that are
  1391	// pressed into service as the scheduling stack and current
  1392	// goroutine for the duration of the cgo callback.
  1393	//
  1394	// When the callback is done with the m, it calls dropm to
  1395	// put the m back on the list.
  1396	//go:nosplit
  1397	func needm(x byte) {
  1398		if iscgo && !cgoHasExtraM {
  1399			// Can happen if C/C++ code calls Go from a global ctor.
  1400			// Can not throw, because scheduler is not initialized yet.
  1401			write(2, unsafe.Pointer(&earlycgocallback[0]), int32(len(earlycgocallback)))
  1402			exit(1)
  1403		}
  1404	
  1405		// Lock extra list, take head, unlock popped list.
  1406		// nilokay=false is safe here because of the invariant above,
  1407		// that the extra list always contains or will soon contain
  1408		// at least one m.
  1409		mp := lockextra(false)
  1410	
  1411		// Set needextram when we've just emptied the list,
  1412		// so that the eventual call into cgocallbackg will
  1413		// allocate a new m for the extra list. We delay the
  1414		// allocation until then so that it can be done
  1415		// after exitsyscall makes sure it is okay to be
  1416		// running at all (that is, there's no garbage collection
  1417		// running right now).
  1418		mp.needextram = mp.schedlink == 0
  1419		unlockextra(mp.schedlink.ptr())
  1420	
  1421		// Save and block signals before installing g.
  1422		// Once g is installed, any incoming signals will try to execute,
  1423		// but we won't have the sigaltstack settings and other data
  1424		// set up appropriately until the end of minit, which will
  1425		// unblock the signals. This is the same dance as when
  1426		// starting a new m to run Go code via newosproc.
  1427		msigsave(mp)
  1428		sigblock()
  1429	
  1430		// Install g (= m->g0) and set the stack bounds
  1431		// to match the current stack. We don't actually know
  1432		// how big the stack is, like we don't know how big any
  1433		// scheduling stack is, but we assume there's at least 32 kB,
  1434		// which is more than enough for us.
  1435		setg(mp.g0)
  1436		_g_ := getg()
  1437		_g_.stack.hi = uintptr(noescape(unsafe.Pointer(&x))) + 1024
  1438		_g_.stack.lo = uintptr(noescape(unsafe.Pointer(&x))) - 32*1024
  1439		_g_.stackguard0 = _g_.stack.lo + _StackGuard
  1440	
  1441		// Initialize this thread to use the m.
  1442		asminit()
  1443		minit()
  1444	}
  1445	
  1446	var earlycgocallback = []byte("fatal error: cgo callback before cgo call\n")
  1447	
  1448	// newextram allocates m's and puts them on the extra list.
  1449	// It is called with a working local m, so that it can do things
  1450	// like call schedlock and allocate.
  1451	func newextram() {
  1452		c := atomic.Xchg(&extraMWaiters, 0)
  1453		if c > 0 {
  1454			for i := uint32(0); i < c; i++ {
  1455				oneNewExtraM()
  1456			}
  1457		} else {
  1458			// Make sure there is at least one extra M.
  1459			mp := lockextra(true)
  1460			unlockextra(mp)
  1461			if mp == nil {
  1462				oneNewExtraM()
  1463			}
  1464		}
  1465	}
  1466	
  1467	// oneNewExtraM allocates an m and puts it on the extra list.
  1468	func oneNewExtraM() {
  1469		// Create extra goroutine locked to extra m.
  1470		// The goroutine is the context in which the cgo callback will run.
  1471		// The sched.pc will never be returned to, but setting it to
  1472		// goexit makes clear to the traceback routines where
  1473		// the goroutine stack ends.
  1474		mp := allocm(nil, nil)
  1475		gp := malg(4096)
  1476		gp.sched.pc = funcPC(goexit) + sys.PCQuantum
  1477		gp.sched.sp = gp.stack.hi
  1478		gp.sched.sp -= 4 * sys.RegSize // extra space in case of reads slightly beyond frame
  1479		gp.sched.lr = 0
  1480		gp.sched.g = guintptr(unsafe.Pointer(gp))
  1481		gp.syscallpc = gp.sched.pc
  1482		gp.syscallsp = gp.sched.sp
  1483		gp.stktopsp = gp.sched.sp
  1484		gp.gcscanvalid = true // fresh G, so no dequeueRescan necessary
  1485		gp.gcscandone = true
  1486		gp.gcRescan = -1
  1487		// malg returns status as Gidle, change to Gsyscall before adding to allg
  1488		// where GC will see it.
  1489		casgstatus(gp, _Gidle, _Gsyscall)
  1490		gp.m = mp
  1491		mp.curg = gp
  1492		mp.locked = _LockInternal
  1493		mp.lockedg = gp
  1494		gp.lockedm = mp
  1495		gp.goid = int64(atomic.Xadd64(&sched.goidgen, 1))
  1496		if raceenabled {
  1497			gp.racectx = racegostart(funcPC(newextram) + sys.PCQuantum)
  1498		}
  1499		// put on allg for garbage collector
  1500		allgadd(gp)
  1501	
  1502		// Add m to the extra list.
  1503		mnext := lockextra(true)
  1504		mp.schedlink.set(mnext)
  1505		unlockextra(mp)
  1506	}
  1507	
  1508	// dropm is called when a cgo callback has called needm but is now
  1509	// done with the callback and returning back into the non-Go thread.
  1510	// It puts the current m back onto the extra list.
  1511	//
  1512	// The main expense here is the call to signalstack to release the
  1513	// m's signal stack, and then the call to needm on the next callback
  1514	// from this thread. It is tempting to try to save the m for next time,
  1515	// which would eliminate both these costs, but there might not be
  1516	// a next time: the current thread (which Go does not control) might exit.
  1517	// If we saved the m for that thread, there would be an m leak each time
  1518	// such a thread exited. Instead, we acquire and release an m on each
  1519	// call. These should typically not be scheduling operations, just a few
  1520	// atomics, so the cost should be small.
  1521	//
  1522	// TODO(rsc): An alternative would be to allocate a dummy pthread per-thread
  1523	// variable using pthread_key_create. Unlike the pthread keys we already use
  1524	// on OS X, this dummy key would never be read by Go code. It would exist
  1525	// only so that we could register at thread-exit-time destructor.
  1526	// That destructor would put the m back onto the extra list.
  1527	// This is purely a performance optimization. The current version,
  1528	// in which dropm happens on each cgo call, is still correct too.
  1529	// We may have to keep the current version on systems with cgo
  1530	// but without pthreads, like Windows.
  1531	func dropm() {
  1532		// Clear m and g, and return m to the extra list.
  1533		// After the call to setg we can only call nosplit functions
  1534		// with no pointer manipulation.
  1535		mp := getg().m
  1536	
  1537		// Block signals before unminit.
  1538		// Unminit unregisters the signal handling stack (but needs g on some systems).
  1539		// Setg(nil) clears g, which is the signal handler's cue not to run Go handlers.
  1540		// It's important not to try to handle a signal between those two steps.
  1541		sigmask := mp.sigmask
  1542		sigblock()
  1543		unminit()
  1544	
  1545		mnext := lockextra(true)
  1546		mp.schedlink.set(mnext)
  1547	
  1548		setg(nil)
  1549	
  1550		// Commit the release of mp.
  1551		unlockextra(mp)
  1552	
  1553		msigrestore(sigmask)
  1554	}
  1555	
  1556	// A helper function for EnsureDropM.
  1557	func getm() uintptr {
  1558		return uintptr(unsafe.Pointer(getg().m))
  1559	}
  1560	
  1561	var extram uintptr
  1562	var extraMWaiters uint32
  1563	
  1564	// lockextra locks the extra list and returns the list head.
  1565	// The caller must unlock the list by storing a new list head
  1566	// to extram. If nilokay is true, then lockextra will
  1567	// return a nil list head if that's what it finds. If nilokay is false,
  1568	// lockextra will keep waiting until the list head is no longer nil.
  1569	//go:nosplit
  1570	func lockextra(nilokay bool) *m {
  1571		const locked = 1
  1572	
  1573		incr := false
  1574		for {
  1575			old := atomic.Loaduintptr(&extram)
  1576			if old == locked {
  1577				yield := osyield
  1578				yield()
  1579				continue
  1580			}
  1581			if old == 0 && !nilokay {
  1582				if !incr {
  1583					// Add 1 to the number of threads
  1584					// waiting for an M.
  1585					// This is cleared by newextram.
  1586					atomic.Xadd(&extraMWaiters, 1)
  1587					incr = true
  1588				}
  1589				usleep(1)
  1590				continue
  1591			}
  1592			if atomic.Casuintptr(&extram, old, locked) {
  1593				return (*m)(unsafe.Pointer(old))
  1594			}
  1595			yield := osyield
  1596			yield()
  1597			continue
  1598		}
  1599	}
  1600	
  1601	//go:nosplit
  1602	func unlockextra(mp *m) {
  1603		atomic.Storeuintptr(&extram, uintptr(unsafe.Pointer(mp)))
  1604	}
  1605	
  1606	// Create a new m. It will start off with a call to fn, or else the scheduler.
  1607	// fn needs to be static and not a heap allocated closure.
  1608	// May run with m.p==nil, so write barriers are not allowed.
  1609	//go:nowritebarrierrec
  1610	func newm(fn func(), _p_ *p) {
  1611		mp := allocm(_p_, fn)
  1612		mp.nextp.set(_p_)
  1613		mp.sigmask = initSigmask
  1614		if iscgo {
  1615			var ts cgothreadstart
  1616			if _cgo_thread_start == nil {
  1617				throw("_cgo_thread_start missing")
  1618			}
  1619			ts.g.set(mp.g0)
  1620			ts.tls = (*uint64)(unsafe.Pointer(&mp.tls[0]))
  1621			ts.fn = unsafe.Pointer(funcPC(mstart))
  1622			if msanenabled {
  1623				msanwrite(unsafe.Pointer(&ts), unsafe.Sizeof(ts))
  1624			}
  1625			asmcgocall(_cgo_thread_start, unsafe.Pointer(&ts))
  1626			return
  1627		}
  1628		newosproc(mp, unsafe.Pointer(mp.g0.stack.hi))
  1629	}
  1630	
  1631	// Stops execution of the current m until new work is available.
  1632	// Returns with acquired P.
  1633	func stopm() {
  1634		_g_ := getg()
  1635	
  1636		if _g_.m.locks != 0 {
  1637			throw("stopm holding locks")
  1638		}
  1639		if _g_.m.p != 0 {
  1640			throw("stopm holding p")
  1641		}
  1642		if _g_.m.spinning {
  1643			throw("stopm spinning")
  1644		}
  1645	
  1646	retry:
  1647		lock(&sched.lock)
  1648		mput(_g_.m)
  1649		unlock(&sched.lock)
  1650		notesleep(&_g_.m.park)
  1651		noteclear(&_g_.m.park)
  1652		if _g_.m.helpgc != 0 {
  1653			gchelper()
  1654			_g_.m.helpgc = 0
  1655			_g_.m.mcache = nil
  1656			_g_.m.p = 0
  1657			goto retry
  1658		}
  1659		acquirep(_g_.m.nextp.ptr())
  1660		_g_.m.nextp = 0
  1661	}
  1662	
  1663	func mspinning() {
  1664		// startm's caller incremented nmspinning. Set the new M's spinning.
  1665		getg().m.spinning = true
  1666	}
  1667	
  1668	// Schedules some M to run the p (creates an M if necessary).
  1669	// If p==nil, tries to get an idle P, if no idle P's does nothing.
  1670	// May run with m.p==nil, so write barriers are not allowed.
  1671	// If spinning is set, the caller has incremented nmspinning and startm will
  1672	// either decrement nmspinning or set m.spinning in the newly started M.
  1673	//go:nowritebarrierrec
  1674	func startm(_p_ *p, spinning bool) {
  1675		lock(&sched.lock)
  1676		if _p_ == nil {
  1677			_p_ = pidleget()
  1678			if _p_ == nil {
  1679				unlock(&sched.lock)
  1680				if spinning {
  1681					// The caller incremented nmspinning, but there are no idle Ps,
  1682					// so it's okay to just undo the increment and give up.
  1683					if int32(atomic.Xadd(&sched.nmspinning, -1)) < 0 {
  1684						throw("startm: negative nmspinning")
  1685					}
  1686				}
  1687				return
  1688			}
  1689		}
  1690		mp := mget()
  1691		unlock(&sched.lock)
  1692		if mp == nil {
  1693			var fn func()
  1694			if spinning {
  1695				// The caller incremented nmspinning, so set m.spinning in the new M.
  1696				fn = mspinning
  1697			}
  1698			newm(fn, _p_)
  1699			return
  1700		}
  1701		if mp.spinning {
  1702			throw("startm: m is spinning")
  1703		}
  1704		if mp.nextp != 0 {
  1705			throw("startm: m has p")
  1706		}
  1707		if spinning && !runqempty(_p_) {
  1708			throw("startm: p has runnable gs")
  1709		}
  1710		// The caller incremented nmspinning, so set m.spinning in the new M.
  1711		mp.spinning = spinning
  1712		mp.nextp.set(_p_)
  1713		notewakeup(&mp.park)
  1714	}
  1715	
  1716	// Hands off P from syscall or locked M.
  1717	// Always runs without a P, so write barriers are not allowed.
  1718	//go:nowritebarrierrec
  1719	func handoffp(_p_ *p) {
  1720		// handoffp must start an M in any situation where
  1721		// findrunnable would return a G to run on _p_.
  1722	
  1723		// if it has local work, start it straight away
  1724		if !runqempty(_p_) || sched.runqsize != 0 {
  1725			startm(_p_, false)
  1726			return
  1727		}
  1728		// if it has GC work, start it straight away
  1729		if gcBlackenEnabled != 0 && gcMarkWorkAvailable(_p_) {
  1730			startm(_p_, false)
  1731			return
  1732		}
  1733		// no local work, check that there are no spinning/idle M's,
  1734		// otherwise our help is not required
  1735		if atomic.Load(&sched.nmspinning)+atomic.Load(&sched.npidle) == 0 && atomic.Cas(&sched.nmspinning, 0, 1) { // TODO: fast atomic
  1736			startm(_p_, true)
  1737			return
  1738		}
  1739		lock(&sched.lock)
  1740		if sched.gcwaiting != 0 {
  1741			_p_.status = _Pgcstop
  1742			sched.stopwait--
  1743			if sched.stopwait == 0 {
  1744				notewakeup(&sched.stopnote)
  1745			}
  1746			unlock(&sched.lock)
  1747			return
  1748		}
  1749		if _p_.runSafePointFn != 0 && atomic.Cas(&_p_.runSafePointFn, 1, 0) {
  1750			sched.safePointFn(_p_)
  1751			sched.safePointWait--
  1752			if sched.safePointWait == 0 {
  1753				notewakeup(&sched.safePointNote)
  1754			}
  1755		}
  1756		if sched.runqsize != 0 {
  1757			unlock(&sched.lock)
  1758			startm(_p_, false)
  1759			return
  1760		}
  1761		// If this is the last running P and nobody is polling network,
  1762		// need to wakeup another M to poll network.
  1763		if sched.npidle == uint32(gomaxprocs-1) && atomic.Load64(&sched.lastpoll) != 0 {
  1764			unlock(&sched.lock)
  1765			startm(_p_, false)
  1766			return
  1767		}
  1768		pidleput(_p_)
  1769		unlock(&sched.lock)
  1770	}
  1771	
  1772	// Tries to add one more P to execute G's.
  1773	// Called when a G is made runnable (newproc, ready).
  1774	func wakep() {
  1775		// be conservative about spinning threads
  1776		if !atomic.Cas(&sched.nmspinning, 0, 1) {
  1777			return
  1778		}
  1779		startm(nil, true)
  1780	}
  1781	
  1782	// Stops execution of the current m that is locked to a g until the g is runnable again.
  1783	// Returns with acquired P.
  1784	func stoplockedm() {
  1785		_g_ := getg()
  1786	
  1787		if _g_.m.lockedg == nil || _g_.m.lockedg.lockedm != _g_.m {
  1788			throw("stoplockedm: inconsistent locking")
  1789		}
  1790		if _g_.m.p != 0 {
  1791			// Schedule another M to run this p.
  1792			_p_ := releasep()
  1793			handoffp(_p_)
  1794		}
  1795		incidlelocked(1)
  1796		// Wait until another thread schedules lockedg again.
  1797		notesleep(&_g_.m.park)
  1798		noteclear(&_g_.m.park)
  1799		status := readgstatus(_g_.m.lockedg)
  1800		if status&^_Gscan != _Grunnable {
  1801			print("runtime:stoplockedm: g is not Grunnable or Gscanrunnable\n")
  1802			dumpgstatus(_g_)
  1803			throw("stoplockedm: not runnable")
  1804		}
  1805		acquirep(_g_.m.nextp.ptr())
  1806		_g_.m.nextp = 0
  1807	}
  1808	
  1809	// Schedules the locked m to run the locked gp.
  1810	// May run during STW, so write barriers are not allowed.
  1811	//go:nowritebarrierrec
  1812	func startlockedm(gp *g) {
  1813		_g_ := getg()
  1814	
  1815		mp := gp.lockedm
  1816		if mp == _g_.m {
  1817			throw("startlockedm: locked to me")
  1818		}
  1819		if mp.nextp != 0 {
  1820			throw("startlockedm: m has p")
  1821		}
  1822		// directly handoff current P to the locked m
  1823		incidlelocked(-1)
  1824		_p_ := releasep()
  1825		mp.nextp.set(_p_)
  1826		notewakeup(&mp.park)
  1827		stopm()
  1828	}
  1829	
  1830	// Stops the current m for stopTheWorld.
  1831	// Returns when the world is restarted.
  1832	func gcstopm() {
  1833		_g_ := getg()
  1834	
  1835		if sched.gcwaiting == 0 {
  1836			throw("gcstopm: not waiting for gc")
  1837		}
  1838		if _g_.m.spinning {
  1839			_g_.m.spinning = false
  1840			// OK to just drop nmspinning here,
  1841			// startTheWorld will unpark threads as necessary.
  1842			if int32(atomic.Xadd(&sched.nmspinning, -1)) < 0 {
  1843				throw("gcstopm: negative nmspinning")
  1844			}
  1845		}
  1846		_p_ := releasep()
  1847		lock(&sched.lock)
  1848		_p_.status = _Pgcstop
  1849		sched.stopwait--
  1850		if sched.stopwait == 0 {
  1851			notewakeup(&sched.stopnote)
  1852		}
  1853		unlock(&sched.lock)
  1854		stopm()
  1855	}
  1856	
  1857	// Schedules gp to run on the current M.
  1858	// If inheritTime is true, gp inherits the remaining time in the
  1859	// current time slice. Otherwise, it starts a new time slice.
  1860	// Never returns.
  1861	//
  1862	// Write barriers are allowed because this is called immediately after
  1863	// acquiring a P in several places.
  1864	//
  1865	//go:yeswritebarrierrec
  1866	func execute(gp *g, inheritTime bool) {
  1867		_g_ := getg()
  1868	
  1869		casgstatus(gp, _Grunnable, _Grunning)
  1870		gp.waitsince = 0
  1871		gp.preempt = false
  1872		gp.stackguard0 = gp.stack.lo + _StackGuard
  1873		if !inheritTime {
  1874			_g_.m.p.ptr().schedtick++
  1875		}
  1876		_g_.m.curg = gp
  1877		gp.m = _g_.m
  1878	
  1879		// Check whether the profiler needs to be turned on or off.
  1880		hz := sched.profilehz
  1881		if _g_.m.profilehz != hz {
  1882			resetcpuprofiler(hz)
  1883		}
  1884	
  1885		if trace.enabled {
  1886			// GoSysExit has to happen when we have a P, but before GoStart.
  1887			// So we emit it here.
  1888			if gp.syscallsp != 0 && gp.sysblocktraced {
  1889				traceGoSysExit(gp.sysexitticks)
  1890			}
  1891			traceGoStart()
  1892		}
  1893	
  1894		gogo(&gp.sched)
  1895	}
  1896	
  1897	// Finds a runnable goroutine to execute.
  1898	// Tries to steal from other P's, get g from global queue, poll network.
  1899	func findrunnable() (gp *g, inheritTime bool) {
  1900		_g_ := getg()
  1901	
  1902		// The conditions here and in handoffp must agree: if
  1903		// findrunnable would return a G to run, handoffp must start
  1904		// an M.
  1905	
  1906	top:
  1907		_p_ := _g_.m.p.ptr()
  1908		if sched.gcwaiting != 0 {
  1909			gcstopm()
  1910			goto top
  1911		}
  1912		if _p_.runSafePointFn != 0 {
  1913			runSafePointFn()
  1914		}
  1915		if fingwait && fingwake {
  1916			if gp := wakefing(); gp != nil {
  1917				ready(gp, 0, true)
  1918			}
  1919		}
  1920	
  1921		// local runq
  1922		if gp, inheritTime := runqget(_p_); gp != nil {
  1923			return gp, inheritTime
  1924		}
  1925	
  1926		// global runq
  1927		if sched.runqsize != 0 {
  1928			lock(&sched.lock)
  1929			gp := globrunqget(_p_, 0)
  1930			unlock(&sched.lock)
  1931			if gp != nil {
  1932				return gp, false
  1933			}
  1934		}
  1935	
  1936		// Poll network.
  1937		// This netpoll is only an optimization before we resort to stealing.
  1938		// We can safely skip it if there a thread blocked in netpoll already.
  1939		// If there is any kind of logical race with that blocked thread
  1940		// (e.g. it has already returned from netpoll, but does not set lastpoll yet),
  1941		// this thread will do blocking netpoll below anyway.
  1942		if netpollinited() && sched.lastpoll != 0 {
  1943			if gp := netpoll(false); gp != nil { // non-blocking
  1944				// netpoll returns list of goroutines linked by schedlink.
  1945				injectglist(gp.schedlink.ptr())
  1946				casgstatus(gp, _Gwaiting, _Grunnable)
  1947				if trace.enabled {
  1948					traceGoUnpark(gp, 0)
  1949				}
  1950				return gp, false
  1951			}
  1952		}
  1953	
  1954		// Steal work from other P's.
  1955		procs := uint32(gomaxprocs)
  1956		if atomic.Load(&sched.npidle) == procs-1 {
  1957			// Either GOMAXPROCS=1 or everybody, except for us, is idle already.
  1958			// New work can appear from returning syscall/cgocall, network or timers.
  1959			// Neither of that submits to local run queues, so no point in stealing.
  1960			goto stop
  1961		}
  1962		// If number of spinning M's >= number of busy P's, block.
  1963		// This is necessary to prevent excessive CPU consumption
  1964		// when GOMAXPROCS>>1 but the program parallelism is low.
  1965		if !_g_.m.spinning && 2*atomic.Load(&sched.nmspinning) >= procs-atomic.Load(&sched.npidle) {
  1966			goto stop
  1967		}
  1968		if !_g_.m.spinning {
  1969			_g_.m.spinning = true
  1970			atomic.Xadd(&sched.nmspinning, 1)
  1971		}
  1972		for i := 0; i < 4; i++ {
  1973			for enum := stealOrder.start(fastrand()); !enum.done(); enum.next() {
  1974				if sched.gcwaiting != 0 {
  1975					goto top
  1976				}
  1977				stealRunNextG := i > 2 // first look for ready queues with more than 1 g
  1978				if gp := runqsteal(_p_, allp[enum.position()], stealRunNextG); gp != nil {
  1979					return gp, false
  1980				}
  1981			}
  1982		}
  1983	
  1984	stop:
  1985	
  1986		// We have nothing to do. If we're in the GC mark phase, can
  1987		// safely scan and blacken objects, and have work to do, run
  1988		// idle-time marking rather than give up the P.
  1989		if gcBlackenEnabled != 0 && _p_.gcBgMarkWorker != 0 && gcMarkWorkAvailable(_p_) {
  1990			_p_.gcMarkWorkerMode = gcMarkWorkerIdleMode
  1991			gp := _p_.gcBgMarkWorker.ptr()
  1992			casgstatus(gp, _Gwaiting, _Grunnable)
  1993			if trace.enabled {
  1994				traceGoUnpark(gp, 0)
  1995			}
  1996			return gp, false
  1997		}
  1998	
  1999		// return P and block
  2000		lock(&sched.lock)
  2001		if sched.gcwaiting != 0 || _p_.runSafePointFn != 0 {
  2002			unlock(&sched.lock)
  2003			goto top
  2004		}
  2005		if sched.runqsize != 0 {
  2006			gp := globrunqget(_p_, 0)
  2007			unlock(&sched.lock)
  2008			return gp, false
  2009		}
  2010		if releasep() != _p_ {
  2011			throw("findrunnable: wrong p")
  2012		}
  2013		pidleput(_p_)
  2014		unlock(&sched.lock)
  2015	
  2016		// Delicate dance: thread transitions from spinning to non-spinning state,
  2017		// potentially concurrently with submission of new goroutines. We must
  2018		// drop nmspinning first and then check all per-P queues again (with
  2019		// #StoreLoad memory barrier in between). If we do it the other way around,
  2020		// another thread can submit a goroutine after we've checked all run queues
  2021		// but before we drop nmspinning; as the result nobody will unpark a thread
  2022		// to run the goroutine.
  2023		// If we discover new work below, we need to restore m.spinning as a signal
  2024		// for resetspinning to unpark a new worker thread (because there can be more
  2025		// than one starving goroutine). However, if after discovering new work
  2026		// we also observe no idle Ps, it is OK to just park the current thread:
  2027		// the system is fully loaded so no spinning threads are required.
  2028		// Also see "Worker thread parking/unparking" comment at the top of the file.
  2029		wasSpinning := _g_.m.spinning
  2030		if _g_.m.spinning {
  2031			_g_.m.spinning = false
  2032			if int32(atomic.Xadd(&sched.nmspinning, -1)) < 0 {
  2033				throw("findrunnable: negative nmspinning")
  2034			}
  2035		}
  2036	
  2037		// check all runqueues once again
  2038		for i := 0; i < int(gomaxprocs); i++ {
  2039			_p_ := allp[i]
  2040			if _p_ != nil && !runqempty(_p_) {
  2041				lock(&sched.lock)
  2042				_p_ = pidleget()
  2043				unlock(&sched.lock)
  2044				if _p_ != nil {
  2045					acquirep(_p_)
  2046					if wasSpinning {
  2047						_g_.m.spinning = true
  2048						atomic.Xadd(&sched.nmspinning, 1)
  2049					}
  2050					goto top
  2051				}
  2052				break
  2053			}
  2054		}
  2055	
  2056		// Check for idle-priority GC work again.
  2057		if gcBlackenEnabled != 0 && gcMarkWorkAvailable(nil) {
  2058			lock(&sched.lock)
  2059			_p_ = pidleget()
  2060			if _p_ != nil && _p_.gcBgMarkWorker == 0 {
  2061				pidleput(_p_)
  2062				_p_ = nil
  2063			}
  2064			unlock(&sched.lock)
  2065			if _p_ != nil {
  2066				acquirep(_p_)
  2067				if wasSpinning {
  2068					_g_.m.spinning = true
  2069					atomic.Xadd(&sched.nmspinning, 1)
  2070				}
  2071				// Go back to idle GC check.
  2072				goto stop
  2073			}
  2074		}
  2075	
  2076		// poll network
  2077		if netpollinited() && atomic.Xchg64(&sched.lastpoll, 0) != 0 {
  2078			if _g_.m.p != 0 {
  2079				throw("findrunnable: netpoll with p")
  2080			}
  2081			if _g_.m.spinning {
  2082				throw("findrunnable: netpoll with spinning")
  2083			}
  2084			gp := netpoll(true) // block until new work is available
  2085			atomic.Store64(&sched.lastpoll, uint64(nanotime()))
  2086			if gp != nil {
  2087				lock(&sched.lock)
  2088				_p_ = pidleget()
  2089				unlock(&sched.lock)
  2090				if _p_ != nil {
  2091					acquirep(_p_)
  2092					injectglist(gp.schedlink.ptr())
  2093					casgstatus(gp, _Gwaiting, _Grunnable)
  2094					if trace.enabled {
  2095						traceGoUnpark(gp, 0)
  2096					}
  2097					return gp, false
  2098				}
  2099				injectglist(gp)
  2100			}
  2101		}
  2102		stopm()
  2103		goto top
  2104	}
  2105	
  2106	// pollWork returns true if there is non-background work this P could
  2107	// be doing. This is a fairly lightweight check to be used for
  2108	// background work loops, like idle GC. It checks a subset of the
  2109	// conditions checked by the actual scheduler.
  2110	func pollWork() bool {
  2111		if sched.runqsize != 0 {
  2112			return true
  2113		}
  2114		p := getg().m.p.ptr()
  2115		if !runqempty(p) {
  2116			return true
  2117		}
  2118		if netpollinited() && sched.lastpoll != 0 {
  2119			if gp := netpoll(false); gp != nil {
  2120				injectglist(gp)
  2121				return true
  2122			}
  2123		}
  2124		return false
  2125	}
  2126	
  2127	func resetspinning() {
  2128		_g_ := getg()
  2129		if !_g_.m.spinning {
  2130			throw("resetspinning: not a spinning m")
  2131		}
  2132		_g_.m.spinning = false
  2133		nmspinning := atomic.Xadd(&sched.nmspinning, -1)
  2134		if int32(nmspinning) < 0 {
  2135			throw("findrunnable: negative nmspinning")
  2136		}
  2137		// M wakeup policy is deliberately somewhat conservative, so check if we
  2138		// need to wakeup another P here. See "Worker thread parking/unparking"
  2139		// comment at the top of the file for details.
  2140		if nmspinning == 0 && atomic.Load(&sched.npidle) > 0 {
  2141			wakep()
  2142		}
  2143	}
  2144	
  2145	// Injects the list of runnable G's into the scheduler.
  2146	// Can run concurrently with GC.
  2147	func injectglist(glist *g) {
  2148		if glist == nil {
  2149			return
  2150		}
  2151		if trace.enabled {
  2152			for gp := glist; gp != nil; gp = gp.schedlink.ptr() {
  2153				traceGoUnpark(gp, 0)
  2154			}
  2155		}
  2156		lock(&sched.lock)
  2157		var n int
  2158		for n = 0; glist != nil; n++ {
  2159			gp := glist
  2160			glist = gp.schedlink.ptr()
  2161			casgstatus(gp, _Gwaiting, _Grunnable)
  2162			globrunqput(gp)
  2163		}
  2164		unlock(&sched.lock)
  2165		for ; n != 0 && sched.npidle != 0; n-- {
  2166			startm(nil, false)
  2167		}
  2168	}
  2169	
  2170	// One round of scheduler: find a runnable goroutine and execute it.
  2171	// Never returns.
  2172	func schedule() {
  2173		_g_ := getg()
  2174	
  2175		if _g_.m.locks != 0 {
  2176			throw("schedule: holding locks")
  2177		}
  2178	
  2179		if _g_.m.lockedg != nil {
  2180			stoplockedm()
  2181			execute(_g_.m.lockedg, false) // Never returns.
  2182		}
  2183	
  2184	top:
  2185		if sched.gcwaiting != 0 {
  2186			gcstopm()
  2187			goto top
  2188		}
  2189		if _g_.m.p.ptr().runSafePointFn != 0 {
  2190			runSafePointFn()
  2191		}
  2192	
  2193		var gp *g
  2194		var inheritTime bool
  2195		if trace.enabled || trace.shutdown {
  2196			gp = traceReader()
  2197			if gp != nil {
  2198				casgstatus(gp, _Gwaiting, _Grunnable)
  2199				traceGoUnpark(gp, 0)
  2200			}
  2201		}
  2202		if gp == nil && gcBlackenEnabled != 0 {
  2203			gp = gcController.findRunnableGCWorker(_g_.m.p.ptr())
  2204		}
  2205		if gp == nil {
  2206			// Check the global runnable queue once in a while to ensure fairness.
  2207			// Otherwise two goroutines can completely occupy the local runqueue
  2208			// by constantly respawning each other.
  2209			if _g_.m.p.ptr().schedtick%61 == 0 && sched.runqsize > 0 {
  2210				lock(&sched.lock)
  2211				gp = globrunqget(_g_.m.p.ptr(), 1)
  2212				unlock(&sched.lock)
  2213			}
  2214		}
  2215		if gp == nil {
  2216			gp, inheritTime = runqget(_g_.m.p.ptr())
  2217			if gp != nil && _g_.m.spinning {
  2218				throw("schedule: spinning with local work")
  2219			}
  2220		}
  2221		if gp == nil {
  2222			gp, inheritTime = findrunnable() // blocks until work is available
  2223		}
  2224	
  2225		// This thread is going to run a goroutine and is not spinning anymore,
  2226		// so if it was marked as spinning we need to reset it now and potentially
  2227		// start a new spinning M.
  2228		if _g_.m.spinning {
  2229			resetspinning()
  2230		}
  2231	
  2232		if gp.lockedm != nil {
  2233			// Hands off own p to the locked m,
  2234			// then blocks waiting for a new p.
  2235			startlockedm(gp)
  2236			goto top
  2237		}
  2238	
  2239		execute(gp, inheritTime)
  2240	}
  2241	
  2242	// dropg removes the association between m and the current goroutine m->curg (gp for short).
  2243	// Typically a caller sets gp's status away from Grunning and then
  2244	// immediately calls dropg to finish the job. The caller is also responsible
  2245	// for arranging that gp will be restarted using ready at an
  2246	// appropriate time. After calling dropg and arranging for gp to be
  2247	// readied later, the caller can do other work but eventually should
  2248	// call schedule to restart the scheduling of goroutines on this m.
  2249	func dropg() {
  2250		_g_ := getg()
  2251	
  2252		setMNoWB(&_g_.m.curg.m, nil)
  2253		setGNoWB(&_g_.m.curg, nil)
  2254	}
  2255	
  2256	func parkunlock_c(gp *g, lock unsafe.Pointer) bool {
  2257		unlock((*mutex)(lock))
  2258		return true
  2259	}
  2260	
  2261	// park continuation on g0.
  2262	func park_m(gp *g) {
  2263		_g_ := getg()
  2264	
  2265		if trace.enabled {
  2266			traceGoPark(_g_.m.waittraceev, _g_.m.waittraceskip, gp)
  2267		}
  2268	
  2269		casgstatus(gp, _Grunning, _Gwaiting)
  2270		dropg()
  2271	
  2272		if _g_.m.waitunlockf != nil {
  2273			fn := *(*func(*g, unsafe.Pointer) bool)(unsafe.Pointer(&_g_.m.waitunlockf))
  2274			ok := fn(gp, _g_.m.waitlock)
  2275			_g_.m.waitunlockf = nil
  2276			_g_.m.waitlock = nil
  2277			if !ok {
  2278				if trace.enabled {
  2279					traceGoUnpark(gp, 2)
  2280				}
  2281				casgstatus(gp, _Gwaiting, _Grunnable)
  2282				execute(gp, true) // Schedule it back, never returns.
  2283			}
  2284		}
  2285		schedule()
  2286	}
  2287	
  2288	func goschedImpl(gp *g) {
  2289		status := readgstatus(gp)
  2290		if status&^_Gscan != _Grunning {
  2291			dumpgstatus(gp)
  2292			throw("bad g status")
  2293		}
  2294		casgstatus(gp, _Grunning, _Grunnable)
  2295		dropg()
  2296		lock(&sched.lock)
  2297		globrunqput(gp)
  2298		unlock(&sched.lock)
  2299	
  2300		schedule()
  2301	}
  2302	
  2303	// Gosched continuation on g0.
  2304	func gosched_m(gp *g) {
  2305		if trace.enabled {
  2306			traceGoSched()
  2307		}
  2308		goschedImpl(gp)
  2309	}
  2310	
  2311	func gopreempt_m(gp *g) {
  2312		if trace.enabled {
  2313			traceGoPreempt()
  2314		}
  2315		goschedImpl(gp)
  2316	}
  2317	
  2318	// Finishes execution of the current goroutine.
  2319	func goexit1() {
  2320		if raceenabled {
  2321			racegoend()
  2322		}
  2323		if trace.enabled {
  2324			traceGoEnd()
  2325		}
  2326		mcall(goexit0)
  2327	}
  2328	
  2329	// goexit continuation on g0.
  2330	func goexit0(gp *g) {
  2331		_g_ := getg()
  2332	
  2333		casgstatus(gp, _Grunning, _Gdead)
  2334		if isSystemGoroutine(gp) {
  2335			atomic.Xadd(&sched.ngsys, -1)
  2336		}
  2337		gp.m = nil
  2338		gp.lockedm = nil
  2339		_g_.m.lockedg = nil
  2340		gp.paniconfault = false
  2341		gp._defer = nil // should be true already but just in case.
  2342		gp._panic = nil // non-nil for Goexit during panic. points at stack-allocated data.
  2343		gp.writebuf = nil
  2344		gp.waitreason = ""
  2345		gp.param = nil
  2346	
  2347		// Note that gp's stack scan is now "valid" because it has no
  2348		// stack. We could dequeueRescan, but that takes a lock and
  2349		// isn't really necessary.
  2350		gp.gcscanvalid = true
  2351		dropg()
  2352	
  2353		if _g_.m.locked&^_LockExternal != 0 {
  2354			print("invalid m->locked = ", _g_.m.locked, "\n")
  2355			throw("internal lockOSThread error")
  2356		}
  2357		_g_.m.locked = 0
  2358		gfput(_g_.m.p.ptr(), gp)
  2359		schedule()
  2360	}
  2361	
  2362	// save updates getg().sched to refer to pc and sp so that a following
  2363	// gogo will restore pc and sp.
  2364	//
  2365	// save must not have write barriers because invoking a write barrier
  2366	// can clobber getg().sched.
  2367	//
  2368	//go:nosplit
  2369	//go:nowritebarrierrec
  2370	func save(pc, sp uintptr) {
  2371		_g_ := getg()
  2372	
  2373		_g_.sched.pc = pc
  2374		_g_.sched.sp = sp
  2375		_g_.sched.lr = 0
  2376		_g_.sched.ret = 0
  2377		_g_.sched.g = guintptr(unsafe.Pointer(_g_))
  2378		// We need to ensure ctxt is zero, but can't have a write
  2379		// barrier here. However, it should always already be zero.
  2380		// Assert that.
  2381		if _g_.sched.ctxt != nil {
  2382			badctxt()
  2383		}
  2384	}
  2385	
  2386	// The goroutine g is about to enter a system call.
  2387	// Record that it's not using the cpu anymore.
  2388	// This is called only from the go syscall library and cgocall,
  2389	// not from the low-level system calls used by the runtime.
  2390	//
  2391	// Entersyscall cannot split the stack: the gosave must
  2392	// make g->sched refer to the caller's stack segment, because
  2393	// entersyscall is going to return immediately after.
  2394	//
  2395	// Nothing entersyscall calls can split the stack either.
  2396	// We cannot safely move the stack during an active call to syscall,
  2397	// because we do not know which of the uintptr arguments are
  2398	// really pointers (back into the stack).
  2399	// In practice, this means that we make the fast path run through
  2400	// entersyscall doing no-split things, and the slow path has to use systemstack
  2401	// to run bigger things on the system stack.
  2402	//
  2403	// reentersyscall is the entry point used by cgo callbacks, where explicitly
  2404	// saved SP and PC are restored. This is needed when exitsyscall will be called
  2405	// from a function further up in the call stack than the parent, as g->syscallsp
  2406	// must always point to a valid stack frame. entersyscall below is the normal
  2407	// entry point for syscalls, which obtains the SP and PC from the caller.
  2408	//
  2409	// Syscall tracing:
  2410	// At the start of a syscall we emit traceGoSysCall to capture the stack trace.
  2411	// If the syscall does not block, that is it, we do not emit any other events.
  2412	// If the syscall blocks (that is, P is retaken), retaker emits traceGoSysBlock;
  2413	// when syscall returns we emit traceGoSysExit and when the goroutine starts running
  2414	// (potentially instantly, if exitsyscallfast returns true) we emit traceGoStart.
  2415	// To ensure that traceGoSysExit is emitted strictly after traceGoSysBlock,
  2416	// we remember current value of syscalltick in m (_g_.m.syscalltick = _g_.m.p.ptr().syscalltick),
  2417	// whoever emits traceGoSysBlock increments p.syscalltick afterwards;
  2418	// and we wait for the increment before emitting traceGoSysExit.
  2419	// Note that the increment is done even if tracing is not enabled,
  2420	// because tracing can be enabled in the middle of syscall. We don't want the wait to hang.
  2421	//
  2422	//go:nosplit
  2423	func reentersyscall(pc, sp uintptr) {
  2424		_g_ := getg()
  2425	
  2426		// Disable preemption because during this function g is in Gsyscall status,
  2427		// but can have inconsistent g->sched, do not let GC observe it.
  2428		_g_.m.locks++
  2429	
  2430		// Entersyscall must not call any function that might split/grow the stack.
  2431		// (See details in comment above.)
  2432		// Catch calls that might, by replacing the stack guard with something that
  2433		// will trip any stack check and leaving a flag to tell newstack to die.
  2434		_g_.stackguard0 = stackPreempt
  2435		_g_.throwsplit = true
  2436	
  2437		// Leave SP around for GC and traceback.
  2438		save(pc, sp)
  2439		_g_.syscallsp = sp
  2440		_g_.syscallpc = pc
  2441		casgstatus(_g_, _Grunning, _Gsyscall)
  2442		if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp {
  2443			systemstack(func() {
  2444				print("entersyscall inconsistent ", hex(_g_.syscallsp), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n")
  2445				throw("entersyscall")
  2446			})
  2447		}
  2448	
  2449		if trace.enabled {
  2450			systemstack(traceGoSysCall)
  2451			// systemstack itself clobbers g.sched.{pc,sp} and we might
  2452			// need them later when the G is genuinely blocked in a
  2453			// syscall
  2454			save(pc, sp)
  2455		}
  2456	
  2457		if atomic.Load(&sched.sysmonwait) != 0 {
  2458			systemstack(entersyscall_sysmon)
  2459			save(pc, sp)
  2460		}
  2461	
  2462		if _g_.m.p.ptr().runSafePointFn != 0 {
  2463			// runSafePointFn may stack split if run on this stack
  2464			systemstack(runSafePointFn)
  2465			save(pc, sp)
  2466		}
  2467	
  2468		_g_.m.syscalltick = _g_.m.p.ptr().syscalltick
  2469		_g_.sysblocktraced = true
  2470		_g_.m.mcache = nil
  2471		_g_.m.p.ptr().m = 0
  2472		atomic.Store(&_g_.m.p.ptr().status, _Psyscall)
  2473		if sched.gcwaiting != 0 {
  2474			systemstack(entersyscall_gcwait)
  2475			save(pc, sp)
  2476		}
  2477	
  2478		// Goroutines must not split stacks in Gsyscall status (it would corrupt g->sched).
  2479		// We set _StackGuard to StackPreempt so that first split stack check calls morestack.
  2480		// Morestack detects this case and throws.
  2481		_g_.stackguard0 = stackPreempt
  2482		_g_.m.locks--
  2483	}
  2484	
  2485	// Standard syscall entry used by the go syscall library and normal cgo calls.
  2486	//go:nosplit
  2487	func entersyscall(dummy int32) {
  2488		reentersyscall(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy)))
  2489	}
  2490	
  2491	func entersyscall_sysmon() {
  2492		lock(&sched.lock)
  2493		if atomic.Load(&sched.sysmonwait) != 0 {
  2494			atomic.Store(&sched.sysmonwait, 0)
  2495			notewakeup(&sched.sysmonnote)
  2496		}
  2497		unlock(&sched.lock)
  2498	}
  2499	
  2500	func entersyscall_gcwait() {
  2501		_g_ := getg()
  2502		_p_ := _g_.m.p.ptr()
  2503	
  2504		lock(&sched.lock)
  2505		if sched.stopwait > 0 && atomic.Cas(&_p_.status, _Psyscall, _Pgcstop) {
  2506			if trace.enabled {
  2507				traceGoSysBlock(_p_)
  2508				traceProcStop(_p_)
  2509			}
  2510			_p_.syscalltick++
  2511			if sched.stopwait--; sched.stopwait == 0 {
  2512				notewakeup(&sched.stopnote)
  2513			}
  2514		}
  2515		unlock(&sched.lock)
  2516	}
  2517	
  2518	// The same as entersyscall(), but with a hint that the syscall is blocking.
  2519	//go:nosplit
  2520	func entersyscallblock(dummy int32) {
  2521		_g_ := getg()
  2522	
  2523		_g_.m.locks++ // see comment in entersyscall
  2524		_g_.throwsplit = true
  2525		_g_.stackguard0 = stackPreempt // see comment in entersyscall
  2526		_g_.m.syscalltick = _g_.m.p.ptr().syscalltick
  2527		_g_.sysblocktraced = true
  2528		_g_.m.p.ptr().syscalltick++
  2529	
  2530		// Leave SP around for GC and traceback.
  2531		pc := getcallerpc(unsafe.Pointer(&dummy))
  2532		sp := getcallersp(unsafe.Pointer(&dummy))
  2533		save(pc, sp)
  2534		_g_.syscallsp = _g_.sched.sp
  2535		_g_.syscallpc = _g_.sched.pc
  2536		if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp {
  2537			sp1 := sp
  2538			sp2 := _g_.sched.sp
  2539			sp3 := _g_.syscallsp
  2540			systemstack(func() {
  2541				print("entersyscallblock inconsistent ", hex(sp1), " ", hex(sp2), " ", hex(sp3), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n")
  2542				throw("entersyscallblock")
  2543			})
  2544		}
  2545		casgstatus(_g_, _Grunning, _Gsyscall)
  2546		if _g_.syscallsp < _g_.stack.lo || _g_.stack.hi < _g_.syscallsp {
  2547			systemstack(func() {
  2548				print("entersyscallblock inconsistent ", hex(sp), " ", hex(_g_.sched.sp), " ", hex(_g_.syscallsp), " [", hex(_g_.stack.lo), ",", hex(_g_.stack.hi), "]\n")
  2549				throw("entersyscallblock")
  2550			})
  2551		}
  2552	
  2553		systemstack(entersyscallblock_handoff)
  2554	
  2555		// Resave for traceback during blocked call.
  2556		save(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy)))
  2557	
  2558		_g_.m.locks--
  2559	}
  2560	
  2561	func entersyscallblock_handoff() {
  2562		if trace.enabled {
  2563			traceGoSysCall()
  2564			traceGoSysBlock(getg().m.p.ptr())
  2565		}
  2566		handoffp(releasep())
  2567	}
  2568	
  2569	// The goroutine g exited its system call.
  2570	// Arrange for it to run on a cpu again.
  2571	// This is called only from the go syscall library, not
  2572	// from the low-level system calls used by the runtime.
  2573	//
  2574	// Write barriers are not allowed because our P may have been stolen.
  2575	//
  2576	//go:nosplit
  2577	//go:nowritebarrierrec
  2578	func exitsyscall(dummy int32) {
  2579		_g_ := getg()
  2580	
  2581		_g_.m.locks++ // see comment in entersyscall
  2582		if getcallersp(unsafe.Pointer(&dummy)) > _g_.syscallsp {
  2583			// throw calls print which may try to grow the stack,
  2584			// but throwsplit == true so the stack can not be grown;
  2585			// use systemstack to avoid that possible problem.
  2586			systemstack(func() {
  2587				throw("exitsyscall: syscall frame is no longer valid")
  2588			})
  2589		}
  2590	
  2591		_g_.waitsince = 0
  2592		oldp := _g_.m.p.ptr()
  2593		if exitsyscallfast() {
  2594			if _g_.m.mcache == nil {
  2595				throw("lost mcache")
  2596			}
  2597			if trace.enabled {
  2598				if oldp != _g_.m.p.ptr() || _g_.m.syscalltick != _g_.m.p.ptr().syscalltick {
  2599					systemstack(traceGoStart)
  2600				}
  2601			}
  2602			// There's a cpu for us, so we can run.
  2603			_g_.m.p.ptr().syscalltick++
  2604			// We need to cas the status and scan before resuming...
  2605			casgstatus(_g_, _Gsyscall, _Grunning)
  2606	
  2607			// Garbage collector isn't running (since we are),
  2608			// so okay to clear syscallsp.
  2609			_g_.syscallsp = 0
  2610			_g_.m.locks--
  2611			if _g_.preempt {
  2612				// restore the preemption request in case we've cleared it in newstack
  2613				_g_.stackguard0 = stackPreempt
  2614			} else {
  2615				// otherwise restore the real _StackGuard, we've spoiled it in entersyscall/entersyscallblock
  2616				_g_.stackguard0 = _g_.stack.lo + _StackGuard
  2617			}
  2618			_g_.throwsplit = false
  2619			return
  2620		}
  2621	
  2622		_g_.sysexitticks = 0
  2623		if trace.enabled {
  2624			// Wait till traceGoSysBlock event is emitted.
  2625			// This ensures consistency of the trace (the goroutine is started after it is blocked).
  2626			for oldp != nil && oldp.syscalltick == _g_.m.syscalltick {
  2627				osyield()
  2628			}
  2629			// We can't trace syscall exit right now because we don't have a P.
  2630			// Tracing code can invoke write barriers that cannot run without a P.
  2631			// So instead we remember the syscall exit time and emit the event
  2632			// in execute when we have a P.
  2633			_g_.sysexitticks = cputicks()
  2634		}
  2635	
  2636		_g_.m.locks--
  2637	
  2638		// Call the scheduler.
  2639		mcall(exitsyscall0)
  2640	
  2641		if _g_.m.mcache == nil {
  2642			throw("lost mcache")
  2643		}
  2644	
  2645		// Scheduler returned, so we're allowed to run now.
  2646		// Delete the syscallsp information that we left for
  2647		// the garbage collector during the system call.
  2648		// Must wait until now because until gosched returns
  2649		// we don't know for sure that the garbage collector
  2650		// is not running.
  2651		_g_.syscallsp = 0
  2652		_g_.m.p.ptr().syscalltick++
  2653		_g_.throwsplit = false
  2654	}
  2655	
  2656	//go:nosplit
  2657	func exitsyscallfast() bool {
  2658		_g_ := getg()
  2659	
  2660		// Freezetheworld sets stopwait but does not retake P's.
  2661		if sched.stopwait == freezeStopWait {
  2662			_g_.m.mcache = nil
  2663			_g_.m.p = 0
  2664			return false
  2665		}
  2666	
  2667		// Try to re-acquire the last P.
  2668		if _g_.m.p != 0 && _g_.m.p.ptr().status == _Psyscall && atomic.Cas(&_g_.m.p.ptr().status, _Psyscall, _Prunning) {
  2669			// There's a cpu for us, so we can run.
  2670			exitsyscallfast_reacquired()
  2671			return true
  2672		}
  2673	
  2674		// Try to get any other idle P.
  2675		oldp := _g_.m.p.ptr()
  2676		_g_.m.mcache = nil
  2677		_g_.m.p = 0
  2678		if sched.pidle != 0 {
  2679			var ok bool
  2680			systemstack(func() {
  2681				ok = exitsyscallfast_pidle()
  2682				if ok && trace.enabled {
  2683					if oldp != nil {
  2684						// Wait till traceGoSysBlock event is emitted.
  2685						// This ensures consistency of the trace (the goroutine is started after it is blocked).
  2686						for oldp.syscalltick == _g_.m.syscalltick {
  2687							osyield()
  2688						}
  2689					}
  2690					traceGoSysExit(0)
  2691				}
  2692			})
  2693			if ok {
  2694				return true
  2695			}
  2696		}
  2697		return false
  2698	}
  2699	
  2700	// exitsyscallfast_reacquired is the exitsyscall path on which this G
  2701	// has successfully reacquired the P it was running on before the
  2702	// syscall.
  2703	//
  2704	// This function is allowed to have write barriers because exitsyscall
  2705	// has acquired a P at this point.
  2706	//
  2707	//go:yeswritebarrierrec
  2708	//go:nosplit
  2709	func exitsyscallfast_reacquired() {
  2710		_g_ := getg()
  2711		_g_.m.mcache = _g_.m.p.ptr().mcache
  2712		_g_.m.p.ptr().m.set(_g_.m)
  2713		if _g_.m.syscalltick != _g_.m.p.ptr().syscalltick {
  2714			if trace.enabled {
  2715				// The p was retaken and then enter into syscall again (since _g_.m.syscalltick has changed).
  2716				// traceGoSysBlock for this syscall was already emitted,
  2717				// but here we effectively retake the p from the new syscall running on the same p.
  2718				systemstack(func() {
  2719					// Denote blocking of the new syscall.
  2720					traceGoSysBlock(_g_.m.p.ptr())
  2721					// Denote completion of the current syscall.
  2722					traceGoSysExit(0)
  2723				})
  2724			}
  2725			_g_.m.p.ptr().syscalltick++
  2726		}
  2727	}
  2728	
  2729	func exitsyscallfast_pidle() bool {
  2730		lock(&sched.lock)
  2731		_p_ := pidleget()
  2732		if _p_ != nil && atomic.Load(&sched.sysmonwait) != 0 {
  2733			atomic.Store(&sched.sysmonwait, 0)
  2734			notewakeup(&sched.sysmonnote)
  2735		}
  2736		unlock(&sched.lock)
  2737		if _p_ != nil {
  2738			acquirep(_p_)
  2739			return true
  2740		}
  2741		return false
  2742	}
  2743	
  2744	// exitsyscall slow path on g0.
  2745	// Failed to acquire P, enqueue gp as runnable.
  2746	//
  2747	//go:nowritebarrierrec
  2748	func exitsyscall0(gp *g) {
  2749		_g_ := getg()
  2750	
  2751		casgstatus(gp, _Gsyscall, _Grunnable)
  2752		dropg()
  2753		lock(&sched.lock)
  2754		_p_ := pidleget()
  2755		if _p_ == nil {
  2756			globrunqput(gp)
  2757		} else if atomic.Load(&sched.sysmonwait) != 0 {
  2758			atomic.Store(&sched.sysmonwait, 0)
  2759			notewakeup(&sched.sysmonnote)
  2760		}
  2761		unlock(&sched.lock)
  2762		if _p_ != nil {
  2763			acquirep(_p_)
  2764			execute(gp, false) // Never returns.
  2765		}
  2766		if _g_.m.lockedg != nil {
  2767			// Wait until another thread schedules gp and so m again.
  2768			stoplockedm()
  2769			execute(gp, false) // Never returns.
  2770		}
  2771		stopm()
  2772		schedule() // Never returns.
  2773	}
  2774	
  2775	func beforefork() {
  2776		gp := getg().m.curg
  2777	
  2778		// Fork can hang if preempted with signals frequently enough (see issue 5517).
  2779		// Ensure that we stay on the same M where we disable profiling.
  2780		gp.m.locks++
  2781		if gp.m.profilehz != 0 {
  2782			resetcpuprofiler(0)
  2783		}
  2784	
  2785		// This function is called before fork in syscall package.
  2786		// Code between fork and exec must not allocate memory nor even try to grow stack.
  2787		// Here we spoil g->_StackGuard to reliably detect any attempts to grow stack.
  2788		// runtime_AfterFork will undo this in parent process, but not in child.
  2789		gp.stackguard0 = stackFork
  2790	}
  2791	
  2792	// Called from syscall package before fork.
  2793	//go:linkname syscall_runtime_BeforeFork syscall.runtime_BeforeFork
  2794	//go:nosplit
  2795	func syscall_runtime_BeforeFork() {
  2796		systemstack(beforefork)
  2797	}
  2798	
  2799	func afterfork() {
  2800		gp := getg().m.curg
  2801	
  2802		// See the comment in beforefork.
  2803		gp.stackguard0 = gp.stack.lo + _StackGuard
  2804	
  2805		hz := sched.profilehz
  2806		if hz != 0 {
  2807			resetcpuprofiler(hz)
  2808		}
  2809		gp.m.locks--
  2810	}
  2811	
  2812	// Called from syscall package after fork in parent.
  2813	//go:linkname syscall_runtime_AfterFork syscall.runtime_AfterFork
  2814	//go:nosplit
  2815	func syscall_runtime_AfterFork() {
  2816		systemstack(afterfork)
  2817	}
  2818	
  2819	// Allocate a new g, with a stack big enough for stacksize bytes.
  2820	func malg(stacksize int32) *g {
  2821		newg := new(g)
  2822		if stacksize >= 0 {
  2823			stacksize = round2(_StackSystem + stacksize)
  2824			systemstack(func() {
  2825				newg.stack, newg.stkbar = stackalloc(uint32(stacksize))
  2826			})
  2827			newg.stackguard0 = newg.stack.lo + _StackGuard
  2828			newg.stackguard1 = ^uintptr(0)
  2829			newg.stackAlloc = uintptr(stacksize)
  2830		}
  2831		return newg
  2832	}
  2833	
  2834	// Create a new g running fn with siz bytes of arguments.
  2835	// Put it on the queue of g's waiting to run.
  2836	// The compiler turns a go statement into a call to this.
  2837	// Cannot split the stack because it assumes that the arguments
  2838	// are available sequentially after &fn; they would not be
  2839	// copied if a stack split occurred.
  2840	//go:nosplit
  2841	func newproc(siz int32, fn *funcval) {
  2842		argp := add(unsafe.Pointer(&fn), sys.PtrSize)
  2843		pc := getcallerpc(unsafe.Pointer(&siz))
  2844		systemstack(func() {
  2845			newproc1(fn, (*uint8)(argp), siz, 0, pc)
  2846		})
  2847	}
  2848	
  2849	// Create a new g running fn with narg bytes of arguments starting
  2850	// at argp and returning nret bytes of results.  callerpc is the
  2851	// address of the go statement that created this. The new g is put
  2852	// on the queue of g's waiting to run.
  2853	func newproc1(fn *funcval, argp *uint8, narg int32, nret int32, callerpc uintptr) *g {
  2854		_g_ := getg()
  2855	
  2856		if fn == nil {
  2857			_g_.m.throwing = -1 // do not dump full stacks
  2858			throw("go of nil func value")
  2859		}
  2860		_g_.m.locks++ // disable preemption because it can be holding p in a local var
  2861		siz := narg + nret
  2862		siz = (siz + 7) &^ 7
  2863	
  2864		// We could allocate a larger initial stack if necessary.
  2865		// Not worth it: this is almost always an error.
  2866		// 4*sizeof(uintreg): extra space added below
  2867		// sizeof(uintreg): caller's LR (arm) or return address (x86, in gostartcall).
  2868		if siz >= _StackMin-4*sys.RegSize-sys.RegSize {
  2869			throw("newproc: function arguments too large for new goroutine")
  2870		}
  2871	
  2872		_p_ := _g_.m.p.ptr()
  2873		newg := gfget(_p_)
  2874		if newg == nil {
  2875			newg = malg(_StackMin)
  2876			casgstatus(newg, _Gidle, _Gdead)
  2877			newg.gcRescan = -1
  2878			allgadd(newg) // publishes with a g->status of Gdead so GC scanner doesn't look at uninitialized stack.
  2879		}
  2880		if newg.stack.hi == 0 {
  2881			throw("newproc1: newg missing stack")
  2882		}
  2883	
  2884		if readgstatus(newg) != _Gdead {
  2885			throw("newproc1: new g is not Gdead")
  2886		}
  2887	
  2888		totalSize := 4*sys.RegSize + uintptr(siz) + sys.MinFrameSize // extra space in case of reads slightly beyond frame
  2889		totalSize += -totalSize & (sys.SpAlign - 1)                  // align to spAlign
  2890		sp := newg.stack.hi - totalSize
  2891		spArg := sp
  2892		if usesLR {
  2893			// caller's LR
  2894			*(*uintptr)(unsafe.Pointer(sp)) = 0
  2895			prepGoExitFrame(sp)
  2896			spArg += sys.MinFrameSize
  2897		}
  2898		if narg > 0 {
  2899			memmove(unsafe.Pointer(spArg), unsafe.Pointer(argp), uintptr(narg))
  2900			// This is a stack-to-stack copy. If write barriers
  2901			// are enabled and the source stack is grey (the
  2902			// destination is always black), then perform a
  2903			// barrier copy. We do this *after* the memmove
  2904			// because the destination stack may have garbage on
  2905			// it.
  2906			if writeBarrier.needed && !_g_.m.curg.gcscandone {
  2907				f := findfunc(fn.fn)
  2908				stkmap := (*stackmap)(funcdata(f, _FUNCDATA_ArgsPointerMaps))
  2909				// We're in the prologue, so it's always stack map index 0.
  2910				bv := stackmapdata(stkmap, 0)
  2911				bulkBarrierBitmap(spArg, spArg, uintptr(narg), 0, bv.bytedata)
  2912			}
  2913		}
  2914	
  2915		memclrNoHeapPointers(unsafe.Pointer(&newg.sched), unsafe.Sizeof(newg.sched))
  2916		newg.sched.sp = sp
  2917		newg.stktopsp = sp
  2918		newg.sched.pc = funcPC(goexit) + sys.PCQuantum // +PCQuantum so that previous instruction is in same function
  2919		newg.sched.g = guintptr(unsafe.Pointer(newg))
  2920		gostartcallfn(&newg.sched, fn)
  2921		newg.gopc = callerpc
  2922		newg.startpc = fn.fn
  2923		if isSystemGoroutine(newg) {
  2924			atomic.Xadd(&sched.ngsys, +1)
  2925		}
  2926		// The stack is dirty from the argument frame, so queue it for
  2927		// scanning. Do this before setting it to runnable so we still
  2928		// own the G. If we're recycling a G, it may already be on the
  2929		// rescan list.
  2930		if newg.gcRescan == -1 {
  2931			queueRescan(newg)
  2932		} else {
  2933			// The recycled G is already on the rescan list. Just
  2934			// mark the stack dirty.
  2935			newg.gcscanvalid = false
  2936		}
  2937		casgstatus(newg, _Gdead, _Grunnable)
  2938	
  2939		if _p_.goidcache == _p_.goidcacheend {
  2940			// Sched.goidgen is the last allocated id,
  2941			// this batch must be [sched.goidgen+1, sched.goidgen+GoidCacheBatch].
  2942			// At startup sched.goidgen=0, so main goroutine receives goid=1.
  2943			_p_.goidcache = atomic.Xadd64(&sched.goidgen, _GoidCacheBatch)
  2944			_p_.goidcache -= _GoidCacheBatch - 1
  2945			_p_.goidcacheend = _p_.goidcache + _GoidCacheBatch
  2946		}
  2947		newg.goid = int64(_p_.goidcache)
  2948		_p_.goidcache++
  2949		if raceenabled {
  2950			newg.racectx = racegostart(callerpc)
  2951		}
  2952		if trace.enabled {
  2953			traceGoCreate(newg, newg.startpc)
  2954		}
  2955		runqput(_p_, newg, true)
  2956	
  2957		if atomic.Load(&sched.npidle) != 0 && atomic.Load(&sched.nmspinning) == 0 && runtimeInitTime != 0 {
  2958			wakep()
  2959		}
  2960		_g_.m.locks--
  2961		if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
  2962			_g_.stackguard0 = stackPreempt
  2963		}
  2964		return newg
  2965	}
  2966	
  2967	// Put on gfree list.
  2968	// If local list is too long, transfer a batch to the global list.
  2969	func gfput(_p_ *p, gp *g) {
  2970		if readgstatus(gp) != _Gdead {
  2971			throw("gfput: bad status (not Gdead)")
  2972		}
  2973	
  2974		stksize := gp.stackAlloc
  2975	
  2976		if stksize != _FixedStack {
  2977			// non-standard stack size - free it.
  2978			stackfree(gp.stack, gp.stackAlloc)
  2979			gp.stack.lo = 0
  2980			gp.stack.hi = 0
  2981			gp.stackguard0 = 0
  2982			gp.stkbar = nil
  2983			gp.stkbarPos = 0
  2984		} else {
  2985			// Reset stack barriers.
  2986			gp.stkbar = gp.stkbar[:0]
  2987			gp.stkbarPos = 0
  2988		}
  2989	
  2990		gp.schedlink.set(_p_.gfree)
  2991		_p_.gfree = gp
  2992		_p_.gfreecnt++
  2993		if _p_.gfreecnt >= 64 {
  2994			lock(&sched.gflock)
  2995			for _p_.gfreecnt >= 32 {
  2996				_p_.gfreecnt--
  2997				gp = _p_.gfree
  2998				_p_.gfree = gp.schedlink.ptr()
  2999				if gp.stack.lo == 0 {
  3000					gp.schedlink.set(sched.gfreeNoStack)
  3001					sched.gfreeNoStack = gp
  3002				} else {
  3003					gp.schedlink.set(sched.gfreeStack)
  3004					sched.gfreeStack = gp
  3005				}
  3006				sched.ngfree++
  3007			}
  3008			unlock(&sched.gflock)
  3009		}
  3010	}
  3011	
  3012	// Get from gfree list.
  3013	// If local list is empty, grab a batch from global list.
  3014	func gfget(_p_ *p) *g {
  3015	retry:
  3016		gp := _p_.gfree
  3017		if gp == nil && (sched.gfreeStack != nil || sched.gfreeNoStack != nil) {
  3018			lock(&sched.gflock)
  3019			for _p_.gfreecnt < 32 {
  3020				if sched.gfreeStack != nil {
  3021					// Prefer Gs with stacks.
  3022					gp = sched.gfreeStack
  3023					sched.gfreeStack = gp.schedlink.ptr()
  3024				} else if sched.gfreeNoStack != nil {
  3025					gp = sched.gfreeNoStack
  3026					sched.gfreeNoStack = gp.schedlink.ptr()
  3027				} else {
  3028					break
  3029				}
  3030				_p_.gfreecnt++
  3031				sched.ngfree--
  3032				gp.schedlink.set(_p_.gfree)
  3033				_p_.gfree = gp
  3034			}
  3035			unlock(&sched.gflock)
  3036			goto retry
  3037		}
  3038		if gp != nil {
  3039			_p_.gfree = gp.schedlink.ptr()
  3040			_p_.gfreecnt--
  3041			if gp.stack.lo == 0 {
  3042				// Stack was deallocated in gfput. Allocate a new one.
  3043				systemstack(func() {
  3044					gp.stack, gp.stkbar = stackalloc(_FixedStack)
  3045				})
  3046				gp.stackguard0 = gp.stack.lo + _StackGuard
  3047				gp.stackAlloc = _FixedStack
  3048			} else {
  3049				if raceenabled {
  3050					racemalloc(unsafe.Pointer(gp.stack.lo), gp.stackAlloc)
  3051				}
  3052				if msanenabled {
  3053					msanmalloc(unsafe.Pointer(gp.stack.lo), gp.stackAlloc)
  3054				}
  3055			}
  3056		}
  3057		return gp
  3058	}
  3059	
  3060	// Purge all cached G's from gfree list to the global list.
  3061	func gfpurge(_p_ *p) {
  3062		lock(&sched.gflock)
  3063		for _p_.gfreecnt != 0 {
  3064			_p_.gfreecnt--
  3065			gp := _p_.gfree
  3066			_p_.gfree = gp.schedlink.ptr()
  3067			if gp.stack.lo == 0 {
  3068				gp.schedlink.set(sched.gfreeNoStack)
  3069				sched.gfreeNoStack = gp
  3070			} else {
  3071				gp.schedlink.set(sched.gfreeStack)
  3072				sched.gfreeStack = gp
  3073			}
  3074			sched.ngfree++
  3075		}
  3076		unlock(&sched.gflock)
  3077	}
  3078	
  3079	// Breakpoint executes a breakpoint trap.
  3080	func Breakpoint() {
  3081		breakpoint()
  3082	}
  3083	
  3084	// dolockOSThread is called by LockOSThread and lockOSThread below
  3085	// after they modify m.locked. Do not allow preemption during this call,
  3086	// or else the m might be different in this function than in the caller.
  3087	//go:nosplit
  3088	func dolockOSThread() {
  3089		_g_ := getg()
  3090		_g_.m.lockedg = _g_
  3091		_g_.lockedm = _g_.m
  3092	}
  3093	
  3094	//go:nosplit
  3095	
  3096	// LockOSThread wires the calling goroutine to its current operating system thread.
  3097	// Until the calling goroutine exits or calls UnlockOSThread, it will always
  3098	// execute in that thread, and no other goroutine can.
  3099	func LockOSThread() {
  3100		getg().m.locked |= _LockExternal
  3101		dolockOSThread()
  3102	}
  3103	
  3104	//go:nosplit
  3105	func lockOSThread() {
  3106		getg().m.locked += _LockInternal
  3107		dolockOSThread()
  3108	}
  3109	
  3110	// dounlockOSThread is called by UnlockOSThread and unlockOSThread below
  3111	// after they update m->locked. Do not allow preemption during this call,
  3112	// or else the m might be in different in this function than in the caller.
  3113	//go:nosplit
  3114	func dounlockOSThread() {
  3115		_g_ := getg()
  3116		if _g_.m.locked != 0 {
  3117			return
  3118		}
  3119		_g_.m.lockedg = nil
  3120		_g_.lockedm = nil
  3121	}
  3122	
  3123	//go:nosplit
  3124	
  3125	// UnlockOSThread unwires the calling goroutine from its fixed operating system thread.
  3126	// If the calling goroutine has not called LockOSThread, UnlockOSThread is a no-op.
  3127	func UnlockOSThread() {
  3128		getg().m.locked &^= _LockExternal
  3129		dounlockOSThread()
  3130	}
  3131	
  3132	//go:nosplit
  3133	func unlockOSThread() {
  3134		_g_ := getg()
  3135		if _g_.m.locked < _LockInternal {
  3136			systemstack(badunlockosthread)
  3137		}
  3138		_g_.m.locked -= _LockInternal
  3139		dounlockOSThread()
  3140	}
  3141	
  3142	func badunlockosthread() {
  3143		throw("runtime: internal error: misuse of lockOSThread/unlockOSThread")
  3144	}
  3145	
  3146	func gcount() int32 {
  3147		n := int32(allglen) - sched.ngfree - int32(atomic.Load(&sched.ngsys))
  3148		for i := 0; ; i++ {
  3149			_p_ := allp[i]
  3150			if _p_ == nil {
  3151				break
  3152			}
  3153			n -= _p_.gfreecnt
  3154		}
  3155	
  3156		// All these variables can be changed concurrently, so the result can be inconsistent.
  3157		// But at least the current goroutine is running.
  3158		if n < 1 {
  3159			n = 1
  3160		}
  3161		return n
  3162	}
  3163	
  3164	func mcount() int32 {
  3165		return sched.mcount
  3166	}
  3167	
  3168	var prof struct {
  3169		lock uint32
  3170		hz   int32
  3171	}
  3172	
  3173	func _System()       { _System() }
  3174	func _ExternalCode() { _ExternalCode() }
  3175	func _GC()           { _GC() }
  3176	
  3177	// Called if we receive a SIGPROF signal.
  3178	// Called by the signal handler, may run during STW.
  3179	//go:nowritebarrierrec
  3180	func sigprof(pc, sp, lr uintptr, gp *g, mp *m) {
  3181		if prof.hz == 0 {
  3182			return
  3183		}
  3184	
  3185		// Profiling runs concurrently with GC, so it must not allocate.
  3186		// Set a trap in case the code does allocate.
  3187		// Note that on windows, one thread takes profiles of all the
  3188		// other threads, so mp is usually not getg().m.
  3189		// In fact mp may not even be stopped.
  3190		// See golang.org/issue/17165.
  3191		getg().m.mallocing++
  3192	
  3193		// Define that a "user g" is a user-created goroutine, and a "system g"
  3194		// is one that is m->g0 or m->gsignal.
  3195		//
  3196		// We might be interrupted for profiling halfway through a
  3197		// goroutine switch. The switch involves updating three (or four) values:
  3198		// g, PC, SP, and (on arm) LR. The PC must be the last to be updated,
  3199		// because once it gets updated the new g is running.
  3200		//
  3201		// When switching from a user g to a system g, LR is not considered live,
  3202		// so the update only affects g, SP, and PC. Since PC must be last, there
  3203		// the possible partial transitions in ordinary execution are (1) g alone is updated,
  3204		// (2) both g and SP are updated, and (3) SP alone is updated.
  3205		// If SP or g alone is updated, we can detect the partial transition by checking
  3206		// whether the SP is within g's stack bounds. (We could also require that SP
  3207		// be changed only after g, but the stack bounds check is needed by other
  3208		// cases, so there is no need to impose an additional requirement.)
  3209		//
  3210		// There is one exceptional transition to a system g, not in ordinary execution.
  3211		// When a signal arrives, the operating system starts the signal handler running
  3212		// with an updated PC and SP. The g is updated last, at the beginning of the
  3213		// handler. There are two reasons this is okay. First, until g is updated the
  3214		// g and SP do not match, so the stack bounds check detects the partial transition.
  3215		// Second, signal handlers currently run with signals disabled, so a profiling
  3216		// signal cannot arrive during the handler.
  3217		//
  3218		// When switching from a system g to a user g, there are three possibilities.
  3219		//
  3220		// First, it may be that the g switch has no PC update, because the SP
  3221		// either corresponds to a user g throughout (as in asmcgocall)
  3222		// or because it has been arranged to look like a user g frame
  3223		// (as in cgocallback_gofunc). In this case, since the entire
  3224		// transition is a g+SP update, a partial transition updating just one of
  3225		// those will be detected by the stack bounds check.
  3226		//
  3227		// Second, when returning from a signal handler, the PC and SP updates
  3228		// are performed by the operating system in an atomic update, so the g
  3229		// update must be done before them. The stack bounds check detects
  3230		// the partial transition here, and (again) signal handlers run with signals
  3231		// disabled, so a profiling signal cannot arrive then anyway.
  3232		//
  3233		// Third, the common case: it may be that the switch updates g, SP, and PC
  3234		// separately. If the PC is within any of the functions that does this,
  3235		// we don't ask for a traceback. C.F. the function setsSP for more about this.
  3236		//
  3237		// There is another apparently viable approach, recorded here in case
  3238		// the "PC within setsSP function" check turns out not to be usable.
  3239		// It would be possible to delay the update of either g or SP until immediately
  3240		// before the PC update instruction. Then, because of the stack bounds check,
  3241		// the only problematic interrupt point is just before that PC update instruction,
  3242		// and the sigprof handler can detect that instruction and simulate stepping past
  3243		// it in order to reach a consistent state. On ARM, the update of g must be made
  3244		// in two places (in R10 and also in a TLS slot), so the delayed update would
  3245		// need to be the SP update. The sigprof handler must read the instruction at
  3246		// the current PC and if it was the known instruction (for example, JMP BX or
  3247		// MOV R2, PC), use that other register in place of the PC value.
  3248		// The biggest drawback to this solution is that it requires that we can tell
  3249		// whether it's safe to read from the memory pointed at by PC.
  3250		// In a correct program, we can test PC == nil and otherwise read,
  3251		// but if a profiling signal happens at the instant that a program executes
  3252		// a bad jump (before the program manages to handle the resulting fault)
  3253		// the profiling handler could fault trying to read nonexistent memory.
  3254		//
  3255		// To recap, there are no constraints on the assembly being used for the
  3256		// transition. We simply require that g and SP match and that the PC is not
  3257		// in gogo.
  3258		traceback := true
  3259		if gp == nil || sp < gp.stack.lo || gp.stack.hi < sp || setsSP(pc) {
  3260			traceback = false
  3261		}
  3262		var stk [maxCPUProfStack]uintptr
  3263		var haveStackLock *g
  3264		n := 0
  3265		if mp.ncgo > 0 && mp.curg != nil && mp.curg.syscallpc != 0 && mp.curg.syscallsp != 0 {
  3266			cgoOff := 0
  3267			// Check cgoCallersUse to make sure that we are not
  3268			// interrupting other code that is fiddling with
  3269			// cgoCallers.  We are running in a signal handler
  3270			// with all signals blocked, so we don't have to worry
  3271			// about any other code interrupting us.
  3272			if atomic.Load(&mp.cgoCallersUse) == 0 && mp.cgoCallers != nil && mp.cgoCallers[0] != 0 {
  3273				for cgoOff < len(mp.cgoCallers) && mp.cgoCallers[cgoOff] != 0 {
  3274					cgoOff++
  3275				}
  3276				copy(stk[:], mp.cgoCallers[:cgoOff])
  3277				mp.cgoCallers[0] = 0
  3278			}
  3279	
  3280			// Collect Go stack that leads to the cgo call.
  3281			if gcTryLockStackBarriers(mp.curg) {
  3282				haveStackLock = mp.curg
  3283				n = gentraceback(mp.curg.syscallpc, mp.curg.syscallsp, 0, mp.curg, 0, &stk[cgoOff], len(stk)-cgoOff, nil, nil, 0)
  3284			}
  3285		} else if traceback {
  3286			var flags uint = _TraceTrap
  3287			if gp.m.curg != nil && gcTryLockStackBarriers(gp.m.curg) {
  3288				// It's safe to traceback the user stack.
  3289				haveStackLock = gp.m.curg
  3290				flags |= _TraceJumpStack
  3291			}
  3292			// Traceback is safe if we're on the system stack (if
  3293			// necessary, flags will stop it before switching to
  3294			// the user stack), or if we locked the user stack.
  3295			if gp != gp.m.curg || haveStackLock != nil {
  3296				n = gentraceback(pc, sp, lr, gp, 0, &stk[0], len(stk), nil, nil, flags)
  3297			}
  3298		}
  3299		if haveStackLock != nil {
  3300			gcUnlockStackBarriers(haveStackLock)
  3301		}
  3302	
  3303		if n <= 0 {
  3304			// Normal traceback is impossible or has failed.
  3305			// See if it falls into several common cases.
  3306			n = 0
  3307			if GOOS == "windows" && mp.libcallg != 0 && mp.libcallpc != 0 && mp.libcallsp != 0 {
  3308				// Libcall, i.e. runtime syscall on windows.
  3309				// Collect Go stack that leads to the call.
  3310				if gcTryLockStackBarriers(mp.libcallg.ptr()) {
  3311					n = gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg.ptr(), 0, &stk[0], len(stk), nil, nil, 0)
  3312					gcUnlockStackBarriers(mp.libcallg.ptr())
  3313				}
  3314			}
  3315			if n == 0 {
  3316				// If all of the above has failed, account it against abstract "System" or "GC".
  3317				n = 2
  3318				// "ExternalCode" is better than "etext".
  3319				if pc > firstmoduledata.etext {
  3320					pc = funcPC(_ExternalCode) + sys.PCQuantum
  3321				}
  3322				stk[0] = pc
  3323				if mp.preemptoff != "" || mp.helpgc != 0 {
  3324					stk[1] = funcPC(_GC) + sys.PCQuantum
  3325				} else {
  3326					stk[1] = funcPC(_System) + sys.PCQuantum
  3327				}
  3328			}
  3329		}
  3330	
  3331		if prof.hz != 0 {
  3332			// Simple cas-lock to coordinate with setcpuprofilerate.
  3333			for !atomic.Cas(&prof.lock, 0, 1) {
  3334				osyield()
  3335			}
  3336			if prof.hz != 0 {
  3337				cpuprof.add(stk[:n])
  3338			}
  3339			atomic.Store(&prof.lock, 0)
  3340		}
  3341		getg().m.mallocing--
  3342	}
  3343	
  3344	// If the signal handler receives a SIGPROF signal on a non-Go thread,
  3345	// it tries to collect a traceback into sigprofCallers.
  3346	// sigprofCallersUse is set to non-zero while sigprofCallers holds a traceback.
  3347	var sigprofCallers cgoCallers
  3348	var sigprofCallersUse uint32
  3349	
  3350	// sigprofNonGo is called if we receive a SIGPROF signal on a non-Go thread,
  3351	// and the signal handler collected a stack trace in sigprofCallers.
  3352	// When this is called, sigprofCallersUse will be non-zero.
  3353	// g is nil, and what we can do is very limited.
  3354	//go:nosplit
  3355	//go:nowritebarrierrec
  3356	func sigprofNonGo() {
  3357		if prof.hz != 0 {
  3358			n := 0
  3359			for n < len(sigprofCallers) && sigprofCallers[n] != 0 {
  3360				n++
  3361			}
  3362	
  3363			// Simple cas-lock to coordinate with setcpuprofilerate.
  3364			for !atomic.Cas(&prof.lock, 0, 1) {
  3365				osyield()
  3366			}
  3367			if prof.hz != 0 {
  3368				cpuprof.addNonGo(sigprofCallers[:n])
  3369			}
  3370			atomic.Store(&prof.lock, 0)
  3371		}
  3372	
  3373		atomic.Store(&sigprofCallersUse, 0)
  3374	}
  3375	
  3376	// sigprofNonGoPC is called when a profiling signal arrived on a
  3377	// non-Go thread and we have a single PC value, not a stack trace.
  3378	// g is nil, and what we can do is very limited.
  3379	//go:nosplit
  3380	//go:nowritebarrierrec
  3381	func sigprofNonGoPC(pc uintptr) {
  3382		if prof.hz != 0 {
  3383			pc := []uintptr{
  3384				pc,
  3385				funcPC(_ExternalCode) + sys.PCQuantum,
  3386			}
  3387	
  3388			// Simple cas-lock to coordinate with setcpuprofilerate.
  3389			for !atomic.Cas(&prof.lock, 0, 1) {
  3390				osyield()
  3391			}
  3392			if prof.hz != 0 {
  3393				cpuprof.addNonGo(pc)
  3394			}
  3395			atomic.Store(&prof.lock, 0)
  3396		}
  3397	}
  3398	
  3399	// Reports whether a function will set the SP
  3400	// to an absolute value. Important that
  3401	// we don't traceback when these are at the bottom
  3402	// of the stack since we can't be sure that we will
  3403	// find the caller.
  3404	//
  3405	// If the function is not on the bottom of the stack
  3406	// we assume that it will have set it up so that traceback will be consistent,
  3407	// either by being a traceback terminating function
  3408	// or putting one on the stack at the right offset.
  3409	func setsSP(pc uintptr) bool {
  3410		f := findfunc(pc)
  3411		if f == nil {
  3412			// couldn't find the function for this PC,
  3413			// so assume the worst and stop traceback
  3414			return true
  3415		}
  3416		switch f.entry {
  3417		case gogoPC, systemstackPC, mcallPC, morestackPC:
  3418			return true
  3419		}
  3420		return false
  3421	}
  3422	
  3423	// Arrange to call fn with a traceback hz times a second.
  3424	func setcpuprofilerate_m(hz int32) {
  3425		// Force sane arguments.
  3426		if hz < 0 {
  3427			hz = 0
  3428		}
  3429	
  3430		// Disable preemption, otherwise we can be rescheduled to another thread
  3431		// that has profiling enabled.
  3432		_g_ := getg()
  3433		_g_.m.locks++
  3434	
  3435		// Stop profiler on this thread so that it is safe to lock prof.
  3436		// if a profiling signal came in while we had prof locked,
  3437		// it would deadlock.
  3438		resetcpuprofiler(0)
  3439	
  3440		for !atomic.Cas(&prof.lock, 0, 1) {
  3441			osyield()
  3442		}
  3443		prof.hz = hz
  3444		atomic.Store(&prof.lock, 0)
  3445	
  3446		lock(&sched.lock)
  3447		sched.profilehz = hz
  3448		unlock(&sched.lock)
  3449	
  3450		if hz != 0 {
  3451			resetcpuprofiler(hz)
  3452		}
  3453	
  3454		_g_.m.locks--
  3455	}
  3456	
  3457	// Change number of processors. The world is stopped, sched is locked.
  3458	// gcworkbufs are not being modified by either the GC or
  3459	// the write barrier code.
  3460	// Returns list of Ps with local work, they need to be scheduled by the caller.
  3461	func procresize(nprocs int32) *p {
  3462		old := gomaxprocs
  3463		if old < 0 || old > _MaxGomaxprocs || nprocs <= 0 || nprocs > _MaxGomaxprocs {
  3464			throw("procresize: invalid arg")
  3465		}
  3466		if trace.enabled {
  3467			traceGomaxprocs(nprocs)
  3468		}
  3469	
  3470		// update statistics
  3471		now := nanotime()
  3472		if sched.procresizetime != 0 {
  3473			sched.totaltime += int64(old) * (now - sched.procresizetime)
  3474		}
  3475		sched.procresizetime = now
  3476	
  3477		// initialize new P's
  3478		for i := int32(0); i < nprocs; i++ {
  3479			pp := allp[i]
  3480			if pp == nil {
  3481				pp = new(p)
  3482				pp.id = i
  3483				pp.status = _Pgcstop
  3484				pp.sudogcache = pp.sudogbuf[:0]
  3485				for i := range pp.deferpool {
  3486					pp.deferpool[i] = pp.deferpoolbuf[i][:0]
  3487				}
  3488				atomicstorep(unsafe.Pointer(&allp[i]), unsafe.Pointer(pp))
  3489			}
  3490			if pp.mcache == nil {
  3491				if old == 0 && i == 0 {
  3492					if getg().m.mcache == nil {
  3493						throw("missing mcache?")
  3494					}
  3495					pp.mcache = getg().m.mcache // bootstrap
  3496				} else {
  3497					pp.mcache = allocmcache()
  3498				}
  3499			}
  3500			if raceenabled && pp.racectx == 0 {
  3501				if old == 0 && i == 0 {
  3502					pp.racectx = raceprocctx0
  3503					raceprocctx0 = 0 // bootstrap
  3504				} else {
  3505					pp.racectx = raceproccreate()
  3506				}
  3507			}
  3508		}
  3509	
  3510		// free unused P's
  3511		for i := nprocs; i < old; i++ {
  3512			p := allp[i]
  3513			if trace.enabled {
  3514				if p == getg().m.p.ptr() {
  3515					// moving to p[0], pretend that we were descheduled
  3516					// and then scheduled again to keep the trace sane.
  3517					traceGoSched()
  3518					traceProcStop(p)
  3519				}
  3520			}
  3521			// move all runnable goroutines to the global queue
  3522			for p.runqhead != p.runqtail {
  3523				// pop from tail of local queue
  3524				p.runqtail--
  3525				gp := p.runq[p.runqtail%uint32(len(p.runq))].ptr()
  3526				// push onto head of global queue
  3527				globrunqputhead(gp)
  3528			}
  3529			if p.runnext != 0 {
  3530				globrunqputhead(p.runnext.ptr())
  3531				p.runnext = 0
  3532			}
  3533			// if there's a background worker, make it runnable and put
  3534			// it on the global queue so it can clean itself up
  3535			if gp := p.gcBgMarkWorker.ptr(); gp != nil {
  3536				casgstatus(gp, _Gwaiting, _Grunnable)
  3537				if trace.enabled {
  3538					traceGoUnpark(gp, 0)
  3539				}
  3540				globrunqput(gp)
  3541				// This assignment doesn't race because the
  3542				// world is stopped.
  3543				p.gcBgMarkWorker.set(nil)
  3544			}
  3545			for i := range p.sudogbuf {
  3546				p.sudogbuf[i] = nil
  3547			}
  3548			p.sudogcache = p.sudogbuf[:0]
  3549			for i := range p.deferpool {
  3550				for j := range p.deferpoolbuf[i] {
  3551					p.deferpoolbuf[i][j] = nil
  3552				}
  3553				p.deferpool[i] = p.deferpoolbuf[i][:0]
  3554			}
  3555			freemcache(p.mcache)
  3556			p.mcache = nil
  3557			gfpurge(p)
  3558			traceProcFree(p)
  3559			if raceenabled {
  3560				raceprocdestroy(p.racectx)
  3561				p.racectx = 0
  3562			}
  3563			p.status = _Pdead
  3564			// can't free P itself because it can be referenced by an M in syscall
  3565		}
  3566	
  3567		_g_ := getg()
  3568		if _g_.m.p != 0 && _g_.m.p.ptr().id < nprocs {
  3569			// continue to use the current P
  3570			_g_.m.p.ptr().status = _Prunning
  3571		} else {
  3572			// release the current P and acquire allp[0]
  3573			if _g_.m.p != 0 {
  3574				_g_.m.p.ptr().m = 0
  3575			}
  3576			_g_.m.p = 0
  3577			_g_.m.mcache = nil
  3578			p := allp[0]
  3579			p.m = 0
  3580			p.status = _Pidle
  3581			acquirep(p)
  3582			if trace.enabled {
  3583				traceGoStart()
  3584			}
  3585		}
  3586		var runnablePs *p
  3587		for i := nprocs - 1; i >= 0; i-- {
  3588			p := allp[i]
  3589			if _g_.m.p.ptr() == p {
  3590				continue
  3591			}
  3592			p.status = _Pidle
  3593			if runqempty(p) {
  3594				pidleput(p)
  3595			} else {
  3596				p.m.set(mget())
  3597				p.link.set(runnablePs)
  3598				runnablePs = p
  3599			}
  3600		}
  3601		stealOrder.reset(uint32(nprocs))
  3602		var int32p *int32 = &gomaxprocs // make compiler check that gomaxprocs is an int32
  3603		atomic.Store((*uint32)(unsafe.Pointer(int32p)), uint32(nprocs))
  3604		return runnablePs
  3605	}
  3606	
  3607	// Associate p and the current m.
  3608	//
  3609	// This function is allowed to have write barriers even if the caller
  3610	// isn't because it immediately acquires _p_.
  3611	//
  3612	//go:yeswritebarrierrec
  3613	func acquirep(_p_ *p) {
  3614		// Do the part that isn't allowed to have write barriers.
  3615		acquirep1(_p_)
  3616	
  3617		// have p; write barriers now allowed
  3618		_g_ := getg()
  3619		_g_.m.mcache = _p_.mcache
  3620	
  3621		if trace.enabled {
  3622			traceProcStart()
  3623		}
  3624	}
  3625	
  3626	// acquirep1 is the first step of acquirep, which actually acquires
  3627	// _p_. This is broken out so we can disallow write barriers for this
  3628	// part, since we don't yet have a P.
  3629	//
  3630	//go:nowritebarrierrec
  3631	func acquirep1(_p_ *p) {
  3632		_g_ := getg()
  3633	
  3634		if _g_.m.p != 0 || _g_.m.mcache != nil {
  3635			throw("acquirep: already in go")
  3636		}
  3637		if _p_.m != 0 || _p_.status != _Pidle {
  3638			id := int32(0)
  3639			if _p_.m != 0 {
  3640				id = _p_.m.ptr().id
  3641			}
  3642			print("acquirep: p->m=", _p_.m, "(", id, ") p->status=", _p_.status, "\n")
  3643			throw("acquirep: invalid p state")
  3644		}
  3645		_g_.m.p.set(_p_)
  3646		_p_.m.set(_g_.m)
  3647		_p_.status = _Prunning
  3648	}
  3649	
  3650	// Disassociate p and the current m.
  3651	func releasep() *p {
  3652		_g_ := getg()
  3653	
  3654		if _g_.m.p == 0 || _g_.m.mcache == nil {
  3655			throw("releasep: invalid arg")
  3656		}
  3657		_p_ := _g_.m.p.ptr()
  3658		if _p_.m.ptr() != _g_.m || _p_.mcache != _g_.m.mcache || _p_.status != _Prunning {
  3659			print("releasep: m=", _g_.m, " m->p=", _g_.m.p.ptr(), " p->m=", _p_.m, " m->mcache=", _g_.m.mcache, " p->mcache=", _p_.mcache, " p->status=", _p_.status, "\n")
  3660			throw("releasep: invalid p state")
  3661		}
  3662		if trace.enabled {
  3663			traceProcStop(_g_.m.p.ptr())
  3664		}
  3665		_g_.m.p = 0
  3666		_g_.m.mcache = nil
  3667		_p_.m = 0
  3668		_p_.status = _Pidle
  3669		return _p_
  3670	}
  3671	
  3672	func incidlelocked(v int32) {
  3673		lock(&sched.lock)
  3674		sched.nmidlelocked += v
  3675		if v > 0 {
  3676			checkdead()
  3677		}
  3678		unlock(&sched.lock)
  3679	}
  3680	
  3681	// Check for deadlock situation.
  3682	// The check is based on number of running M's, if 0 -> deadlock.
  3683	func checkdead() {
  3684		// For -buildmode=c-shared or -buildmode=c-archive it's OK if
  3685		// there are no running goroutines. The calling program is
  3686		// assumed to be running.
  3687		if islibrary || isarchive {
  3688			return
  3689		}
  3690	
  3691		// If we are dying because of a signal caught on an already idle thread,
  3692		// freezetheworld will cause all running threads to block.
  3693		// And runtime will essentially enter into deadlock state,
  3694		// except that there is a thread that will call exit soon.
  3695		if panicking > 0 {
  3696			return
  3697		}
  3698	
  3699		// -1 for sysmon
  3700		run := sched.mcount - sched.nmidle - sched.nmidlelocked - 1
  3701		if run > 0 {
  3702			return
  3703		}
  3704		if run < 0 {
  3705			print("runtime: checkdead: nmidle=", sched.nmidle, " nmidlelocked=", sched.nmidlelocked, " mcount=", sched.mcount, "\n")
  3706			throw("checkdead: inconsistent counts")
  3707		}
  3708	
  3709		grunning := 0
  3710		lock(&allglock)
  3711		for i := 0; i < len(allgs); i++ {
  3712			gp := allgs[i]
  3713			if isSystemGoroutine(gp) {
  3714				continue
  3715			}
  3716			s := readgstatus(gp)
  3717			switch s &^ _Gscan {
  3718			case _Gwaiting:
  3719				grunning++
  3720			case _Grunnable,
  3721				_Grunning,
  3722				_Gsyscall:
  3723				unlock(&allglock)
  3724				print("runtime: checkdead: find g ", gp.goid, " in status ", s, "\n")
  3725				throw("checkdead: runnable g")
  3726			}
  3727		}
  3728		unlock(&allglock)
  3729		if grunning == 0 { // possible if main goroutine calls runtime·Goexit()
  3730			throw("no goroutines (main called runtime.Goexit) - deadlock!")
  3731		}
  3732	
  3733		// Maybe jump time forward for playground.
  3734		gp := timejump()
  3735		if gp != nil {
  3736			casgstatus(gp, _Gwaiting, _Grunnable)
  3737			globrunqput(gp)
  3738			_p_ := pidleget()
  3739			if _p_ == nil {
  3740				throw("checkdead: no p for timer")
  3741			}
  3742			mp := mget()
  3743			if mp == nil {
  3744				// There should always be a free M since
  3745				// nothing is running.
  3746				throw("checkdead: no m for timer")
  3747			}
  3748			mp.nextp.set(_p_)
  3749			notewakeup(&mp.park)
  3750			return
  3751		}
  3752	
  3753		getg().m.throwing = -1 // do not dump full stacks
  3754		throw("all goroutines are asleep - deadlock!")
  3755	}
  3756	
  3757	// forcegcperiod is the maximum time in nanoseconds between garbage
  3758	// collections. If we go this long without a garbage collection, one
  3759	// is forced to run.
  3760	//
  3761	// This is a variable for testing purposes. It normally doesn't change.
  3762	var forcegcperiod int64 = 2 * 60 * 1e9
  3763	
  3764	// Always runs without a P, so write barriers are not allowed.
  3765	//
  3766	//go:nowritebarrierrec
  3767	func sysmon() {
  3768		// If a heap span goes unused for 5 minutes after a garbage collection,
  3769		// we hand it back to the operating system.
  3770		scavengelimit := int64(5 * 60 * 1e9)
  3771	
  3772		if debug.scavenge > 0 {
  3773			// Scavenge-a-lot for testing.
  3774			forcegcperiod = 10 * 1e6
  3775			scavengelimit = 20 * 1e6
  3776		}
  3777	
  3778		lastscavenge := nanotime()
  3779		nscavenge := 0
  3780	
  3781		lasttrace := int64(0)
  3782		idle := 0 // how many cycles in succession we had not wokeup somebody
  3783		delay := uint32(0)
  3784		for {
  3785			if idle == 0 { // start with 20us sleep...
  3786				delay = 20
  3787			} else if idle > 50 { // start doubling the sleep after 1ms...
  3788				delay *= 2
  3789			}
  3790			if delay > 10*1000 { // up to 10ms
  3791				delay = 10 * 1000
  3792			}
  3793			usleep(delay)
  3794			if debug.schedtrace <= 0 && (sched.gcwaiting != 0 || atomic.Load(&sched.npidle) == uint32(gomaxprocs)) {
  3795				lock(&sched.lock)
  3796				if atomic.Load(&sched.gcwaiting) != 0 || atomic.Load(&sched.npidle) == uint32(gomaxprocs) {
  3797					atomic.Store(&sched.sysmonwait, 1)
  3798					unlock(&sched.lock)
  3799					// Make wake-up period small enough
  3800					// for the sampling to be correct.
  3801					maxsleep := forcegcperiod / 2
  3802					if scavengelimit < forcegcperiod {
  3803						maxsleep = scavengelimit / 2
  3804					}
  3805					notetsleep(&sched.sysmonnote, maxsleep)
  3806					lock(&sched.lock)
  3807					atomic.Store(&sched.sysmonwait, 0)
  3808					noteclear(&sched.sysmonnote)
  3809					idle = 0
  3810					delay = 20
  3811				}
  3812				unlock(&sched.lock)
  3813			}
  3814			// poll network if not polled for more than 10ms
  3815			lastpoll := int64(atomic.Load64(&sched.lastpoll))
  3816			now := nanotime()
  3817			unixnow := unixnanotime()
  3818			if lastpoll != 0 && lastpoll+10*1000*1000 < now {
  3819				atomic.Cas64(&sched.lastpoll, uint64(lastpoll), uint64(now))
  3820				gp := netpoll(false) // non-blocking - returns list of goroutines
  3821				if gp != nil {
  3822					// Need to decrement number of idle locked M's
  3823					// (pretending that one more is running) before injectglist.
  3824					// Otherwise it can lead to the following situation:
  3825					// injectglist grabs all P's but before it starts M's to run the P's,
  3826					// another M returns from syscall, finishes running its G,
  3827					// observes that there is no work to do and no other running M's
  3828					// and reports deadlock.
  3829					incidlelocked(-1)
  3830					injectglist(gp)
  3831					incidlelocked(1)
  3832				}
  3833			}
  3834			// retake P's blocked in syscalls
  3835			// and preempt long running G's
  3836			if retake(now) != 0 {
  3837				idle = 0
  3838			} else {
  3839				idle++
  3840			}
  3841			// check if we need to force a GC
  3842			lastgc := int64(atomic.Load64(&memstats.last_gc))
  3843			if gcphase == _GCoff && lastgc != 0 && unixnow-lastgc > forcegcperiod && atomic.Load(&forcegc.idle) != 0 {
  3844				lock(&forcegc.lock)
  3845				forcegc.idle = 0
  3846				forcegc.g.schedlink = 0
  3847				injectglist(forcegc.g)
  3848				unlock(&forcegc.lock)
  3849			}
  3850			// scavenge heap once in a while
  3851			if lastscavenge+scavengelimit/2 < now {
  3852				mheap_.scavenge(int32(nscavenge), uint64(now), uint64(scavengelimit))
  3853				lastscavenge = now
  3854				nscavenge++
  3855			}
  3856			if debug.schedtrace > 0 && lasttrace+int64(debug.schedtrace)*1000000 <= now {
  3857				lasttrace = now
  3858				schedtrace(debug.scheddetail > 0)
  3859			}
  3860		}
  3861	}
  3862	
  3863	var pdesc [_MaxGomaxprocs]struct {
  3864		schedtick   uint32
  3865		schedwhen   int64
  3866		syscalltick uint32
  3867		syscallwhen int64
  3868	}
  3869	
  3870	// forcePreemptNS is the time slice given to a G before it is
  3871	// preempted.
  3872	const forcePreemptNS = 10 * 1000 * 1000 // 10ms
  3873	
  3874	func retake(now int64) uint32 {
  3875		n := 0
  3876		for i := int32(0); i < gomaxprocs; i++ {
  3877			_p_ := allp[i]
  3878			if _p_ == nil {
  3879				continue
  3880			}
  3881			pd := &pdesc[i]
  3882			s := _p_.status
  3883			if s == _Psyscall {
  3884				// Retake P from syscall if it's there for more than 1 sysmon tick (at least 20us).
  3885				t := int64(_p_.syscalltick)
  3886				if int64(pd.syscalltick) != t {
  3887					pd.syscalltick = uint32(t)
  3888					pd.syscallwhen = now
  3889					continue
  3890				}
  3891				// On the one hand we don't want to retake Ps if there is no other work to do,
  3892				// but on the other hand we want to retake them eventually
  3893				// because they can prevent the sysmon thread from deep sleep.
  3894				if runqempty(_p_) && atomic.Load(&sched.nmspinning)+atomic.Load(&sched.npidle) > 0 && pd.syscallwhen+10*1000*1000 > now {
  3895					continue
  3896				}
  3897				// Need to decrement number of idle locked M's
  3898				// (pretending that one more is running) before the CAS.
  3899				// Otherwise the M from which we retake can exit the syscall,
  3900				// increment nmidle and report deadlock.
  3901				incidlelocked(-1)
  3902				if atomic.Cas(&_p_.status, s, _Pidle) {
  3903					if trace.enabled {
  3904						traceGoSysBlock(_p_)
  3905						traceProcStop(_p_)
  3906					}
  3907					n++
  3908					_p_.syscalltick++
  3909					handoffp(_p_)
  3910				}
  3911				incidlelocked(1)
  3912			} else if s == _Prunning {
  3913				// Preempt G if it's running for too long.
  3914				t := int64(_p_.schedtick)
  3915				if int64(pd.schedtick) != t {
  3916					pd.schedtick = uint32(t)
  3917					pd.schedwhen = now
  3918					continue
  3919				}
  3920				if pd.schedwhen+forcePreemptNS > now {
  3921					continue
  3922				}
  3923				preemptone(_p_)
  3924			}
  3925		}
  3926		return uint32(n)
  3927	}
  3928	
  3929	// Tell all goroutines that they have been preempted and they should stop.
  3930	// This function is purely best-effort. It can fail to inform a goroutine if a
  3931	// processor just started running it.
  3932	// No locks need to be held.
  3933	// Returns true if preemption request was issued to at least one goroutine.
  3934	func preemptall() bool {
  3935		res := false
  3936		for i := int32(0); i < gomaxprocs; i++ {
  3937			_p_ := allp[i]
  3938			if _p_ == nil || _p_.status != _Prunning {
  3939				continue
  3940			}
  3941			if preemptone(_p_) {
  3942				res = true
  3943			}
  3944		}
  3945		return res
  3946	}
  3947	
  3948	// Tell the goroutine running on processor P to stop.
  3949	// This function is purely best-effort. It can incorrectly fail to inform the
  3950	// goroutine. It can send inform the wrong goroutine. Even if it informs the
  3951	// correct goroutine, that goroutine might ignore the request if it is
  3952	// simultaneously executing newstack.
  3953	// No lock needs to be held.
  3954	// Returns true if preemption request was issued.
  3955	// The actual preemption will happen at some point in the future
  3956	// and will be indicated by the gp->status no longer being
  3957	// Grunning
  3958	func preemptone(_p_ *p) bool {
  3959		mp := _p_.m.ptr()
  3960		if mp == nil || mp == getg().m {
  3961			return false
  3962		}
  3963		gp := mp.curg
  3964		if gp == nil || gp == mp.g0 {
  3965			return false
  3966		}
  3967	
  3968		gp.preempt = true
  3969	
  3970		// Every call in a go routine checks for stack overflow by
  3971		// comparing the current stack pointer to gp->stackguard0.
  3972		// Setting gp->stackguard0 to StackPreempt folds
  3973		// preemption into the normal stack overflow check.
  3974		gp.stackguard0 = stackPreempt
  3975		return true
  3976	}
  3977	
  3978	var starttime int64
  3979	
  3980	func schedtrace(detailed bool) {
  3981		now := nanotime()
  3982		if starttime == 0 {
  3983			starttime = now
  3984		}
  3985	
  3986		lock(&sched.lock)
  3987		print("SCHED ", (now-starttime)/1e6, "ms: gomaxprocs=", gomaxprocs, " idleprocs=", sched.npidle, " threads=", sched.mcount, " spinningthreads=", sched.nmspinning, " idlethreads=", sched.nmidle, " runqueue=", sched.runqsize)
  3988		if detailed {
  3989			print(" gcwaiting=", sched.gcwaiting, " nmidlelocked=", sched.nmidlelocked, " stopwait=", sched.stopwait, " sysmonwait=", sched.sysmonwait, "\n")
  3990		}
  3991		// We must be careful while reading data from P's, M's and G's.
  3992		// Even if we hold schedlock, most data can be changed concurrently.
  3993		// E.g. (p->m ? p->m->id : -1) can crash if p->m changes from non-nil to nil.
  3994		for i := int32(0); i < gomaxprocs; i++ {
  3995			_p_ := allp[i]
  3996			if _p_ == nil {
  3997				continue
  3998			}
  3999			mp := _p_.m.ptr()
  4000			h := atomic.Load(&_p_.runqhead)
  4001			t := atomic.Load(&_p_.runqtail)
  4002			if detailed {
  4003				id := int32(-1)
  4004				if mp != nil {
  4005					id = mp.id
  4006				}
  4007				print("  P", i, ": status=", _p_.status, " schedtick=", _p_.schedtick, " syscalltick=", _p_.syscalltick, " m=", id, " runqsize=", t-h, " gfreecnt=", _p_.gfreecnt, "\n")
  4008			} else {
  4009				// In non-detailed mode format lengths of per-P run queues as:
  4010				// [len1 len2 len3 len4]
  4011				print(" ")
  4012				if i == 0 {
  4013					print("[")
  4014				}
  4015				print(t - h)
  4016				if i == gomaxprocs-1 {
  4017					print("]\n")
  4018				}
  4019			}
  4020		}
  4021	
  4022		if !detailed {
  4023			unlock(&sched.lock)
  4024			return
  4025		}
  4026	
  4027		for mp := allm; mp != nil; mp = mp.alllink {
  4028			_p_ := mp.p.ptr()
  4029			gp := mp.curg
  4030			lockedg := mp.lockedg
  4031			id1 := int32(-1)
  4032			if _p_ != nil {
  4033				id1 = _p_.id
  4034			}
  4035			id2 := int64(-1)
  4036			if gp != nil {
  4037				id2 = gp.goid
  4038			}
  4039			id3 := int64(-1)
  4040			if lockedg != nil {
  4041				id3 = lockedg.goid
  4042			}
  4043			print("  M", mp.id, ": p=", id1, " curg=", id2, " mallocing=", mp.mallocing, " throwing=", mp.throwing, " preemptoff=", mp.preemptoff, ""+" locks=", mp.locks, " dying=", mp.dying, " helpgc=", mp.helpgc, " spinning=", mp.spinning, " blocked=", mp.blocked, " lockedg=", id3, "\n")
  4044		}
  4045	
  4046		lock(&allglock)
  4047		for gi := 0; gi < len(allgs); gi++ {
  4048			gp := allgs[gi]
  4049			mp := gp.m
  4050			lockedm := gp.lockedm
  4051			id1 := int32(-1)
  4052			if mp != nil {
  4053				id1 = mp.id
  4054			}
  4055			id2 := int32(-1)
  4056			if lockedm != nil {
  4057				id2 = lockedm.id
  4058			}
  4059			print("  G", gp.goid, ": status=", readgstatus(gp), "(", gp.waitreason, ") m=", id1, " lockedm=", id2, "\n")
  4060		}
  4061		unlock(&allglock)
  4062		unlock(&sched.lock)
  4063	}
  4064	
  4065	// Put mp on midle list.
  4066	// Sched must be locked.
  4067	// May run during STW, so write barriers are not allowed.
  4068	//go:nowritebarrierrec
  4069	func mput(mp *m) {
  4070		mp.schedlink = sched.midle
  4071		sched.midle.set(mp)
  4072		sched.nmidle++
  4073		checkdead()
  4074	}
  4075	
  4076	// Try to get an m from midle list.
  4077	// Sched must be locked.
  4078	// May run during STW, so write barriers are not allowed.
  4079	//go:nowritebarrierrec
  4080	func mget() *m {
  4081		mp := sched.midle.ptr()
  4082		if mp != nil {
  4083			sched.midle = mp.schedlink
  4084			sched.nmidle--
  4085		}
  4086		return mp
  4087	}
  4088	
  4089	// Put gp on the global runnable queue.
  4090	// Sched must be locked.
  4091	// May run during STW, so write barriers are not allowed.
  4092	//go:nowritebarrierrec
  4093	func globrunqput(gp *g) {
  4094		gp.schedlink = 0
  4095		if sched.runqtail != 0 {
  4096			sched.runqtail.ptr().schedlink.set(gp)
  4097		} else {
  4098			sched.runqhead.set(gp)
  4099		}
  4100		sched.runqtail.set(gp)
  4101		sched.runqsize++
  4102	}
  4103	
  4104	// Put gp at the head of the global runnable queue.
  4105	// Sched must be locked.
  4106	// May run during STW, so write barriers are not allowed.
  4107	//go:nowritebarrierrec
  4108	func globrunqputhead(gp *g) {
  4109		gp.schedlink = sched.runqhead
  4110		sched.runqhead.set(gp)
  4111		if sched.runqtail == 0 {
  4112			sched.runqtail.set(gp)
  4113		}
  4114		sched.runqsize++
  4115	}
  4116	
  4117	// Put a batch of runnable goroutines on the global runnable queue.
  4118	// Sched must be locked.
  4119	func globrunqputbatch(ghead *g, gtail *g, n int32) {
  4120		gtail.schedlink = 0
  4121		if sched.runqtail != 0 {
  4122			sched.runqtail.ptr().schedlink.set(ghead)
  4123		} else {
  4124			sched.runqhead.set(ghead)
  4125		}
  4126		sched.runqtail.set(gtail)
  4127		sched.runqsize += n
  4128	}
  4129	
  4130	// Try get a batch of G's from the global runnable queue.
  4131	// Sched must be locked.
  4132	func globrunqget(_p_ *p, max int32) *g {
  4133		if sched.runqsize == 0 {
  4134			return nil
  4135		}
  4136	
  4137		n := sched.runqsize/gomaxprocs + 1
  4138		if n > sched.runqsize {
  4139			n = sched.runqsize
  4140		}
  4141		if max > 0 && n > max {
  4142			n = max
  4143		}
  4144		if n > int32(len(_p_.runq))/2 {
  4145			n = int32(len(_p_.runq)) / 2
  4146		}
  4147	
  4148		sched.runqsize -= n
  4149		if sched.runqsize == 0 {
  4150			sched.runqtail = 0
  4151		}
  4152	
  4153		gp := sched.runqhead.ptr()
  4154		sched.runqhead = gp.schedlink
  4155		n--
  4156		for ; n > 0; n-- {
  4157			gp1 := sched.runqhead.ptr()
  4158			sched.runqhead = gp1.schedlink
  4159			runqput(_p_, gp1, false)
  4160		}
  4161		return gp
  4162	}
  4163	
  4164	// Put p to on _Pidle list.
  4165	// Sched must be locked.
  4166	// May run during STW, so write barriers are not allowed.
  4167	//go:nowritebarrierrec
  4168	func pidleput(_p_ *p) {
  4169		if !runqempty(_p_) {
  4170			throw("pidleput: P has non-empty run queue")
  4171		}
  4172		_p_.link = sched.pidle
  4173		sched.pidle.set(_p_)
  4174		atomic.Xadd(&sched.npidle, 1) // TODO: fast atomic
  4175	}
  4176	
  4177	// Try get a p from _Pidle list.
  4178	// Sched must be locked.
  4179	// May run during STW, so write barriers are not allowed.
  4180	//go:nowritebarrierrec
  4181	func pidleget() *p {
  4182		_p_ := sched.pidle.ptr()
  4183		if _p_ != nil {
  4184			sched.pidle = _p_.link
  4185			atomic.Xadd(&sched.npidle, -1) // TODO: fast atomic
  4186		}
  4187		return _p_
  4188	}
  4189	
  4190	// runqempty returns true if _p_ has no Gs on its local run queue.
  4191	// It never returns true spuriously.
  4192	func runqempty(_p_ *p) bool {
  4193		// Defend against a race where 1) _p_ has G1 in runqnext but runqhead == runqtail,
  4194		// 2) runqput on _p_ kicks G1 to the runq, 3) runqget on _p_ empties runqnext.
  4195		// Simply observing that runqhead == runqtail and then observing that runqnext == nil
  4196		// does not mean the queue is empty.
  4197		for {
  4198			head := atomic.Load(&_p_.runqhead)
  4199			tail := atomic.Load(&_p_.runqtail)
  4200			runnext := atomic.Loaduintptr((*uintptr)(unsafe.Pointer(&_p_.runnext)))
  4201			if tail == atomic.Load(&_p_.runqtail) {
  4202				return head == tail && runnext == 0
  4203			}
  4204		}
  4205	}
  4206	
  4207	// To shake out latent assumptions about scheduling order,
  4208	// we introduce some randomness into scheduling decisions
  4209	// when running with the race detector.
  4210	// The need for this was made obvious by changing the
  4211	// (deterministic) scheduling order in Go 1.5 and breaking
  4212	// many poorly-written tests.
  4213	// With the randomness here, as long as the tests pass
  4214	// consistently with -race, they shouldn't have latent scheduling
  4215	// assumptions.
  4216	const randomizeScheduler = raceenabled
  4217	
  4218	// runqput tries to put g on the local runnable queue.
  4219	// If next if false, runqput adds g to the tail of the runnable queue.
  4220	// If next is true, runqput puts g in the _p_.runnext slot.
  4221	// If the run queue is full, runnext puts g on the global queue.
  4222	// Executed only by the owner P.
  4223	func runqput(_p_ *p, gp *g, next bool) {
  4224		if randomizeScheduler && next && fastrand()%2 == 0 {
  4225			next = false
  4226		}
  4227	
  4228		if next {
  4229		retryNext:
  4230			oldnext := _p_.runnext
  4231			if !_p_.runnext.cas(oldnext, guintptr(unsafe.Pointer(gp))) {
  4232				goto retryNext
  4233			}
  4234			if oldnext == 0 {
  4235				return
  4236			}
  4237			// Kick the old runnext out to the regular run queue.
  4238			gp = oldnext.ptr()
  4239		}
  4240	
  4241	retry:
  4242		h := atomic.Load(&_p_.runqhead) // load-acquire, synchronize with consumers
  4243		t := _p_.runqtail
  4244		if t-h < uint32(len(_p_.runq)) {
  4245			_p_.runq[t%uint32(len(_p_.runq))].set(gp)
  4246			atomic.Store(&_p_.runqtail, t+1) // store-release, makes the item available for consumption
  4247			return
  4248		}
  4249		if runqputslow(_p_, gp, h, t) {
  4250			return
  4251		}
  4252		// the queue is not full, now the put above must succeed
  4253		goto retry
  4254	}
  4255	
  4256	// Put g and a batch of work from local runnable queue on global queue.
  4257	// Executed only by the owner P.
  4258	func runqputslow(_p_ *p, gp *g, h, t uint32) bool {
  4259		var batch [len(_p_.runq)/2 + 1]*g
  4260	
  4261		// First, grab a batch from local queue.
  4262		n := t - h
  4263		n = n / 2
  4264		if n != uint32(len(_p_.runq)/2) {
  4265			throw("runqputslow: queue is not full")
  4266		}
  4267		for i := uint32(0); i < n; i++ {
  4268			batch[i] = _p_.runq[(h+i)%uint32(len(_p_.runq))].ptr()
  4269		}
  4270		if !atomic.Cas(&_p_.runqhead, h, h+n) { // cas-release, commits consume
  4271			return false
  4272		}
  4273		batch[n] = gp
  4274	
  4275		if randomizeScheduler {
  4276			for i := uint32(1); i <= n; i++ {
  4277				j := fastrand() % (i + 1)
  4278				batch[i], batch[j] = batch[j], batch[i]
  4279			}
  4280		}
  4281	
  4282		// Link the goroutines.
  4283		for i := uint32(0); i < n; i++ {
  4284			batch[i].schedlink.set(batch[i+1])
  4285		}
  4286	
  4287		// Now put the batch on global queue.
  4288		lock(&sched.lock)
  4289		globrunqputbatch(batch[0], batch[n], int32(n+1))
  4290		unlock(&sched.lock)
  4291		return true
  4292	}
  4293	
  4294	// Get g from local runnable queue.
  4295	// If inheritTime is true, gp should inherit the remaining time in the
  4296	// current time slice. Otherwise, it should start a new time slice.
  4297	// Executed only by the owner P.
  4298	func runqget(_p_ *p) (gp *g, inheritTime bool) {
  4299		// If there's a runnext, it's the next G to run.
  4300		for {
  4301			next := _p_.runnext
  4302			if next == 0 {
  4303				break
  4304			}
  4305			if _p_.runnext.cas(next, 0) {
  4306				return next.ptr(), true
  4307			}
  4308		}
  4309	
  4310		for {
  4311			h := atomic.Load(&_p_.runqhead) // load-acquire, synchronize with other consumers
  4312			t := _p_.runqtail
  4313			if t == h {
  4314				return nil, false
  4315			}
  4316			gp := _p_.runq[h%uint32(len(_p_.runq))].ptr()
  4317			if atomic.Cas(&_p_.runqhead, h, h+1) { // cas-release, commits consume
  4318				return gp, false
  4319			}
  4320		}
  4321	}
  4322	
  4323	// Grabs a batch of goroutines from _p_'s runnable queue into batch.
  4324	// Batch is a ring buffer starting at batchHead.
  4325	// Returns number of grabbed goroutines.
  4326	// Can be executed by any P.
  4327	func runqgrab(_p_ *p, batch *[256]guintptr, batchHead uint32, stealRunNextG bool) uint32 {
  4328		for {
  4329			h := atomic.Load(&_p_.runqhead) // load-acquire, synchronize with other consumers
  4330			t := atomic.Load(&_p_.runqtail) // load-acquire, synchronize with the producer
  4331			n := t - h
  4332			n = n - n/2
  4333			if n == 0 {
  4334				if stealRunNextG {
  4335					// Try to steal from _p_.runnext.
  4336					if next := _p_.runnext; next != 0 {
  4337						// Sleep to ensure that _p_ isn't about to run the g we
  4338						// are about to steal.
  4339						// The important use case here is when the g running on _p_
  4340						// ready()s another g and then almost immediately blocks.
  4341						// Instead of stealing runnext in this window, back off
  4342						// to give _p_ a chance to schedule runnext. This will avoid
  4343						// thrashing gs between different Ps.
  4344						// A sync chan send/recv takes ~50ns as of time of writing,
  4345						// so 3us gives ~50x overshoot.
  4346						if GOOS != "windows" {
  4347							usleep(3)
  4348						} else {
  4349							// On windows system timer granularity is 1-15ms,
  4350							// which is way too much for this optimization.
  4351							// So just yield.
  4352							osyield()
  4353						}
  4354						if !_p_.runnext.cas(next, 0) {
  4355							continue
  4356						}
  4357						batch[batchHead%uint32(len(batch))] = next
  4358						return 1
  4359					}
  4360				}
  4361				return 0
  4362			}
  4363			if n > uint32(len(_p_.runq)/2) { // read inconsistent h and t
  4364				continue
  4365			}
  4366			for i := uint32(0); i < n; i++ {
  4367				g := _p_.runq[(h+i)%uint32(len(_p_.runq))]
  4368				batch[(batchHead+i)%uint32(len(batch))] = g
  4369			}
  4370			if atomic.Cas(&_p_.runqhead, h, h+n) { // cas-release, commits consume
  4371				return n
  4372			}
  4373		}
  4374	}
  4375	
  4376	// Steal half of elements from local runnable queue of p2
  4377	// and put onto local runnable queue of p.
  4378	// Returns one of the stolen elements (or nil if failed).
  4379	func runqsteal(_p_, p2 *p, stealRunNextG bool) *g {
  4380		t := _p_.runqtail
  4381		n := runqgrab(p2, &_p_.runq, t, stealRunNextG)
  4382		if n == 0 {
  4383			return nil
  4384		}
  4385		n--
  4386		gp := _p_.runq[(t+n)%uint32(len(_p_.runq))].ptr()
  4387		if n == 0 {
  4388			return gp
  4389		}
  4390		h := atomic.Load(&_p_.runqhead) // load-acquire, synchronize with consumers
  4391		if t-h+n >= uint32(len(_p_.runq)) {
  4392			throw("runqsteal: runq overflow")
  4393		}
  4394		atomic.Store(&_p_.runqtail, t+n) // store-release, makes the item available for consumption
  4395		return gp
  4396	}
  4397	
  4398	//go:linkname setMaxThreads runtime/debug.setMaxThreads
  4399	func setMaxThreads(in int) (out int) {
  4400		lock(&sched.lock)
  4401		out = int(sched.maxmcount)
  4402		if in > 0x7fffffff { // MaxInt32
  4403			sched.maxmcount = 0x7fffffff
  4404		} else {
  4405			sched.maxmcount = int32(in)
  4406		}
  4407		checkmcount()
  4408		unlock(&sched.lock)
  4409		return
  4410	}
  4411	
  4412	func haveexperiment(name string) bool {
  4413		if name == "framepointer" {
  4414			return framepointer_enabled // set by linker
  4415		}
  4416		x := sys.Goexperiment
  4417		for x != "" {
  4418			xname := ""
  4419			i := index(x, ",")
  4420			if i < 0 {
  4421				xname, x = x, ""
  4422			} else {
  4423				xname, x = x[:i], x[i+1:]
  4424			}
  4425			if xname == name {
  4426				return true
  4427			}
  4428			if len(xname) > 2 && xname[:2] == "no" && xname[2:] == name {
  4429				return false
  4430			}
  4431		}
  4432		return false
  4433	}
  4434	
  4435	//go:nosplit
  4436	func procPin() int {
  4437		_g_ := getg()
  4438		mp := _g_.m
  4439	
  4440		mp.locks++
  4441		return int(mp.p.ptr().id)
  4442	}
  4443	
  4444	//go:nosplit
  4445	func procUnpin() {
  4446		_g_ := getg()
  4447		_g_.m.locks--
  4448	}
  4449	
  4450	//go:linkname sync_runtime_procPin sync.runtime_procPin
  4451	//go:nosplit
  4452	func sync_runtime_procPin() int {
  4453		return procPin()
  4454	}
  4455	
  4456	//go:linkname sync_runtime_procUnpin sync.runtime_procUnpin
  4457	//go:nosplit
  4458	func sync_runtime_procUnpin() {
  4459		procUnpin()
  4460	}
  4461	
  4462	//go:linkname sync_atomic_runtime_procPin sync/atomic.runtime_procPin
  4463	//go:nosplit
  4464	func sync_atomic_runtime_procPin() int {
  4465		return procPin()
  4466	}
  4467	
  4468	//go:linkname sync_atomic_runtime_procUnpin sync/atomic.runtime_procUnpin
  4469	//go:nosplit
  4470	func sync_atomic_runtime_procUnpin() {
  4471		procUnpin()
  4472	}
  4473	
  4474	// Active spinning for sync.Mutex.
  4475	//go:linkname sync_runtime_canSpin sync.runtime_canSpin
  4476	//go:nosplit
  4477	func sync_runtime_canSpin(i int) bool {
  4478		// sync.Mutex is cooperative, so we are conservative with spinning.
  4479		// Spin only few times and only if running on a multicore machine and
  4480		// GOMAXPROCS>1 and there is at least one other running P and local runq is empty.
  4481		// As opposed to runtime mutex we don't do passive spinning here,
  4482		// because there can be work on global runq on on other Ps.
  4483		if i >= active_spin || ncpu <= 1 || gomaxprocs <= int32(sched.npidle+sched.nmspinning)+1 {
  4484			return false
  4485		}
  4486		if p := getg().m.p.ptr(); !runqempty(p) {
  4487			return false
  4488		}
  4489		return true
  4490	}
  4491	
  4492	//go:linkname sync_runtime_doSpin sync.runtime_doSpin
  4493	//go:nosplit
  4494	func sync_runtime_doSpin() {
  4495		procyield(active_spin_cnt)
  4496	}
  4497	
  4498	var stealOrder randomOrder
  4499	
  4500	// randomOrder/randomEnum are helper types for randomized work stealing.
  4501	// They allow to enumerate all Ps in different pseudo-random orders without repetitions.
  4502	// The algorithm is based on the fact that if we have X such that X and GOMAXPROCS
  4503	// are coprime, then a sequences of (i + X) % GOMAXPROCS gives the required enumeration.
  4504	type randomOrder struct {
  4505		count    uint32
  4506		coprimes []uint32
  4507	}
  4508	
  4509	type randomEnum struct {
  4510		i     uint32
  4511		count uint32
  4512		pos   uint32
  4513		inc   uint32
  4514	}
  4515	
  4516	func (ord *randomOrder) reset(count uint32) {
  4517		ord.count = count
  4518		ord.coprimes = ord.coprimes[:0]
  4519		for i := uint32(1); i <= count; i++ {
  4520			if gcd(i, count) == 1 {
  4521				ord.coprimes = append(ord.coprimes, i)
  4522			}
  4523		}
  4524	}
  4525	
  4526	func (ord *randomOrder) start(i uint32) randomEnum {
  4527		return randomEnum{
  4528			count: ord.count,
  4529			pos:   i % ord.count,
  4530			inc:   ord.coprimes[i%uint32(len(ord.coprimes))],
  4531		}
  4532	}
  4533	
  4534	func (enum *randomEnum) done() bool {
  4535		return enum.i == enum.count
  4536	}
  4537	
  4538	func (enum *randomEnum) next() {
  4539		enum.i++
  4540		enum.pos = (enum.pos + enum.inc) % enum.count
  4541	}
  4542	
  4543	func (enum *randomEnum) position() uint32 {
  4544		return enum.pos
  4545	}
  4546	
  4547	func gcd(a, b uint32) uint32 {
  4548		for b != 0 {
  4549			a, b = b, a%b
  4550		}
  4551		return a
  4552	}
  4553	

View as plain text