...
Run Format

Source file src/runtime/stack.go

     1	// Copyright 2013 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	package runtime
     6	
     7	import (
     8		"runtime/internal/atomic"
     9		"runtime/internal/sys"
    10		"unsafe"
    11	)
    12	
    13	/*
    14	Stack layout parameters.
    15	Included both by runtime (compiled via 6c) and linkers (compiled via gcc).
    16	
    17	The per-goroutine g->stackguard is set to point StackGuard bytes
    18	above the bottom of the stack.  Each function compares its stack
    19	pointer against g->stackguard to check for overflow.  To cut one
    20	instruction from the check sequence for functions with tiny frames,
    21	the stack is allowed to protrude StackSmall bytes below the stack
    22	guard.  Functions with large frames don't bother with the check and
    23	always call morestack.  The sequences are (for amd64, others are
    24	similar):
    25	
    26		guard = g->stackguard
    27		frame = function's stack frame size
    28		argsize = size of function arguments (call + return)
    29	
    30		stack frame size <= StackSmall:
    31			CMPQ guard, SP
    32			JHI 3(PC)
    33			MOVQ m->morearg, $(argsize << 32)
    34			CALL morestack(SB)
    35	
    36		stack frame size > StackSmall but < StackBig
    37			LEAQ (frame-StackSmall)(SP), R0
    38			CMPQ guard, R0
    39			JHI 3(PC)
    40			MOVQ m->morearg, $(argsize << 32)
    41			CALL morestack(SB)
    42	
    43		stack frame size >= StackBig:
    44			MOVQ m->morearg, $((argsize << 32) | frame)
    45			CALL morestack(SB)
    46	
    47	The bottom StackGuard - StackSmall bytes are important: there has
    48	to be enough room to execute functions that refuse to check for
    49	stack overflow, either because they need to be adjacent to the
    50	actual caller's frame (deferproc) or because they handle the imminent
    51	stack overflow (morestack).
    52	
    53	For example, deferproc might call malloc, which does one of the
    54	above checks (without allocating a full frame), which might trigger
    55	a call to morestack.  This sequence needs to fit in the bottom
    56	section of the stack.  On amd64, morestack's frame is 40 bytes, and
    57	deferproc's frame is 56 bytes.  That fits well within the
    58	StackGuard - StackSmall bytes at the bottom.
    59	The linkers explore all possible call traces involving non-splitting
    60	functions to make sure that this limit cannot be violated.
    61	*/
    62	
    63	const (
    64		// StackSystem is a number of additional bytes to add
    65		// to each stack below the usual guard area for OS-specific
    66		// purposes like signal handling. Used on Windows, Plan 9,
    67		// and Darwin/ARM because they do not use a separate stack.
    68		_StackSystem = sys.GoosWindows*512*sys.PtrSize + sys.GoosPlan9*512 + sys.GoosDarwin*sys.GoarchArm*1024
    69	
    70		// The minimum size of stack used by Go code
    71		_StackMin = 2048
    72	
    73		// The minimum stack size to allocate.
    74		// The hackery here rounds FixedStack0 up to a power of 2.
    75		_FixedStack0 = _StackMin + _StackSystem
    76		_FixedStack1 = _FixedStack0 - 1
    77		_FixedStack2 = _FixedStack1 | (_FixedStack1 >> 1)
    78		_FixedStack3 = _FixedStack2 | (_FixedStack2 >> 2)
    79		_FixedStack4 = _FixedStack3 | (_FixedStack3 >> 4)
    80		_FixedStack5 = _FixedStack4 | (_FixedStack4 >> 8)
    81		_FixedStack6 = _FixedStack5 | (_FixedStack5 >> 16)
    82		_FixedStack  = _FixedStack6 + 1
    83	
    84		// Functions that need frames bigger than this use an extra
    85		// instruction to do the stack split check, to avoid overflow
    86		// in case SP - framesize wraps below zero.
    87		// This value can be no bigger than the size of the unmapped
    88		// space at zero.
    89		_StackBig = 4096
    90	
    91		// The stack guard is a pointer this many bytes above the
    92		// bottom of the stack.
    93		_StackGuard = 880*sys.StackGuardMultiplier + _StackSystem
    94	
    95		// After a stack split check the SP is allowed to be this
    96		// many bytes below the stack guard. This saves an instruction
    97		// in the checking sequence for tiny frames.
    98		_StackSmall = 128
    99	
   100		// The maximum number of bytes that a chain of NOSPLIT
   101		// functions can use.
   102		_StackLimit = _StackGuard - _StackSystem - _StackSmall
   103	)
   104	
   105	// Goroutine preemption request.
   106	// Stored into g->stackguard0 to cause split stack check failure.
   107	// Must be greater than any real sp.
   108	// 0xfffffade in hex.
   109	const (
   110		_StackPreempt = uintptrMask & -1314
   111		_StackFork    = uintptrMask & -1234
   112	)
   113	
   114	const (
   115		// stackDebug == 0: no logging
   116		//            == 1: logging of per-stack operations
   117		//            == 2: logging of per-frame operations
   118		//            == 3: logging of per-word updates
   119		//            == 4: logging of per-word reads
   120		stackDebug       = 0
   121		stackFromSystem  = 0 // allocate stacks from system memory instead of the heap
   122		stackFaultOnFree = 0 // old stacks are mapped noaccess to detect use after free
   123		stackPoisonCopy  = 0 // fill stack that should not be accessed with garbage, to detect bad dereferences during copy
   124	
   125		stackCache = 1
   126	
   127		// check the BP links during traceback.
   128		debugCheckBP = false
   129	)
   130	
   131	const (
   132		uintptrMask = 1<<(8*sys.PtrSize) - 1
   133	
   134		// Goroutine preemption request.
   135		// Stored into g->stackguard0 to cause split stack check failure.
   136		// Must be greater than any real sp.
   137		// 0xfffffade in hex.
   138		stackPreempt = uintptrMask & -1314
   139	
   140		// Thread is forking.
   141		// Stored into g->stackguard0 to cause split stack check failure.
   142		// Must be greater than any real sp.
   143		stackFork = uintptrMask & -1234
   144	)
   145	
   146	// Global pool of spans that have free stacks.
   147	// Stacks are assigned an order according to size.
   148	//     order = log_2(size/FixedStack)
   149	// There is a free list for each order.
   150	// TODO: one lock per order?
   151	var stackpool [_NumStackOrders]mSpanList
   152	var stackpoolmu mutex
   153	
   154	// Global pool of large stack spans.
   155	var stackLarge struct {
   156		lock mutex
   157		free [_MHeapMap_Bits]mSpanList // free lists by log_2(s.npages)
   158	}
   159	
   160	func stackinit() {
   161		if _StackCacheSize&_PageMask != 0 {
   162			throw("cache size must be a multiple of page size")
   163		}
   164		for i := range stackpool {
   165			stackpool[i].init()
   166		}
   167		for i := range stackLarge.free {
   168			stackLarge.free[i].init()
   169		}
   170	}
   171	
   172	// stacklog2 returns ⌊log_2(n)⌋.
   173	func stacklog2(n uintptr) int {
   174		log2 := 0
   175		for n > 1 {
   176			n >>= 1
   177			log2++
   178		}
   179		return log2
   180	}
   181	
   182	// Allocates a stack from the free pool. Must be called with
   183	// stackpoolmu held.
   184	func stackpoolalloc(order uint8) gclinkptr {
   185		list := &stackpool[order]
   186		s := list.first
   187		if s == nil {
   188			// no free stacks. Allocate another span worth.
   189			s = mheap_.allocStack(_StackCacheSize >> _PageShift)
   190			if s == nil {
   191				throw("out of memory")
   192			}
   193			if s.allocCount != 0 {
   194				throw("bad allocCount")
   195			}
   196			if s.stackfreelist.ptr() != nil {
   197				throw("bad stackfreelist")
   198			}
   199			for i := uintptr(0); i < _StackCacheSize; i += _FixedStack << order {
   200				x := gclinkptr(s.base() + i)
   201				x.ptr().next = s.stackfreelist
   202				s.stackfreelist = x
   203			}
   204			list.insert(s)
   205		}
   206		x := s.stackfreelist
   207		if x.ptr() == nil {
   208			throw("span has no free stacks")
   209		}
   210		s.stackfreelist = x.ptr().next
   211		s.allocCount++
   212		if s.stackfreelist.ptr() == nil {
   213			// all stacks in s are allocated.
   214			list.remove(s)
   215		}
   216		return x
   217	}
   218	
   219	// Adds stack x to the free pool. Must be called with stackpoolmu held.
   220	func stackpoolfree(x gclinkptr, order uint8) {
   221		s := mheap_.lookup(unsafe.Pointer(x))
   222		if s.state != _MSpanStack {
   223			throw("freeing stack not in a stack span")
   224		}
   225		if s.stackfreelist.ptr() == nil {
   226			// s will now have a free stack
   227			stackpool[order].insert(s)
   228		}
   229		x.ptr().next = s.stackfreelist
   230		s.stackfreelist = x
   231		s.allocCount--
   232		if gcphase == _GCoff && s.allocCount == 0 {
   233			// Span is completely free. Return it to the heap
   234			// immediately if we're sweeping.
   235			//
   236			// If GC is active, we delay the free until the end of
   237			// GC to avoid the following type of situation:
   238			//
   239			// 1) GC starts, scans a SudoG but does not yet mark the SudoG.elem pointer
   240			// 2) The stack that pointer points to is copied
   241			// 3) The old stack is freed
   242			// 4) The containing span is marked free
   243			// 5) GC attempts to mark the SudoG.elem pointer. The
   244			//    marking fails because the pointer looks like a
   245			//    pointer into a free span.
   246			//
   247			// By not freeing, we prevent step #4 until GC is done.
   248			stackpool[order].remove(s)
   249			s.stackfreelist = 0
   250			mheap_.freeStack(s)
   251		}
   252	}
   253	
   254	// stackcacherefill/stackcacherelease implement a global pool of stack segments.
   255	// The pool is required to prevent unlimited growth of per-thread caches.
   256	//
   257	//go:systemstack
   258	func stackcacherefill(c *mcache, order uint8) {
   259		if stackDebug >= 1 {
   260			print("stackcacherefill order=", order, "\n")
   261		}
   262	
   263		// Grab some stacks from the global cache.
   264		// Grab half of the allowed capacity (to prevent thrashing).
   265		var list gclinkptr
   266		var size uintptr
   267		lock(&stackpoolmu)
   268		for size < _StackCacheSize/2 {
   269			x := stackpoolalloc(order)
   270			x.ptr().next = list
   271			list = x
   272			size += _FixedStack << order
   273		}
   274		unlock(&stackpoolmu)
   275		c.stackcache[order].list = list
   276		c.stackcache[order].size = size
   277	}
   278	
   279	//go:systemstack
   280	func stackcacherelease(c *mcache, order uint8) {
   281		if stackDebug >= 1 {
   282			print("stackcacherelease order=", order, "\n")
   283		}
   284		x := c.stackcache[order].list
   285		size := c.stackcache[order].size
   286		lock(&stackpoolmu)
   287		for size > _StackCacheSize/2 {
   288			y := x.ptr().next
   289			stackpoolfree(x, order)
   290			x = y
   291			size -= _FixedStack << order
   292		}
   293		unlock(&stackpoolmu)
   294		c.stackcache[order].list = x
   295		c.stackcache[order].size = size
   296	}
   297	
   298	//go:systemstack
   299	func stackcache_clear(c *mcache) {
   300		if stackDebug >= 1 {
   301			print("stackcache clear\n")
   302		}
   303		lock(&stackpoolmu)
   304		for order := uint8(0); order < _NumStackOrders; order++ {
   305			x := c.stackcache[order].list
   306			for x.ptr() != nil {
   307				y := x.ptr().next
   308				stackpoolfree(x, order)
   309				x = y
   310			}
   311			c.stackcache[order].list = 0
   312			c.stackcache[order].size = 0
   313		}
   314		unlock(&stackpoolmu)
   315	}
   316	
   317	// stackalloc allocates an n byte stack.
   318	//
   319	// stackalloc must run on the system stack because it uses per-P
   320	// resources and must not split the stack.
   321	//
   322	//go:systemstack
   323	func stackalloc(n uint32) (stack, []stkbar) {
   324		// Stackalloc must be called on scheduler stack, so that we
   325		// never try to grow the stack during the code that stackalloc runs.
   326		// Doing so would cause a deadlock (issue 1547).
   327		thisg := getg()
   328		if thisg != thisg.m.g0 {
   329			throw("stackalloc not on scheduler stack")
   330		}
   331		if n&(n-1) != 0 {
   332			throw("stack size not a power of 2")
   333		}
   334		if stackDebug >= 1 {
   335			print("stackalloc ", n, "\n")
   336		}
   337	
   338		// Compute the size of stack barrier array.
   339		maxstkbar := gcMaxStackBarriers(int(n))
   340		nstkbar := unsafe.Sizeof(stkbar{}) * uintptr(maxstkbar)
   341		var stkbarSlice slice
   342	
   343		if debug.efence != 0 || stackFromSystem != 0 {
   344			v := sysAlloc(round(uintptr(n), _PageSize), &memstats.stacks_sys)
   345			if v == nil {
   346				throw("out of memory (stackalloc)")
   347			}
   348			top := uintptr(n) - nstkbar
   349			if maxstkbar != 0 {
   350				stkbarSlice = slice{add(v, top), 0, maxstkbar}
   351			}
   352			return stack{uintptr(v), uintptr(v) + top}, *(*[]stkbar)(unsafe.Pointer(&stkbarSlice))
   353		}
   354	
   355		// Small stacks are allocated with a fixed-size free-list allocator.
   356		// If we need a stack of a bigger size, we fall back on allocating
   357		// a dedicated span.
   358		var v unsafe.Pointer
   359		if stackCache != 0 && n < _FixedStack<<_NumStackOrders && n < _StackCacheSize {
   360			order := uint8(0)
   361			n2 := n
   362			for n2 > _FixedStack {
   363				order++
   364				n2 >>= 1
   365			}
   366			var x gclinkptr
   367			c := thisg.m.mcache
   368			if c == nil || thisg.m.preemptoff != "" || thisg.m.helpgc != 0 {
   369				// c == nil can happen in the guts of exitsyscall or
   370				// procresize. Just get a stack from the global pool.
   371				// Also don't touch stackcache during gc
   372				// as it's flushed concurrently.
   373				lock(&stackpoolmu)
   374				x = stackpoolalloc(order)
   375				unlock(&stackpoolmu)
   376			} else {
   377				x = c.stackcache[order].list
   378				if x.ptr() == nil {
   379					stackcacherefill(c, order)
   380					x = c.stackcache[order].list
   381				}
   382				c.stackcache[order].list = x.ptr().next
   383				c.stackcache[order].size -= uintptr(n)
   384			}
   385			v = unsafe.Pointer(x)
   386		} else {
   387			var s *mspan
   388			npage := uintptr(n) >> _PageShift
   389			log2npage := stacklog2(npage)
   390	
   391			// Try to get a stack from the large stack cache.
   392			lock(&stackLarge.lock)
   393			if !stackLarge.free[log2npage].isEmpty() {
   394				s = stackLarge.free[log2npage].first
   395				stackLarge.free[log2npage].remove(s)
   396			}
   397			unlock(&stackLarge.lock)
   398	
   399			if s == nil {
   400				// Allocate a new stack from the heap.
   401				s = mheap_.allocStack(npage)
   402				if s == nil {
   403					throw("out of memory")
   404				}
   405			}
   406			v = unsafe.Pointer(s.base())
   407		}
   408	
   409		if raceenabled {
   410			racemalloc(v, uintptr(n))
   411		}
   412		if msanenabled {
   413			msanmalloc(v, uintptr(n))
   414		}
   415		if stackDebug >= 1 {
   416			print("  allocated ", v, "\n")
   417		}
   418		top := uintptr(n) - nstkbar
   419		if maxstkbar != 0 {
   420			stkbarSlice = slice{add(v, top), 0, maxstkbar}
   421		}
   422		return stack{uintptr(v), uintptr(v) + top}, *(*[]stkbar)(unsafe.Pointer(&stkbarSlice))
   423	}
   424	
   425	// stackfree frees an n byte stack allocation at stk.
   426	//
   427	// stackfree must run on the system stack because it uses per-P
   428	// resources and must not split the stack.
   429	//
   430	//go:systemstack
   431	func stackfree(stk stack, n uintptr) {
   432		gp := getg()
   433		v := unsafe.Pointer(stk.lo)
   434		if n&(n-1) != 0 {
   435			throw("stack not a power of 2")
   436		}
   437		if stk.lo+n < stk.hi {
   438			throw("bad stack size")
   439		}
   440		if stackDebug >= 1 {
   441			println("stackfree", v, n)
   442			memclrNoHeapPointers(v, n) // for testing, clobber stack data
   443		}
   444		if debug.efence != 0 || stackFromSystem != 0 {
   445			if debug.efence != 0 || stackFaultOnFree != 0 {
   446				sysFault(v, n)
   447			} else {
   448				sysFree(v, n, &memstats.stacks_sys)
   449			}
   450			return
   451		}
   452		if msanenabled {
   453			msanfree(v, n)
   454		}
   455		if stackCache != 0 && n < _FixedStack<<_NumStackOrders && n < _StackCacheSize {
   456			order := uint8(0)
   457			n2 := n
   458			for n2 > _FixedStack {
   459				order++
   460				n2 >>= 1
   461			}
   462			x := gclinkptr(v)
   463			c := gp.m.mcache
   464			if c == nil || gp.m.preemptoff != "" || gp.m.helpgc != 0 {
   465				lock(&stackpoolmu)
   466				stackpoolfree(x, order)
   467				unlock(&stackpoolmu)
   468			} else {
   469				if c.stackcache[order].size >= _StackCacheSize {
   470					stackcacherelease(c, order)
   471				}
   472				x.ptr().next = c.stackcache[order].list
   473				c.stackcache[order].list = x
   474				c.stackcache[order].size += n
   475			}
   476		} else {
   477			s := mheap_.lookup(v)
   478			if s.state != _MSpanStack {
   479				println(hex(s.base()), v)
   480				throw("bad span state")
   481			}
   482			if gcphase == _GCoff {
   483				// Free the stack immediately if we're
   484				// sweeping.
   485				mheap_.freeStack(s)
   486			} else {
   487				// If the GC is running, we can't return a
   488				// stack span to the heap because it could be
   489				// reused as a heap span, and this state
   490				// change would race with GC. Add it to the
   491				// large stack cache instead.
   492				log2npage := stacklog2(s.npages)
   493				lock(&stackLarge.lock)
   494				stackLarge.free[log2npage].insert(s)
   495				unlock(&stackLarge.lock)
   496			}
   497		}
   498	}
   499	
   500	var maxstacksize uintptr = 1 << 20 // enough until runtime.main sets it for real
   501	
   502	var ptrnames = []string{
   503		0: "scalar",
   504		1: "ptr",
   505	}
   506	
   507	// Stack frame layout
   508	//
   509	// (x86)
   510	// +------------------+
   511	// | args from caller |
   512	// +------------------+ <- frame->argp
   513	// |  return address  |
   514	// +------------------+
   515	// |  caller's BP (*) | (*) if framepointer_enabled && varp < sp
   516	// +------------------+ <- frame->varp
   517	// |     locals       |
   518	// +------------------+
   519	// |  args to callee  |
   520	// +------------------+ <- frame->sp
   521	//
   522	// (arm)
   523	// +------------------+
   524	// | args from caller |
   525	// +------------------+ <- frame->argp
   526	// | caller's retaddr |
   527	// +------------------+ <- frame->varp
   528	// |     locals       |
   529	// +------------------+
   530	// |  args to callee  |
   531	// +------------------+
   532	// |  return address  |
   533	// +------------------+ <- frame->sp
   534	
   535	type adjustinfo struct {
   536		old   stack
   537		delta uintptr // ptr distance from old to new stack (newbase - oldbase)
   538		cache pcvalueCache
   539	
   540		// sghi is the highest sudog.elem on the stack.
   541		sghi uintptr
   542	}
   543	
   544	// Adjustpointer checks whether *vpp is in the old stack described by adjinfo.
   545	// If so, it rewrites *vpp to point into the new stack.
   546	func adjustpointer(adjinfo *adjustinfo, vpp unsafe.Pointer) {
   547		pp := (*uintptr)(vpp)
   548		p := *pp
   549		if stackDebug >= 4 {
   550			print("        ", pp, ":", hex(p), "\n")
   551		}
   552		if adjinfo.old.lo <= p && p < adjinfo.old.hi {
   553			*pp = p + adjinfo.delta
   554			if stackDebug >= 3 {
   555				print("        adjust ptr ", pp, ":", hex(p), " -> ", hex(*pp), "\n")
   556			}
   557		}
   558	}
   559	
   560	// Information from the compiler about the layout of stack frames.
   561	type bitvector struct {
   562		n        int32 // # of bits
   563		bytedata *uint8
   564	}
   565	
   566	type gobitvector struct {
   567		n        uintptr
   568		bytedata []uint8
   569	}
   570	
   571	func gobv(bv bitvector) gobitvector {
   572		return gobitvector{
   573			uintptr(bv.n),
   574			(*[1 << 30]byte)(unsafe.Pointer(bv.bytedata))[:(bv.n+7)/8],
   575		}
   576	}
   577	
   578	func ptrbit(bv *gobitvector, i uintptr) uint8 {
   579		return (bv.bytedata[i/8] >> (i % 8)) & 1
   580	}
   581	
   582	// bv describes the memory starting at address scanp.
   583	// Adjust any pointers contained therein.
   584	func adjustpointers(scanp unsafe.Pointer, cbv *bitvector, adjinfo *adjustinfo, f *_func) {
   585		bv := gobv(*cbv)
   586		minp := adjinfo.old.lo
   587		maxp := adjinfo.old.hi
   588		delta := adjinfo.delta
   589		num := bv.n
   590		// If this frame might contain channel receive slots, use CAS
   591		// to adjust pointers. If the slot hasn't been received into
   592		// yet, it may contain stack pointers and a concurrent send
   593		// could race with adjusting those pointers. (The sent value
   594		// itself can never contain stack pointers.)
   595		useCAS := uintptr(scanp) < adjinfo.sghi
   596		for i := uintptr(0); i < num; i++ {
   597			if stackDebug >= 4 {
   598				print("        ", add(scanp, i*sys.PtrSize), ":", ptrnames[ptrbit(&bv, i)], ":", hex(*(*uintptr)(add(scanp, i*sys.PtrSize))), " # ", i, " ", bv.bytedata[i/8], "\n")
   599			}
   600			if ptrbit(&bv, i) == 1 {
   601				pp := (*uintptr)(add(scanp, i*sys.PtrSize))
   602			retry:
   603				p := *pp
   604				if f != nil && 0 < p && p < minLegalPointer && debug.invalidptr != 0 {
   605					// Looks like a junk value in a pointer slot.
   606					// Live analysis wrong?
   607					getg().m.traceback = 2
   608					print("runtime: bad pointer in frame ", funcname(f), " at ", pp, ": ", hex(p), "\n")
   609					throw("invalid pointer found on stack")
   610				}
   611				if minp <= p && p < maxp {
   612					if stackDebug >= 3 {
   613						print("adjust ptr ", hex(p), " ", funcname(f), "\n")
   614					}
   615					if useCAS {
   616						ppu := (*unsafe.Pointer)(unsafe.Pointer(pp))
   617						if !atomic.Casp1(ppu, unsafe.Pointer(p), unsafe.Pointer(p+delta)) {
   618							goto retry
   619						}
   620					} else {
   621						*pp = p + delta
   622					}
   623				}
   624			}
   625		}
   626	}
   627	
   628	// Note: the argument/return area is adjusted by the callee.
   629	func adjustframe(frame *stkframe, arg unsafe.Pointer) bool {
   630		adjinfo := (*adjustinfo)(arg)
   631		targetpc := frame.continpc
   632		if targetpc == 0 {
   633			// Frame is dead.
   634			return true
   635		}
   636		f := frame.fn
   637		if stackDebug >= 2 {
   638			print("    adjusting ", funcname(f), " frame=[", hex(frame.sp), ",", hex(frame.fp), "] pc=", hex(frame.pc), " continpc=", hex(frame.continpc), "\n")
   639		}
   640		if f.entry == systemstack_switchPC {
   641			// A special routine at the bottom of stack of a goroutine that does an systemstack call.
   642			// We will allow it to be copied even though we don't
   643			// have full GC info for it (because it is written in asm).
   644			return true
   645		}
   646		if targetpc != f.entry {
   647			targetpc--
   648		}
   649		pcdata := pcdatavalue(f, _PCDATA_StackMapIndex, targetpc, &adjinfo.cache)
   650		if pcdata == -1 {
   651			pcdata = 0 // in prologue
   652		}
   653	
   654		// Adjust local variables if stack frame has been allocated.
   655		size := frame.varp - frame.sp
   656		var minsize uintptr
   657		switch sys.ArchFamily {
   658		case sys.ARM64:
   659			minsize = sys.SpAlign
   660		default:
   661			minsize = sys.MinFrameSize
   662		}
   663		if size > minsize {
   664			var bv bitvector
   665			stackmap := (*stackmap)(funcdata(f, _FUNCDATA_LocalsPointerMaps))
   666			if stackmap == nil || stackmap.n <= 0 {
   667				print("runtime: frame ", funcname(f), " untyped locals ", hex(frame.varp-size), "+", hex(size), "\n")
   668				throw("missing stackmap")
   669			}
   670			// Locals bitmap information, scan just the pointers in locals.
   671			if pcdata < 0 || pcdata >= stackmap.n {
   672				// don't know where we are
   673				print("runtime: pcdata is ", pcdata, " and ", stackmap.n, " locals stack map entries for ", funcname(f), " (targetpc=", targetpc, ")\n")
   674				throw("bad symbol table")
   675			}
   676			bv = stackmapdata(stackmap, pcdata)
   677			size = uintptr(bv.n) * sys.PtrSize
   678			if stackDebug >= 3 {
   679				print("      locals ", pcdata, "/", stackmap.n, " ", size/sys.PtrSize, " words ", bv.bytedata, "\n")
   680			}
   681			adjustpointers(unsafe.Pointer(frame.varp-size), &bv, adjinfo, f)
   682		}
   683	
   684		// Adjust saved base pointer if there is one.
   685		if sys.ArchFamily == sys.AMD64 && frame.argp-frame.varp == 2*sys.RegSize {
   686			if !framepointer_enabled {
   687				print("runtime: found space for saved base pointer, but no framepointer experiment\n")
   688				print("argp=", hex(frame.argp), " varp=", hex(frame.varp), "\n")
   689				throw("bad frame layout")
   690			}
   691			if stackDebug >= 3 {
   692				print("      saved bp\n")
   693			}
   694			if debugCheckBP {
   695				// Frame pointers should always point to the next higher frame on
   696				// the Go stack (or be nil, for the top frame on the stack).
   697				bp := *(*uintptr)(unsafe.Pointer(frame.varp))
   698				if bp != 0 && (bp < adjinfo.old.lo || bp >= adjinfo.old.hi) {
   699					println("runtime: found invalid frame pointer")
   700					print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n")
   701					throw("bad frame pointer")
   702				}
   703			}
   704			adjustpointer(adjinfo, unsafe.Pointer(frame.varp))
   705		}
   706	
   707		// Adjust arguments.
   708		if frame.arglen > 0 {
   709			var bv bitvector
   710			if frame.argmap != nil {
   711				bv = *frame.argmap
   712			} else {
   713				stackmap := (*stackmap)(funcdata(f, _FUNCDATA_ArgsPointerMaps))
   714				if stackmap == nil || stackmap.n <= 0 {
   715					print("runtime: frame ", funcname(f), " untyped args ", frame.argp, "+", frame.arglen, "\n")
   716					throw("missing stackmap")
   717				}
   718				if pcdata < 0 || pcdata >= stackmap.n {
   719					// don't know where we are
   720					print("runtime: pcdata is ", pcdata, " and ", stackmap.n, " args stack map entries for ", funcname(f), " (targetpc=", targetpc, ")\n")
   721					throw("bad symbol table")
   722				}
   723				bv = stackmapdata(stackmap, pcdata)
   724			}
   725			if stackDebug >= 3 {
   726				print("      args\n")
   727			}
   728			adjustpointers(unsafe.Pointer(frame.argp), &bv, adjinfo, nil)
   729		}
   730		return true
   731	}
   732	
   733	func adjustctxt(gp *g, adjinfo *adjustinfo) {
   734		adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.ctxt))
   735		if !framepointer_enabled {
   736			return
   737		}
   738		if debugCheckBP {
   739			bp := gp.sched.bp
   740			if bp != 0 && (bp < adjinfo.old.lo || bp >= adjinfo.old.hi) {
   741				println("runtime: found invalid top frame pointer")
   742				print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n")
   743				throw("bad top frame pointer")
   744			}
   745		}
   746		adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.bp))
   747	}
   748	
   749	func adjustdefers(gp *g, adjinfo *adjustinfo) {
   750		// Adjust defer argument blocks the same way we adjust active stack frames.
   751		tracebackdefers(gp, adjustframe, noescape(unsafe.Pointer(adjinfo)))
   752	
   753		// Adjust pointers in the Defer structs.
   754		// Defer structs themselves are never on the stack.
   755		for d := gp._defer; d != nil; d = d.link {
   756			adjustpointer(adjinfo, unsafe.Pointer(&d.fn))
   757			adjustpointer(adjinfo, unsafe.Pointer(&d.sp))
   758			adjustpointer(adjinfo, unsafe.Pointer(&d._panic))
   759		}
   760	}
   761	
   762	func adjustpanics(gp *g, adjinfo *adjustinfo) {
   763		// Panics are on stack and already adjusted.
   764		// Update pointer to head of list in G.
   765		adjustpointer(adjinfo, unsafe.Pointer(&gp._panic))
   766	}
   767	
   768	func adjustsudogs(gp *g, adjinfo *adjustinfo) {
   769		// the data elements pointed to by a SudoG structure
   770		// might be in the stack.
   771		for s := gp.waiting; s != nil; s = s.waitlink {
   772			adjustpointer(adjinfo, unsafe.Pointer(&s.elem))
   773			adjustpointer(adjinfo, unsafe.Pointer(&s.selectdone))
   774		}
   775	}
   776	
   777	func adjuststkbar(gp *g, adjinfo *adjustinfo) {
   778		for i := int(gp.stkbarPos); i < len(gp.stkbar); i++ {
   779			adjustpointer(adjinfo, unsafe.Pointer(&gp.stkbar[i].savedLRPtr))
   780		}
   781	}
   782	
   783	func fillstack(stk stack, b byte) {
   784		for p := stk.lo; p < stk.hi; p++ {
   785			*(*byte)(unsafe.Pointer(p)) = b
   786		}
   787	}
   788	
   789	func findsghi(gp *g, stk stack) uintptr {
   790		var sghi uintptr
   791		for sg := gp.waiting; sg != nil; sg = sg.waitlink {
   792			p := uintptr(sg.elem) + uintptr(sg.c.elemsize)
   793			if stk.lo <= p && p < stk.hi && p > sghi {
   794				sghi = p
   795			}
   796			p = uintptr(unsafe.Pointer(sg.selectdone)) + unsafe.Sizeof(sg.selectdone)
   797			if stk.lo <= p && p < stk.hi && p > sghi {
   798				sghi = p
   799			}
   800		}
   801		return sghi
   802	}
   803	
   804	// syncadjustsudogs adjusts gp's sudogs and copies the part of gp's
   805	// stack they refer to while synchronizing with concurrent channel
   806	// operations. It returns the number of bytes of stack copied.
   807	func syncadjustsudogs(gp *g, used uintptr, adjinfo *adjustinfo) uintptr {
   808		if gp.waiting == nil {
   809			return 0
   810		}
   811	
   812		// Lock channels to prevent concurrent send/receive.
   813		// It's important that we *only* do this for async
   814		// copystack; otherwise, gp may be in the middle of
   815		// putting itself on wait queues and this would
   816		// self-deadlock.
   817		var lastc *hchan
   818		for sg := gp.waiting; sg != nil; sg = sg.waitlink {
   819			if sg.c != lastc {
   820				lock(&sg.c.lock)
   821			}
   822			lastc = sg.c
   823		}
   824	
   825		// Adjust sudogs.
   826		adjustsudogs(gp, adjinfo)
   827	
   828		// Copy the part of the stack the sudogs point in to
   829		// while holding the lock to prevent races on
   830		// send/receive slots.
   831		var sgsize uintptr
   832		if adjinfo.sghi != 0 {
   833			oldBot := adjinfo.old.hi - used
   834			newBot := oldBot + adjinfo.delta
   835			sgsize = adjinfo.sghi - oldBot
   836			memmove(unsafe.Pointer(newBot), unsafe.Pointer(oldBot), sgsize)
   837		}
   838	
   839		// Unlock channels.
   840		lastc = nil
   841		for sg := gp.waiting; sg != nil; sg = sg.waitlink {
   842			if sg.c != lastc {
   843				unlock(&sg.c.lock)
   844			}
   845			lastc = sg.c
   846		}
   847	
   848		return sgsize
   849	}
   850	
   851	// Copies gp's stack to a new stack of a different size.
   852	// Caller must have changed gp status to Gcopystack.
   853	//
   854	// If sync is true, this is a self-triggered stack growth and, in
   855	// particular, no other G may be writing to gp's stack (e.g., via a
   856	// channel operation). If sync is false, copystack protects against
   857	// concurrent channel operations.
   858	func copystack(gp *g, newsize uintptr, sync bool) {
   859		if gp.syscallsp != 0 {
   860			throw("stack growth not allowed in system call")
   861		}
   862		old := gp.stack
   863		if old.lo == 0 {
   864			throw("nil stackbase")
   865		}
   866		used := old.hi - gp.sched.sp
   867	
   868		// allocate new stack
   869		new, newstkbar := stackalloc(uint32(newsize))
   870		if stackPoisonCopy != 0 {
   871			fillstack(new, 0xfd)
   872		}
   873		if stackDebug >= 1 {
   874			print("copystack gp=", gp, " [", hex(old.lo), " ", hex(old.hi-used), " ", hex(old.hi), "]/", gp.stackAlloc, " -> [", hex(new.lo), " ", hex(new.hi-used), " ", hex(new.hi), "]/", newsize, "\n")
   875		}
   876	
   877		// Compute adjustment.
   878		var adjinfo adjustinfo
   879		adjinfo.old = old
   880		adjinfo.delta = new.hi - old.hi
   881	
   882		// Adjust sudogs, synchronizing with channel ops if necessary.
   883		ncopy := used
   884		if sync {
   885			adjustsudogs(gp, &adjinfo)
   886		} else {
   887			// sudogs can point in to the stack. During concurrent
   888			// shrinking, these areas may be written to. Find the
   889			// highest such pointer so we can handle everything
   890			// there and below carefully. (This shouldn't be far
   891			// from the bottom of the stack, so there's little
   892			// cost in handling everything below it carefully.)
   893			adjinfo.sghi = findsghi(gp, old)
   894	
   895			// Synchronize with channel ops and copy the part of
   896			// the stack they may interact with.
   897			ncopy -= syncadjustsudogs(gp, used, &adjinfo)
   898		}
   899	
   900		// Copy the stack (or the rest of it) to the new location
   901		memmove(unsafe.Pointer(new.hi-ncopy), unsafe.Pointer(old.hi-ncopy), ncopy)
   902	
   903		// Disallow sigprof scans of this stack and block if there's
   904		// one in progress.
   905		gcLockStackBarriers(gp)
   906	
   907		// Adjust remaining structures that have pointers into stacks.
   908		// We have to do most of these before we traceback the new
   909		// stack because gentraceback uses them.
   910		adjustctxt(gp, &adjinfo)
   911		adjustdefers(gp, &adjinfo)
   912		adjustpanics(gp, &adjinfo)
   913		adjuststkbar(gp, &adjinfo)
   914		if adjinfo.sghi != 0 {
   915			adjinfo.sghi += adjinfo.delta
   916		}
   917	
   918		// copy old stack barriers to new stack barrier array
   919		newstkbar = newstkbar[:len(gp.stkbar)]
   920		copy(newstkbar, gp.stkbar)
   921	
   922		// Swap out old stack for new one
   923		gp.stack = new
   924		gp.stackguard0 = new.lo + _StackGuard // NOTE: might clobber a preempt request
   925		gp.sched.sp = new.hi - used
   926		oldsize := gp.stackAlloc
   927		gp.stackAlloc = newsize
   928		gp.stkbar = newstkbar
   929		gp.stktopsp += adjinfo.delta
   930	
   931		// Adjust pointers in the new stack.
   932		gentraceback(^uintptr(0), ^uintptr(0), 0, gp, 0, nil, 0x7fffffff, adjustframe, noescape(unsafe.Pointer(&adjinfo)), 0)
   933	
   934		gcUnlockStackBarriers(gp)
   935	
   936		// free old stack
   937		if stackPoisonCopy != 0 {
   938			fillstack(old, 0xfc)
   939		}
   940		stackfree(old, oldsize)
   941	}
   942	
   943	// round x up to a power of 2.
   944	func round2(x int32) int32 {
   945		s := uint(0)
   946		for 1<<s < x {
   947			s++
   948		}
   949		return 1 << s
   950	}
   951	
   952	// Called from runtime·morestack when more stack is needed.
   953	// Allocate larger stack and relocate to new stack.
   954	// Stack growth is multiplicative, for constant amortized cost.
   955	//
   956	// g->atomicstatus will be Grunning or Gscanrunning upon entry.
   957	// If the GC is trying to stop this g then it will set preemptscan to true.
   958	//
   959	// ctxt is the value of the context register on morestack. newstack
   960	// will write it to g.sched.ctxt.
   961	func newstack(ctxt unsafe.Pointer) {
   962		thisg := getg()
   963		// TODO: double check all gp. shouldn't be getg().
   964		if thisg.m.morebuf.g.ptr().stackguard0 == stackFork {
   965			throw("stack growth after fork")
   966		}
   967		if thisg.m.morebuf.g.ptr() != thisg.m.curg {
   968			print("runtime: newstack called from g=", hex(thisg.m.morebuf.g), "\n"+"\tm=", thisg.m, " m->curg=", thisg.m.curg, " m->g0=", thisg.m.g0, " m->gsignal=", thisg.m.gsignal, "\n")
   969			morebuf := thisg.m.morebuf
   970			traceback(morebuf.pc, morebuf.sp, morebuf.lr, morebuf.g.ptr())
   971			throw("runtime: wrong goroutine in newstack")
   972		}
   973	
   974		gp := thisg.m.curg
   975		// Write ctxt to gp.sched. We do this here instead of in
   976		// morestack so it has the necessary write barrier.
   977		gp.sched.ctxt = ctxt
   978	
   979		if thisg.m.curg.throwsplit {
   980			// Update syscallsp, syscallpc in case traceback uses them.
   981			morebuf := thisg.m.morebuf
   982			gp.syscallsp = morebuf.sp
   983			gp.syscallpc = morebuf.pc
   984			print("runtime: newstack sp=", hex(gp.sched.sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n",
   985				"\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n",
   986				"\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n")
   987	
   988			traceback(morebuf.pc, morebuf.sp, morebuf.lr, gp)
   989			throw("runtime: stack split at bad time")
   990		}
   991	
   992		morebuf := thisg.m.morebuf
   993		thisg.m.morebuf.pc = 0
   994		thisg.m.morebuf.lr = 0
   995		thisg.m.morebuf.sp = 0
   996		thisg.m.morebuf.g = 0
   997	
   998		// NOTE: stackguard0 may change underfoot, if another thread
   999		// is about to try to preempt gp. Read it just once and use that same
  1000		// value now and below.
  1001		preempt := atomic.Loaduintptr(&gp.stackguard0) == stackPreempt
  1002	
  1003		// Be conservative about where we preempt.
  1004		// We are interested in preempting user Go code, not runtime code.
  1005		// If we're holding locks, mallocing, or preemption is disabled, don't
  1006		// preempt.
  1007		// This check is very early in newstack so that even the status change
  1008		// from Grunning to Gwaiting and back doesn't happen in this case.
  1009		// That status change by itself can be viewed as a small preemption,
  1010		// because the GC might change Gwaiting to Gscanwaiting, and then
  1011		// this goroutine has to wait for the GC to finish before continuing.
  1012		// If the GC is in some way dependent on this goroutine (for example,
  1013		// it needs a lock held by the goroutine), that small preemption turns
  1014		// into a real deadlock.
  1015		if preempt {
  1016			if thisg.m.locks != 0 || thisg.m.mallocing != 0 || thisg.m.preemptoff != "" || thisg.m.p.ptr().status != _Prunning {
  1017				// Let the goroutine keep running for now.
  1018				// gp->preempt is set, so it will be preempted next time.
  1019				gp.stackguard0 = gp.stack.lo + _StackGuard
  1020				gogo(&gp.sched) // never return
  1021			}
  1022		}
  1023	
  1024		if gp.stack.lo == 0 {
  1025			throw("missing stack in newstack")
  1026		}
  1027		sp := gp.sched.sp
  1028		if sys.ArchFamily == sys.AMD64 || sys.ArchFamily == sys.I386 {
  1029			// The call to morestack cost a word.
  1030			sp -= sys.PtrSize
  1031		}
  1032		if stackDebug >= 1 || sp < gp.stack.lo {
  1033			print("runtime: newstack sp=", hex(sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n",
  1034				"\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n",
  1035				"\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n")
  1036		}
  1037		if sp < gp.stack.lo {
  1038			print("runtime: gp=", gp, ", gp->status=", hex(readgstatus(gp)), "\n ")
  1039			print("runtime: split stack overflow: ", hex(sp), " < ", hex(gp.stack.lo), "\n")
  1040			throw("runtime: split stack overflow")
  1041		}
  1042	
  1043		if preempt {
  1044			if gp == thisg.m.g0 {
  1045				throw("runtime: preempt g0")
  1046			}
  1047			if thisg.m.p == 0 && thisg.m.locks == 0 {
  1048				throw("runtime: g is running but p is not")
  1049			}
  1050			// Synchronize with scang.
  1051			casgstatus(gp, _Grunning, _Gwaiting)
  1052			if gp.preemptscan {
  1053				for !castogscanstatus(gp, _Gwaiting, _Gscanwaiting) {
  1054					// Likely to be racing with the GC as
  1055					// it sees a _Gwaiting and does the
  1056					// stack scan. If so, gcworkdone will
  1057					// be set and gcphasework will simply
  1058					// return.
  1059				}
  1060				if !gp.gcscandone {
  1061					// gcw is safe because we're on the
  1062					// system stack.
  1063					gcw := &gp.m.p.ptr().gcw
  1064					scanstack(gp, gcw)
  1065					if gcBlackenPromptly {
  1066						gcw.dispose()
  1067					}
  1068					gp.gcscandone = true
  1069				}
  1070				gp.preemptscan = false
  1071				gp.preempt = false
  1072				casfrom_Gscanstatus(gp, _Gscanwaiting, _Gwaiting)
  1073				// This clears gcscanvalid.
  1074				casgstatus(gp, _Gwaiting, _Grunning)
  1075				gp.stackguard0 = gp.stack.lo + _StackGuard
  1076				gogo(&gp.sched) // never return
  1077			}
  1078	
  1079			// Act like goroutine called runtime.Gosched.
  1080			casgstatus(gp, _Gwaiting, _Grunning)
  1081			gopreempt_m(gp) // never return
  1082		}
  1083	
  1084		// Allocate a bigger segment and move the stack.
  1085		oldsize := int(gp.stackAlloc)
  1086		newsize := oldsize * 2
  1087		if uintptr(newsize) > maxstacksize {
  1088			print("runtime: goroutine stack exceeds ", maxstacksize, "-byte limit\n")
  1089			throw("stack overflow")
  1090		}
  1091	
  1092		// The goroutine must be executing in order to call newstack,
  1093		// so it must be Grunning (or Gscanrunning).
  1094		casgstatus(gp, _Grunning, _Gcopystack)
  1095	
  1096		// The concurrent GC will not scan the stack while we are doing the copy since
  1097		// the gp is in a Gcopystack status.
  1098		copystack(gp, uintptr(newsize), true)
  1099		if stackDebug >= 1 {
  1100			print("stack grow done\n")
  1101		}
  1102		casgstatus(gp, _Gcopystack, _Grunning)
  1103		gogo(&gp.sched)
  1104	}
  1105	
  1106	//go:nosplit
  1107	func nilfunc() {
  1108		*(*uint8)(nil) = 0
  1109	}
  1110	
  1111	// adjust Gobuf as if it executed a call to fn
  1112	// and then did an immediate gosave.
  1113	func gostartcallfn(gobuf *gobuf, fv *funcval) {
  1114		var fn unsafe.Pointer
  1115		if fv != nil {
  1116			fn = unsafe.Pointer(fv.fn)
  1117		} else {
  1118			fn = unsafe.Pointer(funcPC(nilfunc))
  1119		}
  1120		gostartcall(gobuf, fn, unsafe.Pointer(fv))
  1121	}
  1122	
  1123	// Maybe shrink the stack being used by gp.
  1124	// Called at garbage collection time.
  1125	// gp must be stopped, but the world need not be.
  1126	func shrinkstack(gp *g) {
  1127		gstatus := readgstatus(gp)
  1128		if gstatus&^_Gscan == _Gdead {
  1129			if gp.stack.lo != 0 {
  1130				// Free whole stack - it will get reallocated
  1131				// if G is used again.
  1132				stackfree(gp.stack, gp.stackAlloc)
  1133				gp.stack.lo = 0
  1134				gp.stack.hi = 0
  1135				gp.stkbar = nil
  1136				gp.stkbarPos = 0
  1137			}
  1138			return
  1139		}
  1140		if gp.stack.lo == 0 {
  1141			throw("missing stack in shrinkstack")
  1142		}
  1143		if gstatus&_Gscan == 0 {
  1144			throw("bad status in shrinkstack")
  1145		}
  1146	
  1147		if debug.gcshrinkstackoff > 0 {
  1148			return
  1149		}
  1150		if gp.startpc == gcBgMarkWorkerPC {
  1151			// We're not allowed to shrink the gcBgMarkWorker
  1152			// stack (see gcBgMarkWorker for explanation).
  1153			return
  1154		}
  1155	
  1156		oldsize := gp.stackAlloc
  1157		newsize := oldsize / 2
  1158		// Don't shrink the allocation below the minimum-sized stack
  1159		// allocation.
  1160		if newsize < _FixedStack {
  1161			return
  1162		}
  1163		// Compute how much of the stack is currently in use and only
  1164		// shrink the stack if gp is using less than a quarter of its
  1165		// current stack. The currently used stack includes everything
  1166		// down to the SP plus the stack guard space that ensures
  1167		// there's room for nosplit functions.
  1168		avail := gp.stack.hi - gp.stack.lo
  1169		if used := gp.stack.hi - gp.sched.sp + _StackLimit; used >= avail/4 {
  1170			return
  1171		}
  1172	
  1173		// We can't copy the stack if we're in a syscall.
  1174		// The syscall might have pointers into the stack.
  1175		if gp.syscallsp != 0 {
  1176			return
  1177		}
  1178		if sys.GoosWindows != 0 && gp.m != nil && gp.m.libcallsp != 0 {
  1179			return
  1180		}
  1181	
  1182		if stackDebug > 0 {
  1183			print("shrinking stack ", oldsize, "->", newsize, "\n")
  1184		}
  1185	
  1186		copystack(gp, newsize, false)
  1187	}
  1188	
  1189	// freeStackSpans frees unused stack spans at the end of GC.
  1190	func freeStackSpans() {
  1191		lock(&stackpoolmu)
  1192	
  1193		// Scan stack pools for empty stack spans.
  1194		for order := range stackpool {
  1195			list := &stackpool[order]
  1196			for s := list.first; s != nil; {
  1197				next := s.next
  1198				if s.allocCount == 0 {
  1199					list.remove(s)
  1200					s.stackfreelist = 0
  1201					mheap_.freeStack(s)
  1202				}
  1203				s = next
  1204			}
  1205		}
  1206	
  1207		unlock(&stackpoolmu)
  1208	
  1209		// Free large stack spans.
  1210		lock(&stackLarge.lock)
  1211		for i := range stackLarge.free {
  1212			for s := stackLarge.free[i].first; s != nil; {
  1213				next := s.next
  1214				stackLarge.free[i].remove(s)
  1215				mheap_.freeStack(s)
  1216				s = next
  1217			}
  1218		}
  1219		unlock(&stackLarge.lock)
  1220	}
  1221	
  1222	//go:nosplit
  1223	func morestackc() {
  1224		systemstack(func() {
  1225			throw("attempt to execute C code on Go stack")
  1226		})
  1227	}
  1228	

View as plain text