Source file src/runtime/stack.go

     1  // Copyright 2013 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/abi"
     9  	"internal/cpu"
    10  	"internal/goarch"
    11  	"internal/goexperiment"
    12  	"internal/goos"
    13  	"internal/runtime/atomic"
    14  	"internal/runtime/gc"
    15  	"internal/runtime/sys"
    16  	"math/bits"
    17  	"unsafe"
    18  )
    19  
    20  /*
    21  Stack layout parameters.
    22  Included both by runtime (compiled via 6c) and linkers (compiled via gcc).
    23  
    24  The per-goroutine g->stackguard is set to point StackGuard bytes
    25  above the bottom of the stack.  Each function compares its stack
    26  pointer against g->stackguard to check for overflow.  To cut one
    27  instruction from the check sequence for functions with tiny frames,
    28  the stack is allowed to protrude StackSmall bytes below the stack
    29  guard.  Functions with large frames don't bother with the check and
    30  always call morestack.  The sequences are (for amd64, others are
    31  similar):
    32  
    33  	guard = g->stackguard
    34  	frame = function's stack frame size
    35  	argsize = size of function arguments (call + return)
    36  
    37  	stack frame size <= StackSmall:
    38  		CMPQ guard, SP
    39  		JHI 3(PC)
    40  		MOVQ m->morearg, $(argsize << 32)
    41  		CALL morestack(SB)
    42  
    43  	stack frame size > StackSmall but < StackBig
    44  		LEAQ (frame-StackSmall)(SP), R0
    45  		CMPQ guard, R0
    46  		JHI 3(PC)
    47  		MOVQ m->morearg, $(argsize << 32)
    48  		CALL morestack(SB)
    49  
    50  	stack frame size >= StackBig:
    51  		MOVQ m->morearg, $((argsize << 32) | frame)
    52  		CALL morestack(SB)
    53  
    54  The bottom StackGuard - StackSmall bytes are important: there has
    55  to be enough room to execute functions that refuse to check for
    56  stack overflow, either because they need to be adjacent to the
    57  actual caller's frame (deferproc) or because they handle the imminent
    58  stack overflow (morestack).
    59  
    60  For example, deferproc might call malloc, which does one of the
    61  above checks (without allocating a full frame), which might trigger
    62  a call to morestack.  This sequence needs to fit in the bottom
    63  section of the stack.  On amd64, morestack's frame is 40 bytes, and
    64  deferproc's frame is 56 bytes.  That fits well within the
    65  StackGuard - StackSmall bytes at the bottom.
    66  The linkers explore all possible call traces involving non-splitting
    67  functions to make sure that this limit cannot be violated.
    68  */
    69  
    70  const (
    71  	// stackSystem is a number of additional bytes to add
    72  	// to each stack below the usual guard area for OS-specific
    73  	// purposes like signal handling. Used on Windows, Plan 9,
    74  	// and iOS because they do not use a separate stack.
    75  	stackSystem = goos.IsWindows*4096 + goos.IsPlan9*512 + goos.IsIos*goarch.IsArm64*1024
    76  
    77  	// The minimum size of stack used by Go code
    78  	stackMin = 2048
    79  
    80  	// The minimum stack size to allocate.
    81  	// The hackery here rounds fixedStack0 up to a power of 2.
    82  	fixedStack0 = stackMin + stackSystem
    83  	fixedStack1 = fixedStack0 - 1
    84  	fixedStack2 = fixedStack1 | (fixedStack1 >> 1)
    85  	fixedStack3 = fixedStack2 | (fixedStack2 >> 2)
    86  	fixedStack4 = fixedStack3 | (fixedStack3 >> 4)
    87  	fixedStack5 = fixedStack4 | (fixedStack4 >> 8)
    88  	fixedStack6 = fixedStack5 | (fixedStack5 >> 16)
    89  	fixedStack  = fixedStack6 + 1
    90  
    91  	// stackNosplit is the maximum number of bytes that a chain of NOSPLIT
    92  	// functions can use.
    93  	// This arithmetic must match that in cmd/internal/objabi/stack.go:StackNosplit.
    94  	stackNosplit = abi.StackNosplitBase * sys.StackGuardMultiplier
    95  
    96  	// The stack guard is a pointer this many bytes above the
    97  	// bottom of the stack.
    98  	//
    99  	// The guard leaves enough room for a stackNosplit chain of NOSPLIT calls
   100  	// plus one stackSmall frame plus stackSystem bytes for the OS.
   101  	// This arithmetic must match that in cmd/internal/objabi/stack.go:StackLimit.
   102  	stackGuard = stackNosplit + stackSystem + abi.StackSmall
   103  )
   104  
   105  const (
   106  	// stackDebug == 0: no logging
   107  	//            == 1: logging of per-stack operations
   108  	//            == 2: logging of per-frame operations
   109  	//            == 3: logging of per-word updates
   110  	//            == 4: logging of per-word reads
   111  	stackDebug       = 0
   112  	stackFromSystem  = 0 // allocate stacks from system memory instead of the heap
   113  	stackFaultOnFree = 0 // old stacks are mapped noaccess to detect use after free
   114  	stackNoCache     = 0 // disable per-P small stack caches
   115  
   116  	// check the BP links during traceback.
   117  	debugCheckBP = false
   118  )
   119  
   120  var (
   121  	stackPoisonCopy = 0 // fill stack that should not be accessed with garbage, to detect bad dereferences during copy
   122  )
   123  
   124  const (
   125  	uintptrMask = 1<<(8*goarch.PtrSize) - 1
   126  
   127  	// The values below can be stored to g.stackguard0 to force
   128  	// the next stack check to fail.
   129  	// These are all larger than any real SP.
   130  
   131  	// Goroutine preemption request.
   132  	// 0xfffffade in hex.
   133  	stackPreempt = uintptrMask & -1314
   134  
   135  	// Thread is forking. Causes a split stack check failure.
   136  	// 0xfffffb2e in hex.
   137  	stackFork = uintptrMask & -1234
   138  
   139  	// Force a stack movement. Used for debugging.
   140  	// 0xfffffeed in hex.
   141  	stackForceMove = uintptrMask & -275
   142  
   143  	// stackPoisonMin is the lowest allowed stack poison value.
   144  	stackPoisonMin = uintptrMask & -4096
   145  )
   146  
   147  // Global pool of spans that have free stacks.
   148  // Stacks are assigned an order according to size.
   149  //
   150  //	order = log_2(size/FixedStack)
   151  //
   152  // There is a free list for each order.
   153  var stackpool [_NumStackOrders]struct {
   154  	item stackpoolItem
   155  	_    [(cpu.CacheLinePadSize - unsafe.Sizeof(stackpoolItem{})%cpu.CacheLinePadSize) % cpu.CacheLinePadSize]byte
   156  }
   157  
   158  type stackpoolItem struct {
   159  	_    sys.NotInHeap
   160  	mu   mutex
   161  	span mSpanList
   162  }
   163  
   164  // Global pool of large stack spans.
   165  var stackLarge struct {
   166  	lock mutex
   167  	free [heapAddrBits - gc.PageShift]mSpanList // free lists by log_2(s.npages)
   168  }
   169  
   170  func stackinit() {
   171  	if _StackCacheSize&pageMask != 0 {
   172  		throw("cache size must be a multiple of page size")
   173  	}
   174  	for i := range stackpool {
   175  		stackpool[i].item.span.init()
   176  		lockInit(&stackpool[i].item.mu, lockRankStackpool)
   177  	}
   178  	for i := range stackLarge.free {
   179  		stackLarge.free[i].init()
   180  		lockInit(&stackLarge.lock, lockRankStackLarge)
   181  	}
   182  }
   183  
   184  // stacklog2 returns ⌊log_2(n)⌋.
   185  func stacklog2(n uintptr) int {
   186  	if n == 0 {
   187  		return 0
   188  	}
   189  	return bits.Len64(uint64(n))
   190  }
   191  
   192  // Allocates a stack from the free pool. Must be called with
   193  // stackpool[order].item.mu held.
   194  func stackpoolalloc(order uint8) gclinkptr {
   195  	list := &stackpool[order].item.span
   196  	s := list.first
   197  	lockWithRankMayAcquire(&mheap_.lock, lockRankMheap)
   198  	if s == nil {
   199  		// no free stacks. Allocate another span worth.
   200  		s = mheap_.allocManual(_StackCacheSize>>gc.PageShift, spanAllocStack)
   201  		if s == nil {
   202  			throw("out of memory")
   203  		}
   204  		if s.allocCount != 0 {
   205  			throw("bad allocCount")
   206  		}
   207  		if s.manualFreeList.ptr() != nil {
   208  			throw("bad manualFreeList")
   209  		}
   210  		osStackAlloc(s)
   211  		s.elemsize = fixedStack << order
   212  		for i := uintptr(0); i < _StackCacheSize; i += s.elemsize {
   213  			x := gclinkptr(s.base() + i)
   214  			if valgrindenabled {
   215  				// The address of x.ptr() becomes the base of stacks. We need to
   216  				// mark it allocated here and in stackfree and stackpoolfree, and free'd in
   217  				// stackalloc in order to avoid overlapping allocations and
   218  				// uninitialized memory errors in valgrind.
   219  				valgrindMalloc(unsafe.Pointer(x.ptr()), unsafe.Sizeof(x.ptr()))
   220  			}
   221  			x.ptr().next = s.manualFreeList
   222  			s.manualFreeList = x
   223  		}
   224  		list.insert(s)
   225  	}
   226  	x := s.manualFreeList
   227  	if x.ptr() == nil {
   228  		throw("span has no free stacks")
   229  	}
   230  	s.manualFreeList = x.ptr().next
   231  	s.allocCount++
   232  	if s.manualFreeList.ptr() == nil {
   233  		// all stacks in s are allocated.
   234  		list.remove(s)
   235  	}
   236  	return x
   237  }
   238  
   239  // Adds stack x to the free pool. Must be called with stackpool[order].item.mu held.
   240  func stackpoolfree(x gclinkptr, order uint8) {
   241  	s := spanOfUnchecked(uintptr(x))
   242  	if s.state.get() != mSpanManual {
   243  		throw("freeing stack not in a stack span")
   244  	}
   245  	if s.manualFreeList.ptr() == nil {
   246  		// s will now have a free stack
   247  		stackpool[order].item.span.insert(s)
   248  	}
   249  	x.ptr().next = s.manualFreeList
   250  	s.manualFreeList = x
   251  	s.allocCount--
   252  	if gcphase == _GCoff && s.allocCount == 0 {
   253  		// Span is completely free. Return it to the heap
   254  		// immediately if we're sweeping.
   255  		//
   256  		// If GC is active, we delay the free until the end of
   257  		// GC to avoid the following type of situation:
   258  		//
   259  		// 1) GC starts, scans a SudoG but does not yet mark the SudoG.elem pointer
   260  		// 2) The stack that pointer points to is copied
   261  		// 3) The old stack is freed
   262  		// 4) The containing span is marked free
   263  		// 5) GC attempts to mark the SudoG.elem pointer. The
   264  		//    marking fails because the pointer looks like a
   265  		//    pointer into a free span.
   266  		//
   267  		// By not freeing, we prevent step #4 until GC is done.
   268  		stackpool[order].item.span.remove(s)
   269  		s.manualFreeList = 0
   270  		osStackFree(s)
   271  		mheap_.freeManual(s, spanAllocStack)
   272  	}
   273  }
   274  
   275  // stackcacherefill/stackcacherelease implement a global pool of stack segments.
   276  // The pool is required to prevent unlimited growth of per-thread caches.
   277  //
   278  //go:systemstack
   279  func stackcacherefill(c *mcache, order uint8) {
   280  	if stackDebug >= 1 {
   281  		print("stackcacherefill order=", order, "\n")
   282  	}
   283  
   284  	// Grab some stacks from the global cache.
   285  	// Grab half of the allowed capacity (to prevent thrashing).
   286  	var list gclinkptr
   287  	var size uintptr
   288  	lock(&stackpool[order].item.mu)
   289  	for size < _StackCacheSize/2 {
   290  		x := stackpoolalloc(order)
   291  		x.ptr().next = list
   292  		list = x
   293  		size += fixedStack << order
   294  	}
   295  	unlock(&stackpool[order].item.mu)
   296  	c.stackcache[order].list = list
   297  	c.stackcache[order].size = size
   298  }
   299  
   300  //go:systemstack
   301  func stackcacherelease(c *mcache, order uint8) {
   302  	if stackDebug >= 1 {
   303  		print("stackcacherelease order=", order, "\n")
   304  	}
   305  	x := c.stackcache[order].list
   306  	size := c.stackcache[order].size
   307  	lock(&stackpool[order].item.mu)
   308  	for size > _StackCacheSize/2 {
   309  		y := x.ptr().next
   310  		stackpoolfree(x, order)
   311  		x = y
   312  		size -= fixedStack << order
   313  	}
   314  	unlock(&stackpool[order].item.mu)
   315  	c.stackcache[order].list = x
   316  	c.stackcache[order].size = size
   317  }
   318  
   319  //go:systemstack
   320  func stackcache_clear(c *mcache) {
   321  	if stackDebug >= 1 {
   322  		print("stackcache clear\n")
   323  	}
   324  	for order := uint8(0); order < _NumStackOrders; order++ {
   325  		lock(&stackpool[order].item.mu)
   326  		x := c.stackcache[order].list
   327  		for x.ptr() != nil {
   328  			y := x.ptr().next
   329  			stackpoolfree(x, order)
   330  			x = y
   331  		}
   332  		c.stackcache[order].list = 0
   333  		c.stackcache[order].size = 0
   334  		unlock(&stackpool[order].item.mu)
   335  	}
   336  }
   337  
   338  // stackalloc allocates an n byte stack.
   339  //
   340  // stackalloc must run on the system stack because it uses per-P
   341  // resources and must not split the stack.
   342  //
   343  //go:systemstack
   344  func stackalloc(n uint32) stack {
   345  	// Stackalloc must be called on scheduler stack, so that we
   346  	// never try to grow the stack during the code that stackalloc runs.
   347  	// Doing so would cause a deadlock (issue 1547).
   348  	thisg := getg()
   349  	if thisg != thisg.m.g0 {
   350  		throw("stackalloc not on scheduler stack")
   351  	}
   352  	if n&(n-1) != 0 {
   353  		throw("stack size not a power of 2")
   354  	}
   355  	if stackDebug >= 1 {
   356  		print("stackalloc ", n, "\n")
   357  	}
   358  
   359  	if debug.efence != 0 || stackFromSystem != 0 {
   360  		n = uint32(alignUp(uintptr(n), physPageSize))
   361  		v := sysAlloc(uintptr(n), &memstats.stacks_sys, "goroutine stack (system)")
   362  		if v == nil {
   363  			throw("out of memory (stackalloc)")
   364  		}
   365  		return stack{uintptr(v), uintptr(v) + uintptr(n)}
   366  	}
   367  
   368  	// Small stacks are allocated with a fixed-size free-list allocator.
   369  	// If we need a stack of a bigger size, we fall back on allocating
   370  	// a dedicated span.
   371  	var v unsafe.Pointer
   372  	if n < fixedStack<<_NumStackOrders && n < _StackCacheSize {
   373  		order := uint8(0)
   374  		n2 := n
   375  		for n2 > fixedStack {
   376  			order++
   377  			n2 >>= 1
   378  		}
   379  		var x gclinkptr
   380  		if stackNoCache != 0 || thisg.m.p == 0 || thisg.m.preemptoff != "" {
   381  			// thisg.m.p == 0 can happen in the guts of exitsyscall
   382  			// or procresize. Just get a stack from the global pool.
   383  			// Also don't touch stackcache during gc
   384  			// as it's flushed concurrently.
   385  			lock(&stackpool[order].item.mu)
   386  			x = stackpoolalloc(order)
   387  			unlock(&stackpool[order].item.mu)
   388  		} else {
   389  			c := thisg.m.p.ptr().mcache
   390  			x = c.stackcache[order].list
   391  			if x.ptr() == nil {
   392  				stackcacherefill(c, order)
   393  				x = c.stackcache[order].list
   394  			}
   395  			c.stackcache[order].list = x.ptr().next
   396  			c.stackcache[order].size -= uintptr(n)
   397  		}
   398  		if valgrindenabled {
   399  			// We're about to allocate the stack region starting at x.ptr().
   400  			// To prevent valgrind from complaining about overlapping allocations,
   401  			// we need to mark the (previously allocated) memory as free'd.
   402  			valgrindFree(unsafe.Pointer(x.ptr()))
   403  		}
   404  		v = unsafe.Pointer(x)
   405  	} else {
   406  		var s *mspan
   407  		npage := uintptr(n) >> gc.PageShift
   408  		log2npage := stacklog2(npage)
   409  
   410  		// Try to get a stack from the large stack cache.
   411  		lock(&stackLarge.lock)
   412  		if !stackLarge.free[log2npage].isEmpty() {
   413  			s = stackLarge.free[log2npage].first
   414  			stackLarge.free[log2npage].remove(s)
   415  		}
   416  		unlock(&stackLarge.lock)
   417  
   418  		lockWithRankMayAcquire(&mheap_.lock, lockRankMheap)
   419  
   420  		if s == nil {
   421  			// Allocate a new stack from the heap.
   422  			s = mheap_.allocManual(npage, spanAllocStack)
   423  			if s == nil {
   424  				throw("out of memory")
   425  			}
   426  			osStackAlloc(s)
   427  			s.elemsize = uintptr(n)
   428  		}
   429  		v = unsafe.Pointer(s.base())
   430  	}
   431  
   432  	if traceAllocFreeEnabled() {
   433  		trace := traceAcquire()
   434  		if trace.ok() {
   435  			trace.GoroutineStackAlloc(uintptr(v), uintptr(n))
   436  			traceRelease(trace)
   437  		}
   438  	}
   439  	if raceenabled {
   440  		racemalloc(v, uintptr(n))
   441  	}
   442  	if msanenabled {
   443  		msanmalloc(v, uintptr(n))
   444  	}
   445  	if asanenabled {
   446  		asanunpoison(v, uintptr(n))
   447  	}
   448  	if valgrindenabled {
   449  		valgrindMalloc(v, uintptr(n))
   450  	}
   451  	if stackDebug >= 1 {
   452  		print("  allocated ", v, "\n")
   453  	}
   454  	return stack{uintptr(v), uintptr(v) + uintptr(n)}
   455  }
   456  
   457  // stackfree frees an n byte stack allocation at stk.
   458  //
   459  // stackfree must run on the system stack because it uses per-P
   460  // resources and must not split the stack.
   461  //
   462  //go:systemstack
   463  func stackfree(stk stack) {
   464  	gp := getg()
   465  	v := unsafe.Pointer(stk.lo)
   466  	n := stk.hi - stk.lo
   467  	if n&(n-1) != 0 {
   468  		throw("stack not a power of 2")
   469  	}
   470  	if stk.lo+n < stk.hi {
   471  		throw("bad stack size")
   472  	}
   473  	if stackDebug >= 1 {
   474  		println("stackfree", v, n)
   475  		memclrNoHeapPointers(v, n) // for testing, clobber stack data
   476  	}
   477  	if debug.efence != 0 || stackFromSystem != 0 {
   478  		if debug.efence != 0 || stackFaultOnFree != 0 {
   479  			sysFault(v, n)
   480  		} else {
   481  			sysFree(v, n, &memstats.stacks_sys)
   482  		}
   483  		return
   484  	}
   485  	if traceAllocFreeEnabled() {
   486  		trace := traceAcquire()
   487  		if trace.ok() {
   488  			trace.GoroutineStackFree(uintptr(v))
   489  			traceRelease(trace)
   490  		}
   491  	}
   492  	if msanenabled {
   493  		msanfree(v, n)
   494  	}
   495  	if asanenabled {
   496  		asanpoison(v, n)
   497  	}
   498  	if valgrindenabled {
   499  		valgrindFree(v)
   500  	}
   501  	if n < fixedStack<<_NumStackOrders && n < _StackCacheSize {
   502  		order := uint8(0)
   503  		n2 := n
   504  		for n2 > fixedStack {
   505  			order++
   506  			n2 >>= 1
   507  		}
   508  		x := gclinkptr(v)
   509  		if stackNoCache != 0 || gp.m.p == 0 || gp.m.preemptoff != "" {
   510  			lock(&stackpool[order].item.mu)
   511  			if valgrindenabled {
   512  				// x.ptr() is the head of the list of free stacks, and will be used
   513  				// when allocating a new stack, so it has to be marked allocated.
   514  				valgrindMalloc(unsafe.Pointer(x.ptr()), unsafe.Sizeof(x.ptr()))
   515  			}
   516  			stackpoolfree(x, order)
   517  			unlock(&stackpool[order].item.mu)
   518  		} else {
   519  			c := gp.m.p.ptr().mcache
   520  			if c.stackcache[order].size >= _StackCacheSize {
   521  				stackcacherelease(c, order)
   522  			}
   523  			if valgrindenabled {
   524  				// x.ptr() is the head of the list of free stacks, and will
   525  				// be used when allocating a new stack, so it has to be
   526  				// marked allocated.
   527  				valgrindMalloc(unsafe.Pointer(x.ptr()), unsafe.Sizeof(x.ptr()))
   528  			}
   529  			x.ptr().next = c.stackcache[order].list
   530  			c.stackcache[order].list = x
   531  			c.stackcache[order].size += n
   532  		}
   533  	} else {
   534  		s := spanOfUnchecked(uintptr(v))
   535  		if s.state.get() != mSpanManual {
   536  			println(hex(s.base()), v)
   537  			throw("bad span state")
   538  		}
   539  		if gcphase == _GCoff {
   540  			// Free the stack immediately if we're
   541  			// sweeping.
   542  			osStackFree(s)
   543  			mheap_.freeManual(s, spanAllocStack)
   544  		} else {
   545  			// If the GC is running, we can't return a
   546  			// stack span to the heap because it could be
   547  			// reused as a heap span, and this state
   548  			// change would race with GC. Add it to the
   549  			// large stack cache instead.
   550  			log2npage := stacklog2(s.npages)
   551  			lock(&stackLarge.lock)
   552  			stackLarge.free[log2npage].insert(s)
   553  			unlock(&stackLarge.lock)
   554  		}
   555  	}
   556  }
   557  
   558  var maxstacksize uintptr = 1 << 20 // enough until runtime.main sets it for real
   559  
   560  var maxstackceiling = maxstacksize
   561  
   562  var ptrnames = []string{
   563  	0: "scalar",
   564  	1: "ptr",
   565  }
   566  
   567  // Stack frame layout
   568  //
   569  // (x86)
   570  // +------------------+
   571  // | args from caller |
   572  // +------------------+ <- frame->argp
   573  // |  return address  |
   574  // +------------------+
   575  // |  caller's BP (*) | (*) if framepointer_enabled && varp > sp
   576  // +------------------+ <- frame->varp
   577  // |     locals       |
   578  // +------------------+
   579  // |  args to callee  |
   580  // +------------------+ <- frame->sp
   581  //
   582  // (arm)
   583  // +------------------+
   584  // | args from caller |
   585  // +------------------+ <- frame->argp
   586  // | caller's retaddr |
   587  // +------------------+
   588  // |  caller's FP (*) | (*) on ARM64, if framepointer_enabled && varp > sp
   589  // +------------------+ <- frame->varp
   590  // |     locals       |
   591  // +------------------+
   592  // |  args to callee  |
   593  // +------------------+
   594  // |  return address  |
   595  // +------------------+ <- frame->sp
   596  //
   597  // varp > sp means that the function has a frame;
   598  // varp == sp means frameless function.
   599  
   600  type adjustinfo struct {
   601  	old   stack
   602  	delta uintptr // ptr distance from old to new stack (newbase - oldbase)
   603  
   604  	// sghi is the highest sudog.elem on the stack.
   605  	sghi uintptr
   606  }
   607  
   608  // adjustpointer checks whether *vpp is in the old stack described by adjinfo.
   609  // If so, it rewrites *vpp to point into the new stack.
   610  func adjustpointer(adjinfo *adjustinfo, vpp unsafe.Pointer) {
   611  	pp := (*uintptr)(vpp)
   612  	p := *pp
   613  	if stackDebug >= 4 {
   614  		print("        ", pp, ":", hex(p), "\n")
   615  	}
   616  	if valgrindenabled {
   617  		// p is a pointer on a stack, it is inherently initialized, as
   618  		// everything on the stack is, but valgrind for _some unknown reason_
   619  		// sometimes thinks it's uninitialized, and flags operations on p below
   620  		// as uninitialized. We just initialize it if valgrind thinks its
   621  		// uninitialized.
   622  		//
   623  		// See go.dev/issues/73801.
   624  		valgrindMakeMemDefined(unsafe.Pointer(&p), unsafe.Sizeof(&p))
   625  	}
   626  	if adjinfo.old.lo <= p && p < adjinfo.old.hi {
   627  		*pp = p + adjinfo.delta
   628  		if stackDebug >= 3 {
   629  			print("        adjust ptr ", pp, ":", hex(p), " -> ", hex(*pp), "\n")
   630  		}
   631  	}
   632  }
   633  
   634  // Information from the compiler about the layout of stack frames.
   635  // Note: this type must agree with reflect.bitVector.
   636  type bitvector struct {
   637  	n        int32 // # of bits
   638  	bytedata *uint8
   639  }
   640  
   641  // ptrbit returns the i'th bit in bv.
   642  // ptrbit is less efficient than iterating directly over bitvector bits,
   643  // and should only be used in non-performance-critical code.
   644  // See adjustpointers for an example of a high-efficiency walk of a bitvector.
   645  func (bv *bitvector) ptrbit(i uintptr) uint8 {
   646  	b := *(addb(bv.bytedata, i/8))
   647  	return (b >> (i % 8)) & 1
   648  }
   649  
   650  // bv describes the memory starting at address scanp.
   651  // Adjust any pointers contained therein.
   652  func adjustpointers(scanp unsafe.Pointer, bv *bitvector, adjinfo *adjustinfo, f funcInfo) {
   653  	minp := adjinfo.old.lo
   654  	maxp := adjinfo.old.hi
   655  	delta := adjinfo.delta
   656  	num := uintptr(bv.n)
   657  	// If this frame might contain channel receive slots, use CAS
   658  	// to adjust pointers. If the slot hasn't been received into
   659  	// yet, it may contain stack pointers and a concurrent send
   660  	// could race with adjusting those pointers. (The sent value
   661  	// itself can never contain stack pointers.)
   662  	useCAS := uintptr(scanp) < adjinfo.sghi
   663  	for i := uintptr(0); i < num; i += 8 {
   664  		if stackDebug >= 4 {
   665  			for j := uintptr(0); j < 8; j++ {
   666  				print("        ", add(scanp, (i+j)*goarch.PtrSize), ":", ptrnames[bv.ptrbit(i+j)], ":", hex(*(*uintptr)(add(scanp, (i+j)*goarch.PtrSize))), " # ", i, " ", *addb(bv.bytedata, i/8), "\n")
   667  			}
   668  		}
   669  		b := *(addb(bv.bytedata, i/8))
   670  		for b != 0 {
   671  			j := uintptr(sys.TrailingZeros8(b))
   672  			b &= b - 1
   673  			pp := (*uintptr)(add(scanp, (i+j)*goarch.PtrSize))
   674  		retry:
   675  			p := *pp
   676  			if f.valid() && 0 < p && p < minLegalPointer && debug.invalidptr != 0 {
   677  				// Looks like a junk value in a pointer slot.
   678  				// Live analysis wrong?
   679  				getg().m.traceback = 2
   680  				print("runtime: bad pointer in frame ", funcname(f), " at ", pp, ": ", hex(p), "\n")
   681  				throw("invalid pointer found on stack")
   682  			}
   683  			if minp <= p && p < maxp {
   684  				if stackDebug >= 3 {
   685  					print("adjust ptr ", hex(p), " ", funcname(f), "\n")
   686  				}
   687  				if useCAS {
   688  					ppu := (*unsafe.Pointer)(unsafe.Pointer(pp))
   689  					if !atomic.Casp1(ppu, unsafe.Pointer(p), unsafe.Pointer(p+delta)) {
   690  						goto retry
   691  					}
   692  				} else {
   693  					*pp = p + delta
   694  				}
   695  			}
   696  		}
   697  	}
   698  }
   699  
   700  // Note: the argument/return area is adjusted by the callee.
   701  func adjustframe(frame *stkframe, adjinfo *adjustinfo) {
   702  	// Adjust saved frame pointer if there is one.
   703  	if (goarch.ArchFamily == goarch.AMD64 || goarch.ArchFamily == goarch.ARM64) && frame.argp-frame.varp == 2*goarch.PtrSize {
   704  		if stackDebug >= 3 {
   705  			print("      saved bp\n")
   706  		}
   707  		if debugCheckBP {
   708  			// Frame pointers should always point to the next higher frame on
   709  			// the Go stack (or be nil, for the top frame on the stack).
   710  			bp := *(*uintptr)(unsafe.Pointer(frame.varp))
   711  			if bp != 0 && (bp < adjinfo.old.lo || bp >= adjinfo.old.hi) {
   712  				println("runtime: found invalid frame pointer")
   713  				print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n")
   714  				throw("bad frame pointer")
   715  			}
   716  		}
   717  		// On AMD64, this is the caller's frame pointer saved in the current
   718  		// frame.
   719  		// On ARM64, this is the frame pointer of the caller's caller saved
   720  		// by the caller in its frame (one word below its SP).
   721  		adjustpointer(adjinfo, unsafe.Pointer(frame.varp))
   722  	}
   723  	if goarch.ArchFamily == goarch.ARM64 && isInjectedCall(frame.fn.funcID) {
   724  		// If this is an injected call on arm64, then we need to adjust
   725  		// the frame pointer saved by the original function into which
   726  		// the call was injected. Normally this would be handled when
   727  		// adjusting the callee's frame or in adjustctxt. But when a
   728  		// call is injected, the frame is placed 16 bytes below the
   729  		// original stack pointer to make room to save the link
   730  		// register, and the frame pointer saved by the original
   731  		// function isn't inside any call frame. We can adjust that
   732  		// saved frame pointer here by looking just above frame.fp.
   733  		//
   734  		// ^  original call    ^
   735  		// |  frame above...   |
   736  		// +-------------------+ <- stack pointer at the time of injection
   737  		// :  FP saved by      :
   738  		// :  original func    :
   739  		// :···················: <- frame pointer register from original function
   740  		// :  LR saved during  :
   741  		// :  injection        :
   742  		// +-------------------+ <- frame.fp (injection decrements SP by 16 bytes)
   743  		// |  FP saved         |
   744  		// |  during injection |
   745  		// +-------------------+
   746  		// |  injected call    |
   747  		// V  frame below...   V
   748  		adjustpointer(adjinfo, unsafe.Pointer(frame.fp+goarch.PtrSize))
   749  	}
   750  
   751  	if frame.continpc == 0 {
   752  		// Frame is dead. The program might still see the frame pointer
   753  		// saved in the frame, adjusted above, but we don't need to
   754  		// adjust the rest of the frame.
   755  		return
   756  	}
   757  	f := frame.fn
   758  	if stackDebug >= 2 {
   759  		print("    adjusting ", funcname(f), " frame=[", hex(frame.sp), ",", hex(frame.fp), "] pc=", hex(frame.pc), " continpc=", hex(frame.continpc), "\n")
   760  	}
   761  
   762  	locals, args, objs := frame.getStackMap(true)
   763  
   764  	// Adjust local variables if stack frame has been allocated.
   765  	if locals.n > 0 {
   766  		size := uintptr(locals.n) * goarch.PtrSize
   767  		adjustpointers(unsafe.Pointer(frame.varp-size), &locals, adjinfo, f)
   768  	}
   769  
   770  	// Adjust arguments.
   771  	if args.n > 0 {
   772  		if stackDebug >= 3 {
   773  			print("      args\n")
   774  		}
   775  		adjustpointers(unsafe.Pointer(frame.argp), &args, adjinfo, funcInfo{})
   776  	}
   777  
   778  	// Adjust pointers in all stack objects (whether they are live or not).
   779  	// See comments in mgcmark.go:scanframeworker.
   780  	if frame.varp != 0 {
   781  		for i := range objs {
   782  			obj := &objs[i]
   783  			off := obj.off
   784  			base := frame.varp // locals base pointer
   785  			if off >= 0 {
   786  				base = frame.argp // arguments and return values base pointer
   787  			}
   788  			p := base + uintptr(off)
   789  			if p < frame.sp {
   790  				// Object hasn't been allocated in the frame yet.
   791  				// (Happens when the stack bounds check fails and
   792  				// we call into morestack.)
   793  				continue
   794  			}
   795  			ptrBytes, gcData := obj.gcdata()
   796  			for i := uintptr(0); i < ptrBytes; i += goarch.PtrSize {
   797  				if *addb(gcData, i/(8*goarch.PtrSize))>>(i/goarch.PtrSize&7)&1 != 0 {
   798  					adjustpointer(adjinfo, unsafe.Pointer(p+i))
   799  				}
   800  			}
   801  		}
   802  	}
   803  }
   804  
   805  func adjustctxt(gp *g, adjinfo *adjustinfo) {
   806  	adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.ctxt))
   807  	if !framepointer_enabled {
   808  		return
   809  	}
   810  	if debugCheckBP {
   811  		bp := gp.sched.bp
   812  		if bp != 0 && (bp < adjinfo.old.lo || bp >= adjinfo.old.hi) {
   813  			println("runtime: found invalid top frame pointer")
   814  			print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n")
   815  			throw("bad top frame pointer")
   816  		}
   817  	}
   818  	oldfp := gp.sched.bp
   819  	adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.bp))
   820  	if GOARCH == "arm64" {
   821  		// On ARM64, the frame pointer is saved one word *below* the SP,
   822  		// which is not copied or adjusted in any frame. Do it explicitly
   823  		// here.
   824  		if oldfp == gp.sched.sp-goarch.PtrSize {
   825  			memmove(unsafe.Pointer(gp.sched.bp), unsafe.Pointer(oldfp), goarch.PtrSize)
   826  			adjustpointer(adjinfo, unsafe.Pointer(gp.sched.bp))
   827  		}
   828  	}
   829  }
   830  
   831  func adjustdefers(gp *g, adjinfo *adjustinfo) {
   832  	// Adjust pointers in the Defer structs.
   833  	// We need to do this first because we need to adjust the
   834  	// defer.link fields so we always work on the new stack.
   835  	adjustpointer(adjinfo, unsafe.Pointer(&gp._defer))
   836  	for d := gp._defer; d != nil; d = d.link {
   837  		adjustpointer(adjinfo, unsafe.Pointer(&d.fn))
   838  		adjustpointer(adjinfo, unsafe.Pointer(&d.sp))
   839  		adjustpointer(adjinfo, unsafe.Pointer(&d.link))
   840  	}
   841  }
   842  
   843  func adjustpanics(gp *g, adjinfo *adjustinfo) {
   844  	// Panics are on stack and already adjusted.
   845  	// Update pointer to head of list in G.
   846  	adjustpointer(adjinfo, unsafe.Pointer(&gp._panic))
   847  }
   848  
   849  func adjustsudogs(gp *g, adjinfo *adjustinfo) {
   850  	// the data elements pointed to by a SudoG structure
   851  	// might be in the stack.
   852  	for s := gp.waiting; s != nil; s = s.waitlink {
   853  		adjustpointer(adjinfo, unsafe.Pointer(&s.elem.vu))
   854  		adjustpointer(adjinfo, unsafe.Pointer(&s.elem.vp))
   855  	}
   856  }
   857  
   858  func fillstack(stk stack, b byte) {
   859  	for p := stk.lo; p < stk.hi; p++ {
   860  		*(*byte)(unsafe.Pointer(p)) = b
   861  	}
   862  }
   863  
   864  func findsghi(gp *g, stk stack) uintptr {
   865  	var sghi uintptr
   866  	for sg := gp.waiting; sg != nil; sg = sg.waitlink {
   867  		p := sg.elem.uintptr() + uintptr(sg.c.get().elemsize)
   868  		if stk.lo <= p && p < stk.hi && p > sghi {
   869  			sghi = p
   870  		}
   871  	}
   872  	return sghi
   873  }
   874  
   875  // syncadjustsudogs adjusts gp's sudogs and copies the part of gp's
   876  // stack they refer to while synchronizing with concurrent channel
   877  // operations. It returns the number of bytes of stack copied.
   878  func syncadjustsudogs(gp *g, used uintptr, adjinfo *adjustinfo) uintptr {
   879  	if gp.waiting == nil {
   880  		return 0
   881  	}
   882  
   883  	// Lock channels to prevent concurrent send/receive.
   884  	var lastc *hchan
   885  	for sg := gp.waiting; sg != nil; sg = sg.waitlink {
   886  		if sg.c.get() != lastc {
   887  			// There is a ranking cycle here between gscan bit and
   888  			// hchan locks. Normally, we only allow acquiring hchan
   889  			// locks and then getting a gscan bit. In this case, we
   890  			// already have the gscan bit. We allow acquiring hchan
   891  			// locks here as a special case, since a deadlock can't
   892  			// happen because the G involved must already be
   893  			// suspended. So, we get a special hchan lock rank here
   894  			// that is lower than gscan, but doesn't allow acquiring
   895  			// any other locks other than hchan.
   896  			lockWithRank(&sg.c.get().lock, lockRankHchanLeaf)
   897  		}
   898  		lastc = sg.c.get()
   899  	}
   900  
   901  	// Adjust sudogs.
   902  	adjustsudogs(gp, adjinfo)
   903  
   904  	// Copy the part of the stack the sudogs point in to
   905  	// while holding the lock to prevent races on
   906  	// send/receive slots.
   907  	var sgsize uintptr
   908  	if adjinfo.sghi != 0 {
   909  		oldBot := adjinfo.old.hi - used
   910  		newBot := oldBot + adjinfo.delta
   911  		sgsize = adjinfo.sghi - oldBot
   912  		memmove(unsafe.Pointer(newBot), unsafe.Pointer(oldBot), sgsize)
   913  	}
   914  
   915  	// Unlock channels.
   916  	lastc = nil
   917  	for sg := gp.waiting; sg != nil; sg = sg.waitlink {
   918  		if sg.c.get() != lastc {
   919  			unlock(&sg.c.get().lock)
   920  		}
   921  		lastc = sg.c.get()
   922  	}
   923  
   924  	return sgsize
   925  }
   926  
   927  // Copies gp's stack to a new stack of a different size.
   928  // Caller must have changed gp status to Gcopystack.
   929  func copystack(gp *g, newsize uintptr) {
   930  	if gp.syscallsp != 0 {
   931  		throw("stack growth not allowed in system call")
   932  	}
   933  	old := gp.stack
   934  	if old.lo == 0 {
   935  		throw("nil stackbase")
   936  	}
   937  	used := old.hi - gp.sched.sp
   938  	// Add just the difference to gcController.addScannableStack.
   939  	// g0 stacks never move, so this will never account for them.
   940  	// It's also fine if we have no P, addScannableStack can deal with
   941  	// that case.
   942  	gcController.addScannableStack(getg().m.p.ptr(), int64(newsize)-int64(old.hi-old.lo))
   943  
   944  	// allocate new stack
   945  	new := stackalloc(uint32(newsize))
   946  	if stackPoisonCopy != 0 {
   947  		fillstack(new, 0xfd)
   948  	}
   949  	if stackDebug >= 1 {
   950  		print("copystack gp=", gp, " [", hex(old.lo), " ", hex(old.hi-used), " ", hex(old.hi), "]", " -> [", hex(new.lo), " ", hex(new.hi-used), " ", hex(new.hi), "]/", newsize, "\n")
   951  	}
   952  
   953  	// Compute adjustment.
   954  	var adjinfo adjustinfo
   955  	adjinfo.old = old
   956  	adjinfo.delta = new.hi - old.hi
   957  
   958  	// Adjust sudogs, synchronizing with channel ops if necessary.
   959  	ncopy := used
   960  	if !gp.activeStackChans {
   961  		if newsize < old.hi-old.lo && gp.parkingOnChan.Load() {
   962  			// It's not safe for someone to shrink this stack while we're actively
   963  			// parking on a channel, but it is safe to grow since we do that
   964  			// ourselves and explicitly don't want to synchronize with channels
   965  			// since we could self-deadlock.
   966  			throw("racy sudog adjustment due to parking on channel")
   967  		}
   968  		adjustsudogs(gp, &adjinfo)
   969  	} else {
   970  		// sudogs may be pointing in to the stack and gp has
   971  		// released channel locks, so other goroutines could
   972  		// be writing to gp's stack. Find the highest such
   973  		// pointer so we can handle everything there and below
   974  		// carefully. (This shouldn't be far from the bottom
   975  		// of the stack, so there's little cost in handling
   976  		// everything below it carefully.)
   977  		adjinfo.sghi = findsghi(gp, old)
   978  
   979  		// Synchronize with channel ops and copy the part of
   980  		// the stack they may interact with.
   981  		ncopy -= syncadjustsudogs(gp, used, &adjinfo)
   982  	}
   983  
   984  	// Copy the stack (or the rest of it) to the new location
   985  	memmove(unsafe.Pointer(new.hi-ncopy), unsafe.Pointer(old.hi-ncopy), ncopy)
   986  
   987  	// Adjust remaining structures that have pointers into stacks.
   988  	// We have to do most of these before we traceback the new
   989  	// stack because gentraceback uses them.
   990  	adjustctxt(gp, &adjinfo)
   991  	adjustdefers(gp, &adjinfo)
   992  	adjustpanics(gp, &adjinfo)
   993  	if adjinfo.sghi != 0 {
   994  		adjinfo.sghi += adjinfo.delta
   995  	}
   996  
   997  	// Swap out old stack for new one
   998  	gp.stack = new
   999  	gp.stackguard0 = new.lo + stackGuard // NOTE: might clobber a preempt request
  1000  	gp.sched.sp = new.hi - used
  1001  	gp.stktopsp += adjinfo.delta
  1002  
  1003  	// Adjust pointers in the new stack.
  1004  	var u unwinder
  1005  	for u.init(gp, 0); u.valid(); u.next() {
  1006  		adjustframe(&u.frame, &adjinfo)
  1007  	}
  1008  
  1009  	if valgrindenabled {
  1010  		if gp.valgrindStackID == 0 {
  1011  			gp.valgrindStackID = valgrindRegisterStack(unsafe.Pointer(new.lo), unsafe.Pointer(new.hi))
  1012  		} else {
  1013  			valgrindChangeStack(gp.valgrindStackID, unsafe.Pointer(new.lo), unsafe.Pointer(new.hi))
  1014  		}
  1015  	}
  1016  
  1017  	// free old stack
  1018  	if goexperiment.RuntimeSecret && gp.secret > 0 {
  1019  		// Some portion of the old stack has secret stuff on it.
  1020  		// We don't really know where we entered secret mode,
  1021  		// so just clear the whole thing.
  1022  		// TODO(dmo): traceback until we hit secret.Do? clearing
  1023  		// is fast and optimized, might not be worth it.
  1024  		memclrNoHeapPointers(unsafe.Pointer(old.lo), old.hi-old.lo)
  1025  		// The memmove call above might put secrets from the stack into registers.
  1026  		secretEraseRegisters()
  1027  	}
  1028  	if stackPoisonCopy != 0 {
  1029  		fillstack(old, 0xfc)
  1030  	}
  1031  	stackfree(old)
  1032  }
  1033  
  1034  // round x up to a power of 2.
  1035  func round2(x int32) int32 {
  1036  	s := uint(0)
  1037  	for 1<<s < x {
  1038  		s++
  1039  	}
  1040  	return 1 << s
  1041  }
  1042  
  1043  // Called from runtime·morestack when more stack is needed.
  1044  // Allocate larger stack and relocate to new stack.
  1045  // Stack growth is multiplicative, for constant amortized cost.
  1046  //
  1047  // g->atomicstatus will be Grunning or Gscanrunning upon entry.
  1048  // If the scheduler is trying to stop this g, then it will set preemptStop.
  1049  //
  1050  // This must be nowritebarrierrec because it can be called as part of
  1051  // stack growth from other nowritebarrierrec functions, but the
  1052  // compiler doesn't check this.
  1053  //
  1054  //go:nowritebarrierrec
  1055  func newstack() {
  1056  	thisg := getg()
  1057  	// TODO: double check all gp. shouldn't be getg().
  1058  	if thisg.m.morebuf.g.ptr().stackguard0 == stackFork {
  1059  		throw("stack growth after fork")
  1060  	}
  1061  	if thisg.m.morebuf.g.ptr() != thisg.m.curg {
  1062  		print("runtime: newstack called from g=", hex(thisg.m.morebuf.g), "\n"+"\tm=", thisg.m, " m->curg=", thisg.m.curg, " m->g0=", thisg.m.g0, " m->gsignal=", thisg.m.gsignal, "\n")
  1063  		morebuf := thisg.m.morebuf
  1064  		traceback(morebuf.pc, morebuf.sp, morebuf.lr, morebuf.g.ptr())
  1065  		throw("runtime: wrong goroutine in newstack")
  1066  	}
  1067  
  1068  	gp := thisg.m.curg
  1069  	if goexperiment.RuntimeSecret && gp.secret > 0 {
  1070  		// If we're entering here from a secret context, clear
  1071  		// all the registers. This is important because we
  1072  		// might context switch to a different goroutine which
  1073  		// is not in secret mode, and it will not be careful
  1074  		// about clearing its registers.
  1075  		secretEraseRegisters()
  1076  	}
  1077  
  1078  	if thisg.m.curg.throwsplit {
  1079  		// Update syscallsp, syscallpc in case traceback uses them.
  1080  		morebuf := thisg.m.morebuf
  1081  		gp.syscallsp = morebuf.sp
  1082  		gp.syscallpc = morebuf.pc
  1083  		pcname, pcoff := "(unknown)", uintptr(0)
  1084  		f := findfunc(gp.sched.pc)
  1085  		if f.valid() {
  1086  			pcname = funcname(f)
  1087  			pcoff = gp.sched.pc - f.entry()
  1088  		}
  1089  		print("runtime: newstack at ", pcname, "+", hex(pcoff),
  1090  			" sp=", hex(gp.sched.sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n",
  1091  			"\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n",
  1092  			"\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n")
  1093  
  1094  		thisg.m.traceback = 2 // Include runtime frames
  1095  		traceback(morebuf.pc, morebuf.sp, morebuf.lr, gp)
  1096  		throw("runtime: stack split at bad time")
  1097  	}
  1098  
  1099  	morebuf := thisg.m.morebuf
  1100  	thisg.m.morebuf.pc = 0
  1101  	thisg.m.morebuf.lr = 0
  1102  	thisg.m.morebuf.sp = 0
  1103  	thisg.m.morebuf.g = 0
  1104  
  1105  	// NOTE: stackguard0 may change underfoot, if another thread
  1106  	// is about to try to preempt gp. Read it just once and use that same
  1107  	// value now and below.
  1108  	stackguard0 := atomic.Loaduintptr(&gp.stackguard0)
  1109  
  1110  	// Be conservative about where we preempt.
  1111  	// We are interested in preempting user Go code, not runtime code.
  1112  	// If we're holding locks, mallocing, or preemption is disabled, don't
  1113  	// preempt.
  1114  	// This check is very early in newstack so that even the status change
  1115  	// from Grunning to Gwaiting and back doesn't happen in this case.
  1116  	// That status change by itself can be viewed as a small preemption,
  1117  	// because the GC might change Gwaiting to Gscanwaiting, and then
  1118  	// this goroutine has to wait for the GC to finish before continuing.
  1119  	// If the GC is in some way dependent on this goroutine (for example,
  1120  	// it needs a lock held by the goroutine), that small preemption turns
  1121  	// into a real deadlock.
  1122  	preempt := stackguard0 == stackPreempt
  1123  	if preempt {
  1124  		if !canPreemptM(thisg.m) {
  1125  			// Let the goroutine keep running for now.
  1126  			// gp->preempt is set, so it will be preempted next time.
  1127  			gp.stackguard0 = gp.stack.lo + stackGuard
  1128  			gogo(&gp.sched) // never return
  1129  		}
  1130  	}
  1131  
  1132  	if gp.stack.lo == 0 {
  1133  		throw("missing stack in newstack")
  1134  	}
  1135  	sp := gp.sched.sp
  1136  	if goarch.ArchFamily == goarch.AMD64 || goarch.ArchFamily == goarch.I386 || goarch.ArchFamily == goarch.WASM {
  1137  		// The call to morestack cost a word.
  1138  		sp -= goarch.PtrSize
  1139  	}
  1140  	if stackDebug >= 1 || sp < gp.stack.lo {
  1141  		print("runtime: newstack sp=", hex(sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n",
  1142  			"\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n",
  1143  			"\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n")
  1144  	}
  1145  	if sp < gp.stack.lo {
  1146  		print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->status=", hex(readgstatus(gp)), "\n ")
  1147  		print("runtime: split stack overflow: ", hex(sp), " < ", hex(gp.stack.lo), "\n")
  1148  		throw("runtime: split stack overflow")
  1149  	}
  1150  
  1151  	if preempt {
  1152  		if gp == thisg.m.g0 {
  1153  			throw("runtime: preempt g0")
  1154  		}
  1155  		if thisg.m.p == 0 && thisg.m.locks == 0 {
  1156  			throw("runtime: g is running but p is not")
  1157  		}
  1158  
  1159  		if gp.preemptShrink {
  1160  			// We're at a synchronous safe point now, so
  1161  			// do the pending stack shrink.
  1162  			gp.preemptShrink = false
  1163  			shrinkstack(gp)
  1164  		}
  1165  
  1166  		// Set a flag indicated that we've been synchronously preempted.
  1167  		gp.syncSafePoint = true
  1168  
  1169  		if gp.preemptStop {
  1170  			preemptPark(gp) // never returns
  1171  		}
  1172  
  1173  		// Act like goroutine called runtime.Gosched.
  1174  		gopreempt_m(gp) // never return
  1175  	}
  1176  
  1177  	// Allocate a bigger segment and move the stack.
  1178  	oldsize := gp.stack.hi - gp.stack.lo
  1179  	newsize := oldsize * 2
  1180  
  1181  	// Make sure we grow at least as much as needed to fit the new frame.
  1182  	// (This is just an optimization - the caller of morestack will
  1183  	// recheck the bounds on return.)
  1184  	if f := findfunc(gp.sched.pc); f.valid() {
  1185  		max := uintptr(funcMaxSPDelta(f))
  1186  		needed := max + stackGuard
  1187  		used := gp.stack.hi - gp.sched.sp
  1188  		for newsize-used < needed {
  1189  			newsize *= 2
  1190  		}
  1191  	}
  1192  
  1193  	if stackguard0 == stackForceMove {
  1194  		// Forced stack movement used for debugging.
  1195  		// Don't double the stack (or we may quickly run out
  1196  		// if this is done repeatedly).
  1197  		newsize = oldsize
  1198  	}
  1199  
  1200  	if newsize > maxstacksize || newsize > maxstackceiling {
  1201  		if maxstacksize < maxstackceiling {
  1202  			print("runtime: goroutine stack exceeds ", maxstacksize, "-byte limit\n")
  1203  		} else {
  1204  			print("runtime: goroutine stack exceeds ", maxstackceiling, "-byte limit\n")
  1205  		}
  1206  		print("runtime: sp=", hex(sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n")
  1207  		throw("stack overflow")
  1208  	}
  1209  
  1210  	// The goroutine must be executing in order to call newstack,
  1211  	// so it must be Grunning (or Gscanrunning).
  1212  	casgstatus(gp, _Grunning, _Gcopystack)
  1213  
  1214  	// The concurrent GC will not scan the stack while we are doing the copy since
  1215  	// the gp is in a Gcopystack status.
  1216  	copystack(gp, newsize)
  1217  	if stackDebug >= 1 {
  1218  		print("stack grow done\n")
  1219  	}
  1220  	casgstatus(gp, _Gcopystack, _Grunning)
  1221  	gogo(&gp.sched)
  1222  }
  1223  
  1224  //go:nosplit
  1225  func nilfunc() {
  1226  	*(*uint8)(nil) = 0
  1227  }
  1228  
  1229  // adjust Gobuf as if it executed a call to fn
  1230  // and then stopped before the first instruction in fn.
  1231  func gostartcallfn(gobuf *gobuf, fv *funcval) {
  1232  	var fn unsafe.Pointer
  1233  	if fv != nil {
  1234  		fn = unsafe.Pointer(fv.fn)
  1235  	} else {
  1236  		fn = unsafe.Pointer(abi.FuncPCABIInternal(nilfunc))
  1237  	}
  1238  	gostartcall(gobuf, fn, unsafe.Pointer(fv))
  1239  }
  1240  
  1241  // isShrinkStackSafe returns whether it's safe to attempt to shrink
  1242  // gp's stack. Shrinking the stack is only safe when we have precise
  1243  // pointer maps for all frames on the stack. The caller must hold the
  1244  // _Gscan bit for gp or must be running gp itself.
  1245  func isShrinkStackSafe(gp *g) bool {
  1246  	// We can't copy the stack if we're in a syscall.
  1247  	// The syscall might have pointers into the stack and
  1248  	// often we don't have precise pointer maps for the innermost
  1249  	// frames.
  1250  	if gp.syscallsp != 0 {
  1251  		return false
  1252  	}
  1253  	// We also can't copy the stack if we're at an asynchronous
  1254  	// safe-point because we don't have precise pointer maps for
  1255  	// all frames.
  1256  	if gp.asyncSafePoint {
  1257  		return false
  1258  	}
  1259  	// We also can't *shrink* the stack in the window between the
  1260  	// goroutine calling gopark to park on a channel and
  1261  	// gp.activeStackChans being set.
  1262  	if gp.parkingOnChan.Load() {
  1263  		return false
  1264  	}
  1265  	// We also can't copy the stack while a gp is in _Gwaiting solely
  1266  	// to make itself available to suspendG.
  1267  	//
  1268  	// In these cases, the G is actually executing on the system
  1269  	// stack, and the execution tracer, mutex profiler, etc. may want
  1270  	// to take a stack trace of the G's stack.
  1271  	//
  1272  	// Note: it's safe to access gp.waitreason here.
  1273  	// We're only calling isShrinkStackSafe if we took ownership of the
  1274  	// G with the _Gscan bit. This prevents the goroutine from transitioning,
  1275  	// which prevents gp.waitreason from changing.
  1276  	if readgstatus(gp)&^_Gscan == _Gwaiting && gp.waitreason.isWaitingForSuspendG() {
  1277  		return false
  1278  	}
  1279  	return true
  1280  }
  1281  
  1282  // Maybe shrink the stack being used by gp.
  1283  //
  1284  // gp must be stopped and we must own its stack. It may be in
  1285  // _Grunning, but only if this is our own user G.
  1286  func shrinkstack(gp *g) {
  1287  	if gp.stack.lo == 0 {
  1288  		throw("missing stack in shrinkstack")
  1289  	}
  1290  	if s := readgstatus(gp); s&_Gscan == 0 {
  1291  		// We don't own the stack via _Gscan. We could still
  1292  		// own it if this is our own user G and we're on the
  1293  		// system stack.
  1294  		if !(gp == getg().m.curg && getg() != getg().m.curg && s == _Grunning) {
  1295  			// We don't own the stack.
  1296  			throw("bad status in shrinkstack")
  1297  		}
  1298  	}
  1299  	if !isShrinkStackSafe(gp) {
  1300  		throw("shrinkstack at bad time")
  1301  	}
  1302  	// Check for self-shrinks while in a libcall. These may have
  1303  	// pointers into the stack disguised as uintptrs, but these
  1304  	// code paths should all be nosplit.
  1305  	if gp == getg().m.curg && gp.m.libcallsp != 0 {
  1306  		throw("shrinking stack in libcall")
  1307  	}
  1308  
  1309  	if debug.gcshrinkstackoff > 0 {
  1310  		return
  1311  	}
  1312  
  1313  	oldsize := gp.stack.hi - gp.stack.lo
  1314  	newsize := oldsize / 2
  1315  	// Don't shrink the allocation below the minimum-sized stack
  1316  	// allocation.
  1317  	if newsize < fixedStack {
  1318  		return
  1319  	}
  1320  	// Compute how much of the stack is currently in use and only
  1321  	// shrink the stack if gp is using less than a quarter of its
  1322  	// current stack. The currently used stack includes everything
  1323  	// down to the SP plus the stack guard space that ensures
  1324  	// there's room for nosplit functions.
  1325  	avail := gp.stack.hi - gp.stack.lo
  1326  	if used := gp.stack.hi - gp.sched.sp + stackNosplit; used >= avail/4 {
  1327  		return
  1328  	}
  1329  
  1330  	if stackDebug > 0 {
  1331  		print("shrinking stack ", oldsize, "->", newsize, "\n")
  1332  	}
  1333  
  1334  	copystack(gp, newsize)
  1335  }
  1336  
  1337  // freeStackSpans frees unused stack spans at the end of GC.
  1338  func freeStackSpans() {
  1339  	// Scan stack pools for empty stack spans.
  1340  	for order := range stackpool {
  1341  		lock(&stackpool[order].item.mu)
  1342  		list := &stackpool[order].item.span
  1343  		for s := list.first; s != nil; {
  1344  			next := s.next
  1345  			if s.allocCount == 0 {
  1346  				list.remove(s)
  1347  				s.manualFreeList = 0
  1348  				osStackFree(s)
  1349  				mheap_.freeManual(s, spanAllocStack)
  1350  			}
  1351  			s = next
  1352  		}
  1353  		unlock(&stackpool[order].item.mu)
  1354  	}
  1355  
  1356  	// Free large stack spans.
  1357  	lock(&stackLarge.lock)
  1358  	for i := range stackLarge.free {
  1359  		for s := stackLarge.free[i].first; s != nil; {
  1360  			next := s.next
  1361  			stackLarge.free[i].remove(s)
  1362  			osStackFree(s)
  1363  			mheap_.freeManual(s, spanAllocStack)
  1364  			s = next
  1365  		}
  1366  	}
  1367  	unlock(&stackLarge.lock)
  1368  }
  1369  
  1370  // A stackObjectRecord is generated by the compiler for each stack object in a stack frame.
  1371  // This record must match the generator code in cmd/compile/internal/liveness/plive.go:emitStackObjects.
  1372  type stackObjectRecord struct {
  1373  	// offset in frame
  1374  	// if negative, offset from varp
  1375  	// if non-negative, offset from argp
  1376  	off       int32
  1377  	size      int32
  1378  	ptrBytes  int32
  1379  	gcdataoff uint32 // offset to gcdata from moduledata.rodata
  1380  }
  1381  
  1382  // gcdata returns the number of bytes that contain pointers, and
  1383  // a ptr/nonptr bitmask covering those bytes.
  1384  // Note that this bitmask might be larger than internal/abi.MaxPtrmaskBytes.
  1385  func (r *stackObjectRecord) gcdata() (uintptr, *byte) {
  1386  	ptr := uintptr(unsafe.Pointer(r))
  1387  	var mod *moduledata
  1388  	for datap := &firstmoduledata; datap != nil; datap = datap.next {
  1389  		// The normal case: stackObjectRecord is in funcdata.
  1390  		if datap.gofunc <= ptr && ptr < datap.epclntab {
  1391  			mod = datap
  1392  			break
  1393  		}
  1394  		// A special case: methodValueCallFrameObjs.
  1395  		if datap.noptrbss <= ptr && ptr < datap.enoptrbss {
  1396  			mod = datap
  1397  			break
  1398  		}
  1399  	}
  1400  	// If you get a panic here due to a nil mod,
  1401  	// you may have made a copy of a stackObjectRecord.
  1402  	// You must use the original pointer.
  1403  	res := mod.rodata + uintptr(r.gcdataoff)
  1404  	return uintptr(r.ptrBytes), (*byte)(unsafe.Pointer(res))
  1405  }
  1406  
  1407  // This is exported as ABI0 via linkname so obj can call it.
  1408  //
  1409  //go:nosplit
  1410  //go:linkname morestackc
  1411  func morestackc() {
  1412  	throw("attempt to execute system stack code on user stack")
  1413  }
  1414  
  1415  // startingStackSize is the amount of stack that new goroutines start with.
  1416  // It is a power of 2, and between fixedStack and maxstacksize, inclusive.
  1417  // startingStackSize is updated every GC by tracking the average size of
  1418  // stacks scanned during the GC.
  1419  var startingStackSize uint32 = fixedStack
  1420  
  1421  func gcComputeStartingStackSize() {
  1422  	if debug.adaptivestackstart == 0 {
  1423  		return
  1424  	}
  1425  	// For details, see the design doc at
  1426  	// https://docs.google.com/document/d/1YDlGIdVTPnmUiTAavlZxBI1d9pwGQgZT7IKFKlIXohQ/edit?usp=sharing
  1427  	// The basic algorithm is to track the average size of stacks
  1428  	// and start goroutines with stack equal to that average size.
  1429  	// Starting at the average size uses at most 2x the space that
  1430  	// an ideal algorithm would have used.
  1431  	// This is just a heuristic to avoid excessive stack growth work
  1432  	// early in a goroutine's lifetime. See issue 18138. Stacks that
  1433  	// are allocated too small can still grow, and stacks allocated
  1434  	// too large can still shrink.
  1435  	var scannedStackSize uint64
  1436  	var scannedStacks uint64
  1437  	for _, p := range allp {
  1438  		scannedStackSize += p.scannedStackSize
  1439  		scannedStacks += p.scannedStacks
  1440  		// Reset for next time
  1441  		p.scannedStackSize = 0
  1442  		p.scannedStacks = 0
  1443  	}
  1444  	if scannedStacks == 0 {
  1445  		startingStackSize = fixedStack
  1446  		return
  1447  	}
  1448  	avg := scannedStackSize/scannedStacks + stackGuard
  1449  	// Note: we add stackGuard to ensure that a goroutine that
  1450  	// uses the average space will not trigger a growth.
  1451  	if avg > uint64(maxstacksize) {
  1452  		avg = uint64(maxstacksize)
  1453  	}
  1454  	if avg < fixedStack {
  1455  		avg = fixedStack
  1456  	}
  1457  	// Note: maxstacksize fits in 30 bits, so avg also does.
  1458  	startingStackSize = uint32(round2(int32(avg)))
  1459  }
  1460  

View as plain text