export_test.go

     1  // Copyright 2010 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Export guts for testing.
     6  
     7  package runtime
     8  
     9  import (
    10  	"internal/abi"
    11  	"internal/goarch"
    12  	"internal/goos"
    13  	"internal/runtime/atomic"
    14  	"internal/runtime/gc"
    15  	"internal/runtime/maps"
    16  	"internal/runtime/sys"
    17  	"unsafe"
    18  )
    19  
    20  var Fadd64 = fadd64
    21  var Fsub64 = fsub64
    22  var Fmul64 = fmul64
    23  var Fdiv64 = fdiv64
    24  var F64to32 = f64to32
    25  var F32to64 = f32to64
    26  var Fcmp64 = fcmp64
    27  var Fintto64 = fintto64
    28  var F64toint = f64toint
    29  
    30  var Entersyscall = entersyscall
    31  var Exitsyscall = exitsyscall
    32  var LockedOSThread = lockedOSThread
    33  var Xadduintptr = atomic.Xadduintptr
    34  
    35  var ReadRandomFailed = &readRandomFailed
    36  
    37  var Fastlog2 = fastlog2
    38  
    39  var ParseByteCount = parseByteCount
    40  
    41  var Nanotime = nanotime
    42  var Cputicks = cputicks
    43  var CyclesPerSecond = pprof_cyclesPerSecond
    44  var NetpollBreak = netpollBreak
    45  var Usleep = usleep
    46  
    47  var PhysPageSize = physPageSize
    48  var PhysHugePageSize = physHugePageSize
    49  
    50  var NetpollGenericInit = netpollGenericInit
    51  
    52  var Memmove = memmove
    53  var MemclrNoHeapPointers = memclrNoHeapPointers
    54  
    55  var CgoCheckPointer = cgoCheckPointer
    56  
    57  const CrashStackImplemented = crashStackImplemented
    58  
    59  const TracebackInnerFrames = tracebackInnerFrames
    60  const TracebackOuterFrames = tracebackOuterFrames
    61  
    62  var LockPartialOrder = lockPartialOrder
    63  
    64  type TimeTimer = timeTimer
    65  
    66  type LockRank lockRank
    67  
    68  func (l LockRank) String() string {
    69  	return lockRank(l).String()
    70  }
    71  
    72  const PreemptMSupported = preemptMSupported
    73  
    74  type LFNode struct {
    75  	Next    uint64
    76  	Pushcnt uintptr
    77  }
    78  
    79  func LFStackPush(head *uint64, node *LFNode) {
    80  	(*lfstack)(head).push((*lfnode)(unsafe.Pointer(node)))
    81  }
    82  
    83  func LFStackPop(head *uint64) *LFNode {
    84  	return (*LFNode)((*lfstack)(head).pop())
    85  }
    86  func LFNodeValidate(node *LFNode) {
    87  	lfnodeValidate((*lfnode)(unsafe.Pointer(node)))
    88  }
    89  
    90  func Netpoll(delta int64) {
    91  	systemstack(func() {
    92  		netpoll(delta)
    93  	})
    94  }
    95  
    96  func PointerMask(x any) (ret []byte) {
    97  	systemstack(func() {
    98  		ret = pointerMask(x)
    99  	})
   100  	return
   101  }
   102  
   103  func RunSchedLocalQueueTest() {
   104  	pp := new(p)
   105  	gs := make([]g, len(pp.runq))
   106  	Escape(gs) // Ensure gs doesn't move, since we use guintptrs
   107  	for i := 0; i < len(pp.runq); i++ {
   108  		if g, _ := runqget(pp); g != nil {
   109  			throw("runq is not empty initially")
   110  		}
   111  		for j := 0; j < i; j++ {
   112  			runqput(pp, &gs[i], false)
   113  		}
   114  		for j := 0; j < i; j++ {
   115  			if g, _ := runqget(pp); g != &gs[i] {
   116  				print("bad element at iter ", i, "/", j, "\n")
   117  				throw("bad element")
   118  			}
   119  		}
   120  		if g, _ := runqget(pp); g != nil {
   121  			throw("runq is not empty afterwards")
   122  		}
   123  	}
   124  }
   125  
   126  func RunSchedLocalQueueStealTest() {
   127  	p1 := new(p)
   128  	p2 := new(p)
   129  	gs := make([]g, len(p1.runq))
   130  	Escape(gs) // Ensure gs doesn't move, since we use guintptrs
   131  	for i := 0; i < len(p1.runq); i++ {
   132  		for j := 0; j < i; j++ {
   133  			gs[j].sig = 0
   134  			runqput(p1, &gs[j], false)
   135  		}
   136  		gp := runqsteal(p2, p1, true)
   137  		s := 0
   138  		if gp != nil {
   139  			s++
   140  			gp.sig++
   141  		}
   142  		for {
   143  			gp, _ = runqget(p2)
   144  			if gp == nil {
   145  				break
   146  			}
   147  			s++
   148  			gp.sig++
   149  		}
   150  		for {
   151  			gp, _ = runqget(p1)
   152  			if gp == nil {
   153  				break
   154  			}
   155  			gp.sig++
   156  		}
   157  		for j := 0; j < i; j++ {
   158  			if gs[j].sig != 1 {
   159  				print("bad element ", j, "(", gs[j].sig, ") at iter ", i, "\n")
   160  				throw("bad element")
   161  			}
   162  		}
   163  		if s != i/2 && s != i/2+1 {
   164  			print("bad steal ", s, ", want ", i/2, " or ", i/2+1, ", iter ", i, "\n")
   165  			throw("bad steal")
   166  		}
   167  	}
   168  }
   169  
   170  func RunSchedLocalQueueEmptyTest(iters int) {
   171  	// Test that runq is not spuriously reported as empty.
   172  	// Runq emptiness affects scheduling decisions and spurious emptiness
   173  	// can lead to underutilization (both runnable Gs and idle Ps coexist
   174  	// for arbitrary long time).
   175  	done := make(chan bool, 1)
   176  	p := new(p)
   177  	gs := make([]g, 2)
   178  	Escape(gs) // Ensure gs doesn't move, since we use guintptrs
   179  	ready := new(uint32)
   180  	for i := 0; i < iters; i++ {
   181  		*ready = 0
   182  		next0 := (i & 1) == 0
   183  		next1 := (i & 2) == 0
   184  		runqput(p, &gs[0], next0)
   185  		go func() {
   186  			for atomic.Xadd(ready, 1); atomic.Load(ready) != 2; {
   187  			}
   188  			if runqempty(p) {
   189  				println("next:", next0, next1)
   190  				throw("queue is empty")
   191  			}
   192  			done <- true
   193  		}()
   194  		for atomic.Xadd(ready, 1); atomic.Load(ready) != 2; {
   195  		}
   196  		runqput(p, &gs[1], next1)
   197  		runqget(p)
   198  		<-done
   199  		runqget(p)
   200  	}
   201  }
   202  
   203  var (
   204  	StringHash = stringHash
   205  	BytesHash  = bytesHash
   206  	Int32Hash  = int32Hash
   207  	Int64Hash  = int64Hash
   208  	MemHash    = memhash
   209  	MemHash32  = memhash32
   210  	MemHash64  = memhash64
   211  	EfaceHash  = efaceHash
   212  	IfaceHash  = ifaceHash
   213  )
   214  
   215  var MinAeshashSize = &maps.MinAeshashSize
   216  
   217  var AeshashEnabled = maps.AeshashEnabled
   218  
   219  func MemclrBytes(b []byte) {
   220  	s := (*slice)(unsafe.Pointer(&b))
   221  	memclrNoHeapPointers(s.array, uintptr(s.len))
   222  }
   223  
   224  const HashLoad = hashLoad
   225  
   226  // entry point for testing
   227  func GostringW(w []uint16) (s string) {
   228  	systemstack(func() {
   229  		s = gostringw(&w[0])
   230  	})
   231  	return
   232  }
   233  
   234  var Open = open
   235  var Close = closefd
   236  var Read = read
   237  var Write = write
   238  
   239  func Envs() []string     { return envs }
   240  func SetEnvs(e []string) { envs = e }
   241  
   242  const PtrSize = goarch.PtrSize
   243  
   244  const ClobberdeadPtr = clobberdeadPtr
   245  
   246  func Clobberfree() bool {
   247  	return debug.clobberfree != 0
   248  }
   249  
   250  var ForceGCPeriod = &forcegcperiod
   251  
   252  // SetTracebackEnv is like runtime/debug.SetTraceback, but it raises
   253  // the "environment" traceback level, so later calls to
   254  // debug.SetTraceback (e.g., from testing timeouts) can't lower it.
   255  func SetTracebackEnv(level string) {
   256  	setTraceback(level)
   257  	traceback_env = traceback_cache
   258  }
   259  
   260  var ReadUnaligned64 = readUnaligned64
   261  
   262  func CountPagesInUse() (pagesInUse, counted uintptr) {
   263  	stw := stopTheWorld(stwForTestCountPagesInUse)
   264  
   265  	pagesInUse = mheap_.pagesInUse.Load()
   266  
   267  	for _, s := range mheap_.allspans {
   268  		if s.state.get() == mSpanInUse {
   269  			counted += s.npages
   270  		}
   271  	}
   272  
   273  	startTheWorld(stw)
   274  
   275  	return
   276  }
   277  
   278  func Blocksampled(cycles, rate int64) bool { return blocksampled(cycles, rate) }
   279  
   280  func Cheaprand() uint32         { return cheaprand() }
   281  func Cheaprand64() int64        { return cheaprand64() }
   282  func Fastrand() uint32          { return uint32(rand()) }
   283  func Fastrand64() uint64        { return rand() }
   284  func Fastrandn(n uint32) uint32 { return randn(n) }
   285  
   286  type ProfBuf profBuf
   287  
   288  func NewProfBuf(hdrsize, bufwords, tags int) *ProfBuf {
   289  	return (*ProfBuf)(newProfBuf(hdrsize, bufwords, tags))
   290  }
   291  
   292  func (p *ProfBuf) Write(tag *unsafe.Pointer, now int64, hdr []uint64, stk []uintptr) {
   293  	(*profBuf)(p).write(tag, now, hdr, stk)
   294  }
   295  
   296  const (
   297  	ProfBufBlocking    = profBufBlocking
   298  	ProfBufNonBlocking = profBufNonBlocking
   299  )
   300  
   301  func (p *ProfBuf) Read(mode profBufReadMode) ([]uint64, []unsafe.Pointer, bool) {
   302  	return (*profBuf)(p).read(mode)
   303  }
   304  
   305  func (p *ProfBuf) Close() {
   306  	(*profBuf)(p).close()
   307  }
   308  
   309  type CPUStats = cpuStats
   310  
   311  func ReadCPUStats() CPUStats {
   312  	return work.cpuStats
   313  }
   314  
   315  func ReadMetricsSlow(memStats *MemStats, samplesp unsafe.Pointer, len, cap int) {
   316  	stw := stopTheWorld(stwForTestReadMetricsSlow)
   317  
   318  	// Initialize the metrics beforehand because this could
   319  	// allocate and skew the stats.
   320  	metricsLock()
   321  	initMetrics()
   322  
   323  	systemstack(func() {
   324  		// Donate the racectx to g0. readMetricsLocked calls into the race detector
   325  		// via map access.
   326  		getg().racectx = getg().m.curg.racectx
   327  
   328  		// Read the metrics once before in case it allocates and skews the metrics.
   329  		// readMetricsLocked is designed to only allocate the first time it is called
   330  		// with a given slice of samples. In effect, this extra read tests that this
   331  		// remains true, since otherwise the second readMetricsLocked below could
   332  		// allocate before it returns.
   333  		readMetricsLocked(samplesp, len, cap)
   334  
   335  		// Read memstats first. It's going to flush
   336  		// the mcaches which readMetrics does not do, so
   337  		// going the other way around may result in
   338  		// inconsistent statistics.
   339  		readmemstats_m(memStats)
   340  
   341  		// Read metrics again. We need to be sure we're on the
   342  		// system stack with readmemstats_m so that we don't call into
   343  		// the stack allocator and adjust metrics between there and here.
   344  		readMetricsLocked(samplesp, len, cap)
   345  
   346  		// Undo the donation.
   347  		getg().racectx = 0
   348  	})
   349  	metricsUnlock()
   350  
   351  	startTheWorld(stw)
   352  }
   353  
   354  var DoubleCheckReadMemStats = &doubleCheckReadMemStats
   355  
   356  // ReadMemStatsSlow returns both the runtime-computed MemStats and
   357  // MemStats accumulated by scanning the heap.
   358  func ReadMemStatsSlow() (base, slow MemStats) {
   359  	stw := stopTheWorld(stwForTestReadMemStatsSlow)
   360  
   361  	// Run on the system stack to avoid stack growth allocation.
   362  	systemstack(func() {
   363  		// Make sure stats don't change.
   364  		getg().m.mallocing++
   365  
   366  		readmemstats_m(&base)
   367  
   368  		// Initialize slow from base and zero the fields we're
   369  		// recomputing.
   370  		slow = base
   371  		slow.Alloc = 0
   372  		slow.TotalAlloc = 0
   373  		slow.Mallocs = 0
   374  		slow.Frees = 0
   375  		slow.HeapReleased = 0
   376  		var bySize [gc.NumSizeClasses]struct {
   377  			Mallocs, Frees uint64
   378  		}
   379  
   380  		// Add up current allocations in spans.
   381  		for _, s := range mheap_.allspans {
   382  			if s.state.get() != mSpanInUse {
   383  				continue
   384  			}
   385  			if s.isUnusedUserArenaChunk() {
   386  				continue
   387  			}
   388  			if sizeclass := s.spanclass.sizeclass(); sizeclass == 0 {
   389  				slow.Mallocs++
   390  				slow.Alloc += uint64(s.elemsize)
   391  			} else {
   392  				slow.Mallocs += uint64(s.allocCount)
   393  				slow.Alloc += uint64(s.allocCount) * uint64(s.elemsize)
   394  				bySize[sizeclass].Mallocs += uint64(s.allocCount)
   395  			}
   396  		}
   397  
   398  		// Add in frees by just reading the stats for those directly.
   399  		var m heapStatsDelta
   400  		memstats.heapStats.unsafeRead(&m)
   401  
   402  		// Collect per-sizeclass free stats.
   403  		var smallFree uint64
   404  		for i := 0; i < gc.NumSizeClasses; i++ {
   405  			slow.Frees += m.smallFreeCount[i]
   406  			bySize[i].Frees += m.smallFreeCount[i]
   407  			bySize[i].Mallocs += m.smallFreeCount[i]
   408  			smallFree += m.smallFreeCount[i] * uint64(gc.SizeClassToSize[i])
   409  		}
   410  		slow.Frees += m.tinyAllocCount + m.largeFreeCount
   411  		slow.Mallocs += slow.Frees
   412  
   413  		slow.TotalAlloc = slow.Alloc + m.largeFree + smallFree
   414  
   415  		for i := range slow.BySize {
   416  			slow.BySize[i].Mallocs = bySize[i].Mallocs
   417  			slow.BySize[i].Frees = bySize[i].Frees
   418  		}
   419  
   420  		for i := mheap_.pages.start; i < mheap_.pages.end; i++ {
   421  			chunk := mheap_.pages.tryChunkOf(i)
   422  			if chunk == nil {
   423  				continue
   424  			}
   425  			pg := chunk.scavenged.popcntRange(0, pallocChunkPages)
   426  			slow.HeapReleased += uint64(pg) * pageSize
   427  		}
   428  		for _, p := range allp {
   429  			// Only count scav bits for pages in the cache
   430  			pg := sys.OnesCount64(p.pcache.cache & p.pcache.scav)
   431  			slow.HeapReleased += uint64(pg) * pageSize
   432  		}
   433  
   434  		getg().m.mallocing--
   435  	})
   436  
   437  	startTheWorld(stw)
   438  	return
   439  }
   440  
   441  // ShrinkStackAndVerifyFramePointers attempts to shrink the stack of the current goroutine
   442  // and verifies that unwinding the new stack doesn't crash, even if the old
   443  // stack has been freed or reused (simulated via poisoning).
   444  func ShrinkStackAndVerifyFramePointers() {
   445  	before := stackPoisonCopy
   446  	defer func() { stackPoisonCopy = before }()
   447  	stackPoisonCopy = 1
   448  
   449  	gp := getg()
   450  	systemstack(func() {
   451  		shrinkstack(gp)
   452  	})
   453  	// If our new stack contains frame pointers into the old stack, this will
   454  	// crash because the old stack has been poisoned.
   455  	FPCallers(make([]uintptr, 1024))
   456  }
   457  
   458  type StackPoisonCopyRestore int
   459  
   460  func (s StackPoisonCopyRestore) Restore() { stackPoisonCopy = int(s) }
   461  
   462  func StackPoisonCopy() StackPoisonCopyRestore {
   463  	before := stackPoisonCopy
   464  	stackPoisonCopy = 1
   465  	return StackPoisonCopyRestore(before)
   466  }
   467  
   468  // BlockOnSystemStack switches to the system stack, prints "x\n" to
   469  // stderr, and blocks in a stack containing
   470  // "runtime.blockOnSystemStackInternal".
   471  func BlockOnSystemStack() {
   472  	systemstack(blockOnSystemStackInternal)
   473  }
   474  
   475  func blockOnSystemStackInternal() {
   476  	print("x\n")
   477  	lock(&deadlock)
   478  	lock(&deadlock)
   479  }
   480  
   481  type RWMutex struct {
   482  	rw rwmutex
   483  }
   484  
   485  func (rw *RWMutex) Init() {
   486  	rw.rw.init(lockRankTestR, lockRankTestRInternal, lockRankTestW)
   487  }
   488  
   489  func (rw *RWMutex) RLock() {
   490  	rw.rw.rlock()
   491  }
   492  
   493  func (rw *RWMutex) RUnlock() {
   494  	rw.rw.runlock()
   495  }
   496  
   497  func (rw *RWMutex) Lock() {
   498  	rw.rw.lock()
   499  }
   500  
   501  func (rw *RWMutex) Unlock() {
   502  	rw.rw.unlock()
   503  }
   504  
   505  func LockOSCounts() (external, internal uint32) {
   506  	gp := getg()
   507  	if gp.m.lockedExt+gp.m.lockedInt == 0 {
   508  		if gp.lockedm != 0 {
   509  			panic("lockedm on non-locked goroutine")
   510  		}
   511  	} else {
   512  		if gp.lockedm == 0 {
   513  			panic("nil lockedm on locked goroutine")
   514  		}
   515  	}
   516  	return gp.m.lockedExt, gp.m.lockedInt
   517  }
   518  
   519  //go:noinline
   520  func TracebackSystemstack(stk []uintptr, i int) int {
   521  	if i == 0 {
   522  		pc, sp := sys.GetCallerPC(), sys.GetCallerSP()
   523  		var u unwinder
   524  		u.initAt(pc, sp, 0, getg(), unwindJumpStack) // Don't ignore errors, for testing
   525  		return tracebackPCs(&u, 0, stk)
   526  	}
   527  	n := 0
   528  	systemstack(func() {
   529  		n = TracebackSystemstack(stk, i-1)
   530  	})
   531  	return n
   532  }
   533  
   534  func KeepNArenaHints(n int) {
   535  	hint := mheap_.arenaHints
   536  	for i := 1; i < n; i++ {
   537  		hint = hint.next
   538  		if hint == nil {
   539  			return
   540  		}
   541  	}
   542  	hint.next = nil
   543  }
   544  
   545  // MapNextArenaHint reserves a page at the next arena growth hint,
   546  // preventing the arena from growing there, and returns the range of
   547  // addresses that are no longer viable.
   548  //
   549  // This may fail to reserve memory. If it fails, it still returns the
   550  // address range it attempted to reserve.
   551  func MapNextArenaHint() (start, end uintptr, ok bool) {
   552  	hint := mheap_.arenaHints
   553  	addr := hint.addr
   554  	if hint.down {
   555  		start, end = addr-heapArenaBytes, addr
   556  		addr -= physPageSize
   557  	} else {
   558  		start, end = addr, addr+heapArenaBytes
   559  	}
   560  	got := sysReserve(unsafe.Pointer(addr), physPageSize, "")
   561  	ok = (addr == uintptr(got))
   562  	if !ok {
   563  		// We were unable to get the requested reservation.
   564  		// Release what we did get and fail.
   565  		sysUnreserve(got, physPageSize)
   566  	}
   567  	return
   568  }
   569  
   570  func NextArenaHint() (uintptr, bool) {
   571  	if mheap_.arenaHints == nil {
   572  		return 0, false
   573  	}
   574  	return mheap_.arenaHints.addr, true
   575  }
   576  
   577  const RandomizeHeapBase = randomizeHeapBase
   578  
   579  // ArenaHintAddrs returns the heap's remaining arena hint addresses, in
   580  // chain order.
   581  func ArenaHintAddrs() []uintptr {
   582  	// Preallocate: appending while holding the heap lock would allocate
   583  	// under mheap_.lock. mallocinit generates at most 64 heap hints.
   584  	out := make([]uintptr, 0, 128)
   585  	lock(&mheap_.lock)
   586  	for h := mheap_.arenaHints; h != nil; h = h.next {
   587  		out = append(out, h.addr)
   588  	}
   589  	unlock(&mheap_.lock)
   590  	return out
   591  }
   592  
   593  type G = g
   594  
   595  type Sudog = sudog
   596  
   597  type XRegPerG = xRegPerG
   598  
   599  func Getg() *G {
   600  	return getg()
   601  }
   602  
   603  func Goid() uint64 {
   604  	return getg().goid
   605  }
   606  
   607  func GIsWaitingOnMutex(gp *G) bool {
   608  	return readgstatus(gp) == _Gwaiting && gp.waitreason.isMutexWait()
   609  }
   610  
   611  var CasGStatusAlwaysTrack = &casgstatusAlwaysTrack
   612  
   613  //go:noinline
   614  func PanicForTesting(b []byte, i int) byte {
   615  	return unexportedPanicForTesting(b, i)
   616  }
   617  
   618  //go:noinline
   619  func unexportedPanicForTesting(b []byte, i int) byte {
   620  	return b[i]
   621  }
   622  
   623  func G0StackOverflow() {
   624  	systemstack(func() {
   625  		g0 := getg()
   626  		sp := sys.GetCallerSP()
   627  		// The stack bounds for g0 stack is not always precise.
   628  		// Use an artificially small stack, to trigger a stack overflow
   629  		// without actually run out of the system stack (which may seg fault).
   630  		g0.stack.lo = sp - 4096 - stackSystem
   631  		g0.stackguard0 = g0.stack.lo + stackGuard
   632  		g0.stackguard1 = g0.stackguard0
   633  
   634  		stackOverflow(nil)
   635  	})
   636  }
   637  
   638  func stackOverflow(x *byte) {
   639  	var buf [256]byte
   640  	stackOverflow(&buf[0])
   641  }
   642  
   643  func RunGetgThreadSwitchTest() {
   644  	// Test that getg works correctly with thread switch.
   645  	// With gccgo, if we generate getg inlined, the backend
   646  	// may cache the address of the TLS variable, which
   647  	// will become invalid after a thread switch. This test
   648  	// checks that the bad caching doesn't happen.
   649  
   650  	ch := make(chan int)
   651  	go func(ch chan int) {
   652  		ch <- 5
   653  		LockOSThread()
   654  	}(ch)
   655  
   656  	g1 := getg()
   657  
   658  	// Block on a receive. This is likely to get us a thread
   659  	// switch. If we yield to the sender goroutine, it will
   660  	// lock the thread, forcing us to resume on a different
   661  	// thread.
   662  	<-ch
   663  
   664  	g2 := getg()
   665  	if g1 != g2 {
   666  		panic("g1 != g2")
   667  	}
   668  
   669  	// Also test getg after some control flow, as the
   670  	// backend is sensitive to control flow.
   671  	g3 := getg()
   672  	if g1 != g3 {
   673  		panic("g1 != g3")
   674  	}
   675  }
   676  
   677  // Expose freegc for testing.
   678  func Freegc(p unsafe.Pointer, size uintptr, noscan bool) {
   679  	freegc(p, size, noscan)
   680  }
   681  
   682  // Expose gcAssistBytes for the current g for testing.
   683  func AssistCredit() int64 {
   684  	assistG := getg()
   685  	if assistG.m.curg != nil {
   686  		assistG = assistG.m.curg
   687  	}
   688  	return assistG.gcAssistBytes
   689  }
   690  
   691  // Expose gcBlackenEnabled for testing.
   692  func GcBlackenEnable() bool {
   693  	// Note we do a non-atomic load here.
   694  	// Some checks against gcBlackenEnabled (e.g., in mallocgc)
   695  	// are currently done via non-atomic load for performance reasons,
   696  	// but other checks are done via atomic load (e.g., in mgcmark.go),
   697  	// so interpreting this value in a test may be subtle.
   698  	return gcBlackenEnabled != 0
   699  }
   700  
   701  const SizeSpecializedMallocEnabled = sizeSpecializedMallocEnabled
   702  
   703  const RuntimeFreegcEnabled = runtimeFreegcEnabled
   704  
   705  const (
   706  	PageSize         = pageSize
   707  	PallocChunkPages = pallocChunkPages
   708  	PageAlloc64Bit   = pageAlloc64Bit
   709  	PallocSumBytes   = pallocSumBytes
   710  )
   711  
   712  // Expose pallocSum for testing.
   713  type PallocSum pallocSum
   714  
   715  func PackPallocSum(start, max, end uint) PallocSum { return PallocSum(packPallocSum(start, max, end)) }
   716  func (m PallocSum) Start() uint                    { return pallocSum(m).start() }
   717  func (m PallocSum) Max() uint                      { return pallocSum(m).max() }
   718  func (m PallocSum) End() uint                      { return pallocSum(m).end() }
   719  
   720  // Expose pallocBits for testing.
   721  type PallocBits pallocBits
   722  
   723  func (b *PallocBits) Find(npages uintptr, searchIdx uint) (uint, uint) {
   724  	return (*pallocBits)(b).find(npages, searchIdx)
   725  }
   726  func (b *PallocBits) AllocRange(i, n uint)       { (*pallocBits)(b).allocRange(i, n) }
   727  func (b *PallocBits) Free(i, n uint)             { (*pallocBits)(b).free(i, n) }
   728  func (b *PallocBits) Summarize() PallocSum       { return PallocSum((*pallocBits)(b).summarize()) }
   729  func (b *PallocBits) PopcntRange(i, n uint) uint { return (*pageBits)(b).popcntRange(i, n) }
   730  
   731  // SummarizeSlow is a slow but more obviously correct implementation
   732  // of (*pallocBits).summarize. Used for testing.
   733  func SummarizeSlow(b *PallocBits) PallocSum {
   734  	var start, most, end uint
   735  
   736  	const N = uint(len(b)) * 64
   737  	for start < N && (*pageBits)(b).get(start) == 0 {
   738  		start++
   739  	}
   740  	for end < N && (*pageBits)(b).get(N-end-1) == 0 {
   741  		end++
   742  	}
   743  	run := uint(0)
   744  	for i := uint(0); i < N; i++ {
   745  		if (*pageBits)(b).get(i) == 0 {
   746  			run++
   747  		} else {
   748  			run = 0
   749  		}
   750  		most = max(most, run)
   751  	}
   752  	return PackPallocSum(start, most, end)
   753  }
   754  
   755  // Expose non-trivial helpers for testing.
   756  func FindBitRange64(c uint64, n uint) uint { return findBitRange64(c, n) }
   757  
   758  // Given two PallocBits, returns a set of bit ranges where
   759  // they differ.
   760  func DiffPallocBits(a, b *PallocBits) []BitRange {
   761  	ba := (*pageBits)(a)
   762  	bb := (*pageBits)(b)
   763  
   764  	var d []BitRange
   765  	base, size := uint(0), uint(0)
   766  	for i := uint(0); i < uint(len(ba))*64; i++ {
   767  		if ba.get(i) != bb.get(i) {
   768  			if size == 0 {
   769  				base = i
   770  			}
   771  			size++
   772  		} else {
   773  			if size != 0 {
   774  				d = append(d, BitRange{base, size})
   775  			}
   776  			size = 0
   777  		}
   778  	}
   779  	if size != 0 {
   780  		d = append(d, BitRange{base, size})
   781  	}
   782  	return d
   783  }
   784  
   785  // StringifyPallocBits gets the bits in the bit range r from b,
   786  // and returns a string containing the bits as ASCII 0 and 1
   787  // characters.
   788  func StringifyPallocBits(b *PallocBits, r BitRange) string {
   789  	str := ""
   790  	for j := r.I; j < r.I+r.N; j++ {
   791  		if (*pageBits)(b).get(j) != 0 {
   792  			str += "1"
   793  		} else {
   794  			str += "0"
   795  		}
   796  	}
   797  	return str
   798  }
   799  
   800  // Expose pallocData for testing.
   801  type PallocData pallocData
   802  
   803  func (d *PallocData) FindScavengeCandidate(searchIdx uint, min, max uintptr) (uint, uint) {
   804  	return (*pallocData)(d).findScavengeCandidate(searchIdx, min, max)
   805  }
   806  func (d *PallocData) AllocRange(i, n uint) { (*pallocData)(d).allocRange(i, n) }
   807  func (d *PallocData) ScavengedSetRange(i, n uint) {
   808  	(*pallocData)(d).scavenged.setRange(i, n)
   809  }
   810  func (d *PallocData) PallocBits() *PallocBits {
   811  	return (*PallocBits)(&(*pallocData)(d).pallocBits)
   812  }
   813  func (d *PallocData) Scavenged() *PallocBits {
   814  	return (*PallocBits)(&(*pallocData)(d).scavenged)
   815  }
   816  
   817  // Expose fillAligned for testing.
   818  func FillAligned(x uint64, m uint) uint64 { return fillAligned(x, m) }
   819  
   820  // Expose pageCache for testing.
   821  type PageCache pageCache
   822  
   823  const PageCachePages = pageCachePages
   824  
   825  func NewPageCache(base uintptr, cache, scav uint64) PageCache {
   826  	return PageCache(pageCache{base: base, cache: cache, scav: scav})
   827  }
   828  func (c *PageCache) Empty() bool   { return (*pageCache)(c).empty() }
   829  func (c *PageCache) Base() uintptr { return (*pageCache)(c).base }
   830  func (c *PageCache) Cache() uint64 { return (*pageCache)(c).cache }
   831  func (c *PageCache) Scav() uint64  { return (*pageCache)(c).scav }
   832  func (c *PageCache) Alloc(npages uintptr) (uintptr, uintptr) {
   833  	return (*pageCache)(c).alloc(npages)
   834  }
   835  func (c *PageCache) Flush(s *PageAlloc) {
   836  	cp := (*pageCache)(c)
   837  	sp := (*pageAlloc)(s)
   838  
   839  	systemstack(func() {
   840  		// None of the tests need any higher-level locking, so we just
   841  		// take the lock internally.
   842  		lock(sp.mheapLock)
   843  		cp.flush(sp)
   844  		unlock(sp.mheapLock)
   845  	})
   846  }
   847  
   848  // Expose chunk index type.
   849  type ChunkIdx chunkIdx
   850  
   851  // Expose pageAlloc for testing. Note that because pageAlloc is
   852  // not in the heap, so is PageAlloc.
   853  type PageAlloc pageAlloc
   854  
   855  func (p *PageAlloc) Alloc(npages uintptr) (uintptr, uintptr) {
   856  	pp := (*pageAlloc)(p)
   857  
   858  	var addr, scav uintptr
   859  	systemstack(func() {
   860  		// None of the tests need any higher-level locking, so we just
   861  		// take the lock internally.
   862  		lock(pp.mheapLock)
   863  		addr, scav = pp.alloc(npages)
   864  		unlock(pp.mheapLock)
   865  	})
   866  	return addr, scav
   867  }
   868  func (p *PageAlloc) AllocToCache() PageCache {
   869  	pp := (*pageAlloc)(p)
   870  
   871  	var c PageCache
   872  	systemstack(func() {
   873  		// None of the tests need any higher-level locking, so we just
   874  		// take the lock internally.
   875  		lock(pp.mheapLock)
   876  		c = PageCache(pp.allocToCache())
   877  		unlock(pp.mheapLock)
   878  	})
   879  	return c
   880  }
   881  func (p *PageAlloc) Free(base, npages uintptr) {
   882  	pp := (*pageAlloc)(p)
   883  
   884  	systemstack(func() {
   885  		// None of the tests need any higher-level locking, so we just
   886  		// take the lock internally.
   887  		lock(pp.mheapLock)
   888  		pp.free(base, npages)
   889  		unlock(pp.mheapLock)
   890  	})
   891  }
   892  func (p *PageAlloc) Bounds() (ChunkIdx, ChunkIdx) {
   893  	return ChunkIdx((*pageAlloc)(p).start), ChunkIdx((*pageAlloc)(p).end)
   894  }
   895  func (p *PageAlloc) Scavenge(nbytes uintptr) (r uintptr) {
   896  	pp := (*pageAlloc)(p)
   897  	systemstack(func() {
   898  		r = pp.scavenge(nbytes, nil, true)
   899  	})
   900  	return
   901  }
   902  func (p *PageAlloc) InUse() []AddrRange {
   903  	ranges := make([]AddrRange, 0, len(p.inUse.ranges))
   904  	for _, r := range p.inUse.ranges {
   905  		ranges = append(ranges, AddrRange{r})
   906  	}
   907  	return ranges
   908  }
   909  
   910  // Returns nil if the PallocData's L2 is missing.
   911  func (p *PageAlloc) PallocData(i ChunkIdx) *PallocData {
   912  	ci := chunkIdx(i)
   913  	return (*PallocData)((*pageAlloc)(p).tryChunkOf(ci))
   914  }
   915  
   916  // AddrRange is a wrapper around addrRange for testing.
   917  type AddrRange struct {
   918  	addrRange
   919  }
   920  
   921  // MakeAddrRange creates a new address range.
   922  func MakeAddrRange(base, limit uintptr) AddrRange {
   923  	return AddrRange{makeAddrRange(base, limit)}
   924  }
   925  
   926  // Base returns the virtual base address of the address range.
   927  func (a AddrRange) Base() uintptr {
   928  	return a.addrRange.base.addr()
   929  }
   930  
   931  // Base returns the virtual address of the limit of the address range.
   932  func (a AddrRange) Limit() uintptr {
   933  	return a.addrRange.limit.addr()
   934  }
   935  
   936  // Equals returns true if the two address ranges are exactly equal.
   937  func (a AddrRange) Equals(b AddrRange) bool {
   938  	return a == b
   939  }
   940  
   941  // Size returns the size in bytes of the address range.
   942  func (a AddrRange) Size() uintptr {
   943  	return a.addrRange.size()
   944  }
   945  
   946  // testSysStat is the sysStat passed to test versions of various
   947  // runtime structures. We do actually have to keep track of this
   948  // because otherwise memstats.mappedReady won't actually line up
   949  // with other stats in the runtime during tests.
   950  var testSysStat = &memstats.other_sys
   951  
   952  // AddrRanges is a wrapper around addrRanges for testing.
   953  type AddrRanges struct {
   954  	addrRanges
   955  	mutable bool
   956  }
   957  
   958  // NewAddrRanges creates a new empty addrRanges.
   959  //
   960  // Note that this initializes addrRanges just like in the
   961  // runtime, so its memory is persistentalloc'd. Call this
   962  // function sparingly since the memory it allocates is
   963  // leaked.
   964  //
   965  // This AddrRanges is mutable, so we can test methods like
   966  // Add.
   967  func NewAddrRanges() AddrRanges {
   968  	r := addrRanges{}
   969  	r.init(testSysStat)
   970  	return AddrRanges{r, true}
   971  }
   972  
   973  // MakeAddrRanges creates a new addrRanges populated with
   974  // the ranges in a.
   975  //
   976  // The returned AddrRanges is immutable, so methods like
   977  // Add will fail.
   978  func MakeAddrRanges(a ...AddrRange) AddrRanges {
   979  	// Methods that manipulate the backing store of addrRanges.ranges should
   980  	// not be used on the result from this function (e.g. add) since they may
   981  	// trigger reallocation. That would normally be fine, except the new
   982  	// backing store won't come from the heap, but from persistentalloc, so
   983  	// we'll leak some memory implicitly.
   984  	ranges := make([]addrRange, 0, len(a))
   985  	total := uintptr(0)
   986  	for _, r := range a {
   987  		ranges = append(ranges, r.addrRange)
   988  		total += r.Size()
   989  	}
   990  	return AddrRanges{addrRanges{
   991  		ranges:     ranges,
   992  		totalBytes: total,
   993  		sysStat:    testSysStat,
   994  	}, false}
   995  }
   996  
   997  // Ranges returns a copy of the ranges described by the
   998  // addrRanges.
   999  func (a *AddrRanges) Ranges() []AddrRange {
  1000  	result := make([]AddrRange, 0, len(a.addrRanges.ranges))
  1001  	for _, r := range a.addrRanges.ranges {
  1002  		result = append(result, AddrRange{r})
  1003  	}
  1004  	return result
  1005  }
  1006  
  1007  // FindSucc returns the successor to base. See addrRanges.findSucc
  1008  // for more details.
  1009  func (a *AddrRanges) FindSucc(base uintptr) int {
  1010  	return a.findSucc(base)
  1011  }
  1012  
  1013  // Add adds a new AddrRange to the AddrRanges.
  1014  //
  1015  // The AddrRange must be mutable (i.e. created by NewAddrRanges),
  1016  // otherwise this method will throw.
  1017  func (a *AddrRanges) Add(r AddrRange) {
  1018  	if !a.mutable {
  1019  		throw("attempt to mutate immutable AddrRanges")
  1020  	}
  1021  	a.add(r.addrRange)
  1022  }
  1023  
  1024  // TotalBytes returns the totalBytes field of the addrRanges.
  1025  func (a *AddrRanges) TotalBytes() uintptr {
  1026  	return a.addrRanges.totalBytes
  1027  }
  1028  
  1029  // BitRange represents a range over a bitmap.
  1030  type BitRange struct {
  1031  	I, N uint // bit index and length in bits
  1032  }
  1033  
  1034  // NewPageAlloc creates a new page allocator for testing and
  1035  // initializes it with the scav and chunks maps. Each key in these maps
  1036  // represents a chunk index and each value is a series of bit ranges to
  1037  // set within each bitmap's chunk.
  1038  //
  1039  // The initialization of the pageAlloc preserves the invariant that if a
  1040  // scavenged bit is set the alloc bit is necessarily unset, so some
  1041  // of the bits described by scav may be cleared in the final bitmap if
  1042  // ranges in chunks overlap with them.
  1043  //
  1044  // scav is optional, and if nil, the scavenged bitmap will be cleared
  1045  // (as opposed to all 1s, which it usually is). Furthermore, every
  1046  // chunk index in scav must appear in chunks; ones that do not are
  1047  // ignored.
  1048  func NewPageAlloc(chunks, scav map[ChunkIdx][]BitRange) *PageAlloc {
  1049  	p := new(pageAlloc)
  1050  
  1051  	// We've got an entry, so initialize the pageAlloc.
  1052  	p.init(new(mutex), testSysStat, true)
  1053  	lockInit(p.mheapLock, lockRankMheap)
  1054  	for i, init := range chunks {
  1055  		addr := chunkBase(chunkIdx(i))
  1056  
  1057  		// Mark the chunk's existence in the pageAlloc.
  1058  		systemstack(func() {
  1059  			lock(p.mheapLock)
  1060  			p.grow(addr, pallocChunkBytes)
  1061  			unlock(p.mheapLock)
  1062  		})
  1063  
  1064  		// Initialize the bitmap and update pageAlloc metadata.
  1065  		ci := chunkIndex(addr)
  1066  		chunk := p.chunkOf(ci)
  1067  
  1068  		// Clear all the scavenged bits which grow set.
  1069  		chunk.scavenged.clearRange(0, pallocChunkPages)
  1070  
  1071  		// Simulate the allocation and subsequent free of all pages in
  1072  		// the chunk for the scavenge index. This sets the state equivalent
  1073  		// with all pages within the index being free.
  1074  		p.scav.index.alloc(ci, pallocChunkPages)
  1075  		p.scav.index.free(ci, 0, pallocChunkPages)
  1076  
  1077  		// Apply scavenge state if applicable.
  1078  		if scav != nil {
  1079  			if scvg, ok := scav[i]; ok {
  1080  				for _, s := range scvg {
  1081  					// Ignore the case of s.N == 0. setRange doesn't handle
  1082  					// it and it's a no-op anyway.
  1083  					if s.N != 0 {
  1084  						chunk.scavenged.setRange(s.I, s.N)
  1085  					}
  1086  				}
  1087  			}
  1088  		}
  1089  
  1090  		// Apply alloc state.
  1091  		for _, s := range init {
  1092  			// Ignore the case of s.N == 0. allocRange doesn't handle
  1093  			// it and it's a no-op anyway.
  1094  			if s.N != 0 {
  1095  				chunk.allocRange(s.I, s.N)
  1096  
  1097  				// Make sure the scavenge index is updated.
  1098  				p.scav.index.alloc(ci, s.N)
  1099  			}
  1100  		}
  1101  
  1102  		// Update heap metadata for the allocRange calls above.
  1103  		systemstack(func() {
  1104  			lock(p.mheapLock)
  1105  			p.update(addr, pallocChunkPages, false, false)
  1106  			unlock(p.mheapLock)
  1107  		})
  1108  	}
  1109  
  1110  	return (*PageAlloc)(p)
  1111  }
  1112  
  1113  // FreePageAlloc releases hard OS resources owned by the pageAlloc. Once this
  1114  // is called the pageAlloc may no longer be used. The object itself will be
  1115  // collected by the garbage collector once it is no longer live.
  1116  func FreePageAlloc(pp *PageAlloc) {
  1117  	p := (*pageAlloc)(pp)
  1118  
  1119  	// Free all the mapped space for the summary levels.
  1120  	if pageAlloc64Bit != 0 {
  1121  		for l := 0; l < summaryLevels; l++ {
  1122  			// This isn't quite right, as some of this memory may
  1123  			// be Ready instead of Reserved. The mappedReady and
  1124  			// testSysStat adjustments below correct for the
  1125  			// difference.
  1126  			sysUnreserve(unsafe.Pointer(&p.summary[l][0]), uintptr(cap(p.summary[l]))*pallocSumBytes)
  1127  		}
  1128  	} else {
  1129  		resSize := uintptr(0)
  1130  		for _, s := range p.summary {
  1131  			resSize += uintptr(cap(s)) * pallocSumBytes
  1132  		}
  1133  		// See sysUnreserve comment above.
  1134  		sysUnreserve(unsafe.Pointer(&p.summary[0][0]), alignUp(resSize, physPageSize))
  1135  	}
  1136  
  1137  	// Subtract back out whatever we mapped for the summaries.
  1138  	// sysUsed adds to p.sysStat and memstats.mappedReady no matter what
  1139  	// (and in anger should actually be accounted for), and there's no other
  1140  	// way to figure out how much we actually mapped.
  1141  	gcController.mappedReady.Add(-int64(p.summaryMappedReady))
  1142  	testSysStat.add(-int64(p.summaryMappedReady))
  1143  
  1144  	// Free extra data structures.
  1145  	//
  1146  	// TODO(prattmic): As above, some of this may be Ready, so we should
  1147  	// manually adjust mappedReady and testSysStat?
  1148  	sysUnreserve(unsafe.Pointer(&p.scav.index.chunks[0]), uintptr(cap(p.scav.index.chunks))*unsafe.Sizeof(atomicScavChunkData{}))
  1149  
  1150  	// Free the mapped space for chunks.
  1151  	for i := range p.chunks {
  1152  		if x := p.chunks[i]; x != nil {
  1153  			p.chunks[i] = nil
  1154  			// This memory comes from sysAlloc and will always be page-aligned.
  1155  			sysFree(unsafe.Pointer(x), unsafe.Sizeof(*p.chunks[0]), testSysStat)
  1156  		}
  1157  	}
  1158  }
  1159  
  1160  // BaseChunkIdx is a convenient chunkIdx value which works on both
  1161  // 64 bit and 32 bit platforms, allowing the tests to share code
  1162  // between the two.
  1163  //
  1164  // This should not be higher than 0x100*pallocChunkBytes to support
  1165  // mips and mipsle, which only have 31-bit address spaces.
  1166  var BaseChunkIdx = func() ChunkIdx {
  1167  	var prefix uintptr
  1168  	if pageAlloc64Bit != 0 {
  1169  		prefix = 0xc000
  1170  	} else {
  1171  		prefix = 0x100
  1172  	}
  1173  	baseAddr := prefix * pallocChunkBytes
  1174  	if goos.IsAix != 0 {
  1175  		baseAddr += arenaBaseOffset
  1176  	}
  1177  	return ChunkIdx(chunkIndex(baseAddr))
  1178  }()
  1179  
  1180  // PageBase returns an address given a chunk index and a page index
  1181  // relative to that chunk.
  1182  func PageBase(c ChunkIdx, pageIdx uint) uintptr {
  1183  	return chunkBase(chunkIdx(c)) + uintptr(pageIdx)*pageSize
  1184  }
  1185  
  1186  type BitsMismatch struct {
  1187  	Base      uintptr
  1188  	Got, Want uint64
  1189  }
  1190  
  1191  func CheckScavengedBitsCleared(mismatches []BitsMismatch) (n int, ok bool) {
  1192  	ok = true
  1193  
  1194  	// Run on the system stack to avoid stack growth allocation.
  1195  	systemstack(func() {
  1196  		getg().m.mallocing++
  1197  
  1198  		// Lock so that we can safely access the bitmap.
  1199  		lock(&mheap_.lock)
  1200  
  1201  	chunkLoop:
  1202  		for i := mheap_.pages.start; i < mheap_.pages.end; i++ {
  1203  			chunk := mheap_.pages.tryChunkOf(i)
  1204  			if chunk == nil {
  1205  				continue
  1206  			}
  1207  			cb := chunkBase(i)
  1208  			for j := 0; j < pallocChunkPages/64; j++ {
  1209  				// Run over each 64-bit bitmap section and ensure
  1210  				// scavenged is being cleared properly on allocation.
  1211  				// If a used bit and scavenged bit are both set, that's
  1212  				// an error, and could indicate a larger problem, or
  1213  				// an accounting problem.
  1214  				want := chunk.scavenged[j] &^ chunk.pallocBits[j]
  1215  				got := chunk.scavenged[j]
  1216  				if want != got {
  1217  					ok = false
  1218  					if n >= len(mismatches) {
  1219  						break chunkLoop
  1220  					}
  1221  					mismatches[n] = BitsMismatch{
  1222  						Base: cb + uintptr(j)*64*pageSize,
  1223  						Got:  got,
  1224  						Want: want,
  1225  					}
  1226  					n++
  1227  				}
  1228  			}
  1229  		}
  1230  		unlock(&mheap_.lock)
  1231  
  1232  		getg().m.mallocing--
  1233  	})
  1234  
  1235  	if randomizeHeapBase && len(mismatches) > 0 {
  1236  		// When goexperiment.RandomizedHeapBase64 is set we use a series of
  1237  		// padding pages to generate randomized heap base address which have
  1238  		// both the alloc and scav bits set. Because of this we expect exactly
  1239  		// one arena will have mismatches, so check for that explicitly and
  1240  		// remove the mismatches if that property holds. If we see more than one
  1241  		// arena with this property, that is an indication something has
  1242  		// actually gone wrong, so return the mismatches.
  1243  		//
  1244  		// We do this, instead of ignoring the mismatches in the chunkLoop, because
  1245  		// it's not easy to determine which arena we added the padding pages to
  1246  		// programmatically, without explicitly recording the base address somewhere
  1247  		// in a global variable (which we'd rather not do as the address of that variable
  1248  		// is likely to be somewhat predictable, potentially defeating the purpose
  1249  		// of our randomization).
  1250  		affectedArenas := map[arenaIdx]bool{}
  1251  		for _, mismatch := range mismatches {
  1252  			if mismatch.Base > 0 {
  1253  				affectedArenas[arenaIndex(mismatch.Base)] = true
  1254  			}
  1255  		}
  1256  		if len(affectedArenas) == 1 {
  1257  			ok = true
  1258  			// zero the mismatches
  1259  			for i := range n {
  1260  				mismatches[i] = BitsMismatch{}
  1261  			}
  1262  		}
  1263  	}
  1264  
  1265  	return
  1266  }
  1267  
  1268  func PageCachePagesLeaked() (leaked uintptr) {
  1269  	stw := stopTheWorld(stwForTestPageCachePagesLeaked)
  1270  
  1271  	// Walk over destroyed Ps and look for unflushed caches.
  1272  	deadp := allp[len(allp):cap(allp)]
  1273  	for _, p := range deadp {
  1274  		// Since we're going past len(allp) we may see nil Ps.
  1275  		// Just ignore them.
  1276  		if p != nil {
  1277  			leaked += uintptr(sys.OnesCount64(p.pcache.cache))
  1278  		}
  1279  	}
  1280  
  1281  	startTheWorld(stw)
  1282  	return
  1283  }
  1284  
  1285  var ProcYield = procyield
  1286  var OSYield = osyield
  1287  
  1288  func StealRunnextSleep(ns int64) {
  1289  	systemstack(func() { stealRunnextSleep(ns) })
  1290  }
  1291  
  1292  type Note = note
  1293  
  1294  var NoteClear = noteclear
  1295  var NoteTSleepG = notetsleepg
  1296  
  1297  // NoteTSleepG0 calls notetsleep on the system stack
  1298  func NoteTSleepG0(n *Note, ns int64) bool {
  1299  	var ok bool
  1300  	systemstack(func() {
  1301  		ok = notetsleep(n, ns)
  1302  	})
  1303  	return ok
  1304  }
  1305  
  1306  type Mutex = mutex
  1307  
  1308  var Lock = lock
  1309  var Unlock = unlock
  1310  
  1311  var MutexContended = mutexContended
  1312  
  1313  func SemRootLock(addr *uint32) *mutex {
  1314  	root := semtable.rootFor(addr)
  1315  	return &root.lock
  1316  }
  1317  
  1318  var Semacquire = semacquire
  1319  var Semrelease1 = semrelease1
  1320  
  1321  func SemNwait(addr *uint32) uint32 {
  1322  	root := semtable.rootFor(addr)
  1323  	return root.nwait.Load()
  1324  }
  1325  
  1326  const SemTableSize = semTabSize
  1327  
  1328  // SemTable is a wrapper around semTable exported for testing.
  1329  type SemTable struct {
  1330  	semTable
  1331  }
  1332  
  1333  // Enqueue simulates enqueuing a waiter for a semaphore (or lock) at addr.
  1334  func (t *SemTable) Enqueue(addr *uint32) {
  1335  	s := acquireSudog()
  1336  	s.releasetime = 0
  1337  	s.acquiretime = 0
  1338  	s.ticket = 0
  1339  	t.semTable.rootFor(addr).queue(addr, s, false)
  1340  }
  1341  
  1342  // Dequeue simulates dequeuing a waiter for a semaphore (or lock) at addr.
  1343  //
  1344  // Returns true if there actually was a waiter to be dequeued.
  1345  func (t *SemTable) Dequeue(addr *uint32) bool {
  1346  	s, _, _ := t.semTable.rootFor(addr).dequeue(addr)
  1347  	if s != nil {
  1348  		releaseSudog(s)
  1349  		return true
  1350  	}
  1351  	return false
  1352  }
  1353  
  1354  // mspan wrapper for testing.
  1355  type MSpan mspan
  1356  
  1357  // Allocate an mspan for testing.
  1358  func AllocMSpan() *MSpan {
  1359  	var s *mspan
  1360  	systemstack(func() {
  1361  		lock(&mheap_.lock)
  1362  		s = (*mspan)(mheap_.spanalloc.alloc())
  1363  		s.init(0, 0)
  1364  		unlock(&mheap_.lock)
  1365  	})
  1366  	return (*MSpan)(s)
  1367  }
  1368  
  1369  // Free an allocated mspan.
  1370  func FreeMSpan(s *MSpan) {
  1371  	systemstack(func() {
  1372  		lock(&mheap_.lock)
  1373  		mheap_.spanalloc.free(unsafe.Pointer(s))
  1374  		unlock(&mheap_.lock)
  1375  	})
  1376  }
  1377  
  1378  func MSpanCountAlloc(ms *MSpan, bits []byte) int {
  1379  	s := (*mspan)(ms)
  1380  	s.nelems = uint16(len(bits) * 8)
  1381  	s.gcmarkBits = (*gcBits)(unsafe.Pointer(&bits[0]))
  1382  	result := s.countAlloc()
  1383  	s.gcmarkBits = nil
  1384  	return result
  1385  }
  1386  
  1387  const (
  1388  	TimeHistSubBucketBits = timeHistSubBucketBits
  1389  	TimeHistNumSubBuckets = timeHistNumSubBuckets
  1390  	TimeHistNumBuckets    = timeHistNumBuckets
  1391  	TimeHistMinBucketBits = timeHistMinBucketBits
  1392  	TimeHistMaxBucketBits = timeHistMaxBucketBits
  1393  )
  1394  
  1395  type TimeHistogram timeHistogram
  1396  
  1397  // Count returns the counts for the given bucket, subBucket indices.
  1398  // Returns true if the bucket was valid, otherwise returns the counts
  1399  // for the overflow bucket if bucket > 0 or the underflow bucket if
  1400  // bucket < 0, and false.
  1401  func (th *TimeHistogram) Count(bucket, subBucket int) (uint64, bool) {
  1402  	t := (*timeHistogram)(th)
  1403  	if bucket < 0 {
  1404  		return t.underflow.Load(), false
  1405  	}
  1406  	i := bucket*TimeHistNumSubBuckets + subBucket
  1407  	if i >= len(t.counts) {
  1408  		return t.overflow.Load(), false
  1409  	}
  1410  	return t.counts[i].Load(), true
  1411  }
  1412  
  1413  func (th *TimeHistogram) Record(duration int64) {
  1414  	(*timeHistogram)(th).record(duration)
  1415  }
  1416  
  1417  var TimeHistogramMetricsBuckets = timeHistogramMetricsBuckets
  1418  
  1419  func SetIntArgRegs(a int) int {
  1420  	lock(&finlock)
  1421  	old := intArgRegs
  1422  	if a >= 0 {
  1423  		intArgRegs = a
  1424  	}
  1425  	unlock(&finlock)
  1426  	return old
  1427  }
  1428  
  1429  func FinalizerGAsleep() bool {
  1430  	return fingStatus.Load()&fingWait != 0
  1431  }
  1432  
  1433  // For GCTestMoveStackOnNextCall, it's important not to introduce an
  1434  // extra layer of call, since then there's a return before the "real"
  1435  // next call.
  1436  var GCTestMoveStackOnNextCall = gcTestMoveStackOnNextCall
  1437  
  1438  // For GCTestIsReachable, it's important that we do this as a call so
  1439  // escape analysis can see through it.
  1440  func GCTestIsReachable(ptrs ...unsafe.Pointer) (mask uint64) {
  1441  	return gcTestIsReachable(ptrs...)
  1442  }
  1443  
  1444  // For GCTestPointerClass, it's important that we do this as a call so
  1445  // escape analysis can see through it.
  1446  //
  1447  // This is nosplit because gcTestPointerClass is.
  1448  //
  1449  //go:nosplit
  1450  func GCTestPointerClass(p unsafe.Pointer) string {
  1451  	return gcTestPointerClass(p)
  1452  }
  1453  
  1454  const Raceenabled = raceenabled
  1455  
  1456  const (
  1457  	GCBackgroundUtilization            = gcBackgroundUtilization
  1458  	GCGoalUtilization                  = gcGoalUtilization
  1459  	DefaultHeapMinimum                 = defaultHeapMinimum
  1460  	MemoryLimitHeapGoalHeadroomPercent = memoryLimitHeapGoalHeadroomPercent
  1461  	MemoryLimitMinHeapGoalHeadroom     = memoryLimitMinHeapGoalHeadroom
  1462  )
  1463  
  1464  type GCController struct {
  1465  	gcControllerState
  1466  }
  1467  
  1468  func NewGCController(gcPercent int, memoryLimit int64) *GCController {
  1469  	// Force the controller to escape. We're going to
  1470  	// do 64-bit atomics on it, and if it gets stack-allocated
  1471  	// on a 32-bit architecture, it may get allocated unaligned
  1472  	// space.
  1473  	g := Escape(new(GCController))
  1474  	g.gcControllerState.test = true // Mark it as a test copy.
  1475  	g.init(int32(gcPercent), memoryLimit)
  1476  	return g
  1477  }
  1478  
  1479  func (c *GCController) StartCycle(stackSize, globalsSize uint64, scannableFrac float64, gomaxprocs int) {
  1480  	trigger, _ := c.trigger()
  1481  	if c.heapMarked > trigger {
  1482  		trigger = c.heapMarked
  1483  	}
  1484  	c.maxStackScan.Store(stackSize)
  1485  	c.globalsScan.Store(globalsSize)
  1486  	c.heapLive.Store(trigger)
  1487  	c.heapScan.Add(int64(float64(trigger-c.heapMarked) * scannableFrac))
  1488  	c.startCycle(0, gomaxprocs, gcTrigger{kind: gcTriggerHeap})
  1489  }
  1490  
  1491  func (c *GCController) AssistWorkPerByte() float64 {
  1492  	return c.assistWorkPerByte.Load()
  1493  }
  1494  
  1495  func (c *GCController) HeapGoal() uint64 {
  1496  	return c.heapGoal()
  1497  }
  1498  
  1499  func (c *GCController) HeapLive() uint64 {
  1500  	return c.heapLive.Load()
  1501  }
  1502  
  1503  func (c *GCController) HeapMarked() uint64 {
  1504  	return c.heapMarked
  1505  }
  1506  
  1507  func (c *GCController) Triggered() uint64 {
  1508  	return c.triggered
  1509  }
  1510  
  1511  type GCControllerReviseDelta struct {
  1512  	HeapLive        int64
  1513  	HeapScan        int64
  1514  	HeapScanWork    int64
  1515  	StackScanWork   int64
  1516  	GlobalsScanWork int64
  1517  }
  1518  
  1519  func (c *GCController) Revise(d GCControllerReviseDelta) {
  1520  	c.heapLive.Add(d.HeapLive)
  1521  	c.heapScan.Add(d.HeapScan)
  1522  	c.heapScanWork.Add(d.HeapScanWork)
  1523  	c.stackScanWork.Add(d.StackScanWork)
  1524  	c.globalsScanWork.Add(d.GlobalsScanWork)
  1525  	c.revise()
  1526  }
  1527  
  1528  func (c *GCController) EndCycle(bytesMarked uint64, assistTime, elapsed int64, gomaxprocs int) {
  1529  	c.assistTime.Store(assistTime)
  1530  	c.endCycle(elapsed, gomaxprocs)
  1531  	c.resetLive(bytesMarked)
  1532  	c.commit(false)
  1533  }
  1534  
  1535  func (c *GCController) AddIdleMarkWorker() bool {
  1536  	return c.addIdleMarkWorker()
  1537  }
  1538  
  1539  func (c *GCController) NeedIdleMarkWorker() bool {
  1540  	return c.needIdleMarkWorker()
  1541  }
  1542  
  1543  func (c *GCController) RemoveIdleMarkWorker() {
  1544  	c.removeIdleMarkWorker()
  1545  }
  1546  
  1547  func (c *GCController) SetMaxIdleMarkWorkers(max int32) {
  1548  	c.setMaxIdleMarkWorkers(max)
  1549  }
  1550  
  1551  var alwaysFalse bool
  1552  var escapeSink any
  1553  
  1554  func Escape[T any](x T) T {
  1555  	if alwaysFalse {
  1556  		escapeSink = x
  1557  	}
  1558  	return x
  1559  }
  1560  
  1561  // Acquirem blocks preemption.
  1562  func Acquirem() {
  1563  	acquirem()
  1564  }
  1565  
  1566  func Releasem() {
  1567  	releasem(getg().m)
  1568  }
  1569  
  1570  // GoschedIfBusy is an explicit preemption check to call back
  1571  // into the scheduler. This is useful for tests that run code
  1572  // which spend most of their time as non-preemptible, as it
  1573  // can be placed right after becoming preemptible again to ensure
  1574  // that the scheduler gets a chance to preempt the goroutine.
  1575  func GoschedIfBusy() {
  1576  	goschedIfBusy()
  1577  }
  1578  
  1579  type PIController struct {
  1580  	piController
  1581  }
  1582  
  1583  func NewPIController(kp, ti, tt, min, max float64) *PIController {
  1584  	return &PIController{piController{
  1585  		kp:  kp,
  1586  		ti:  ti,
  1587  		tt:  tt,
  1588  		min: min,
  1589  		max: max,
  1590  	}}
  1591  }
  1592  
  1593  func (c *PIController) Next(input, setpoint, period float64) (float64, bool) {
  1594  	return c.piController.next(input, setpoint, period)
  1595  }
  1596  
  1597  const (
  1598  	CapacityPerProc          = capacityPerProc
  1599  	GCCPULimiterUpdatePeriod = gcCPULimiterUpdatePeriod
  1600  )
  1601  
  1602  type GCCPULimiter struct {
  1603  	limiter gcCPULimiterState
  1604  }
  1605  
  1606  func NewGCCPULimiter(now int64, gomaxprocs int32) *GCCPULimiter {
  1607  	// Force the controller to escape. We're going to
  1608  	// do 64-bit atomics on it, and if it gets stack-allocated
  1609  	// on a 32-bit architecture, it may get allocated unaligned
  1610  	// space.
  1611  	l := Escape(new(GCCPULimiter))
  1612  	l.limiter.test = true
  1613  	l.limiter.resetCapacity(now, gomaxprocs)
  1614  	return l
  1615  }
  1616  
  1617  func (l *GCCPULimiter) Fill() uint64 {
  1618  	return l.limiter.bucket.fill
  1619  }
  1620  
  1621  func (l *GCCPULimiter) Capacity() uint64 {
  1622  	return l.limiter.bucket.capacity
  1623  }
  1624  
  1625  func (l *GCCPULimiter) Overflow() uint64 {
  1626  	return l.limiter.overflow
  1627  }
  1628  
  1629  func (l *GCCPULimiter) Limiting() bool {
  1630  	return l.limiter.limiting()
  1631  }
  1632  
  1633  func (l *GCCPULimiter) NeedUpdate(now int64) bool {
  1634  	return l.limiter.needUpdate(now)
  1635  }
  1636  
  1637  func (l *GCCPULimiter) StartGCTransition(enableGC bool, now int64) {
  1638  	l.limiter.startGCTransition(enableGC, now)
  1639  }
  1640  
  1641  func (l *GCCPULimiter) FinishGCTransition(now int64) {
  1642  	l.limiter.finishGCTransition(now)
  1643  }
  1644  
  1645  func (l *GCCPULimiter) Update(now int64) {
  1646  	l.limiter.update(now)
  1647  }
  1648  
  1649  func (l *GCCPULimiter) AddAssistTime(t int64) {
  1650  	l.limiter.addAssistTime(t)
  1651  }
  1652  
  1653  func (l *GCCPULimiter) ResetCapacity(now int64, nprocs int32) {
  1654  	l.limiter.resetCapacity(now, nprocs)
  1655  }
  1656  
  1657  const ScavengePercent = scavengePercent
  1658  
  1659  type Scavenger struct {
  1660  	Sleep      func(int64) int64
  1661  	Scavenge   func(uintptr) (uintptr, int64)
  1662  	ShouldStop func() bool
  1663  	GoMaxProcs func() int32
  1664  
  1665  	released  atomic.Uintptr
  1666  	scavenger scavengerState
  1667  	stop      chan<- struct{}
  1668  	done      <-chan struct{}
  1669  }
  1670  
  1671  func (s *Scavenger) Start() {
  1672  	if s.Sleep == nil || s.Scavenge == nil || s.ShouldStop == nil || s.GoMaxProcs == nil {
  1673  		panic("must populate all stubs")
  1674  	}
  1675  
  1676  	// Install hooks.
  1677  	s.scavenger.sleepStub = s.Sleep
  1678  	s.scavenger.scavenge = s.Scavenge
  1679  	s.scavenger.shouldStop = s.ShouldStop
  1680  	s.scavenger.gomaxprocs = s.GoMaxProcs
  1681  
  1682  	// Start up scavenger goroutine, and wait for it to be ready.
  1683  	stop := make(chan struct{})
  1684  	s.stop = stop
  1685  	done := make(chan struct{})
  1686  	s.done = done
  1687  	go func() {
  1688  		// This should match bgscavenge, loosely.
  1689  		s.scavenger.init()
  1690  		s.scavenger.park()
  1691  		for {
  1692  			select {
  1693  			case <-stop:
  1694  				close(done)
  1695  				return
  1696  			default:
  1697  			}
  1698  			released, workTime := s.scavenger.run()
  1699  			if released == 0 {
  1700  				s.scavenger.park()
  1701  				continue
  1702  			}
  1703  			s.released.Add(released)
  1704  			s.scavenger.sleep(workTime)
  1705  		}
  1706  	}()
  1707  	if !s.BlockUntilParked(1e9 /* 1 second */) {
  1708  		panic("timed out waiting for scavenger to get ready")
  1709  	}
  1710  }
  1711  
  1712  // BlockUntilParked blocks until the scavenger parks, or until
  1713  // timeout is exceeded. Returns true if the scavenger parked.
  1714  //
  1715  // Note that in testing, parked means something slightly different.
  1716  // In anger, the scavenger parks to sleep, too, but in testing,
  1717  // it only parks when it actually has no work to do.
  1718  func (s *Scavenger) BlockUntilParked(timeout int64) bool {
  1719  	// Just spin, waiting for it to park.
  1720  	//
  1721  	// The actual parking process is racy with respect to
  1722  	// wakeups, which is fine, but for testing we need something
  1723  	// a bit more robust.
  1724  	start := nanotime()
  1725  	for nanotime()-start < timeout {
  1726  		lock(&s.scavenger.lock)
  1727  		parked := s.scavenger.parked
  1728  		unlock(&s.scavenger.lock)
  1729  		if parked {
  1730  			return true
  1731  		}
  1732  		Gosched()
  1733  	}
  1734  	return false
  1735  }
  1736  
  1737  // Released returns how many bytes the scavenger released.
  1738  func (s *Scavenger) Released() uintptr {
  1739  	return s.released.Load()
  1740  }
  1741  
  1742  // Wake wakes up a parked scavenger to keep running.
  1743  func (s *Scavenger) Wake() {
  1744  	s.scavenger.wake()
  1745  }
  1746  
  1747  // Stop cleans up the scavenger's resources. The scavenger
  1748  // must be parked for this to work.
  1749  func (s *Scavenger) Stop() {
  1750  	lock(&s.scavenger.lock)
  1751  	parked := s.scavenger.parked
  1752  	unlock(&s.scavenger.lock)
  1753  	if !parked {
  1754  		panic("tried to clean up scavenger that is not parked")
  1755  	}
  1756  	close(s.stop)
  1757  	s.Wake()
  1758  	<-s.done
  1759  }
  1760  
  1761  type ScavengeIndex struct {
  1762  	i scavengeIndex
  1763  }
  1764  
  1765  func NewScavengeIndex(min, max ChunkIdx) *ScavengeIndex {
  1766  	s := new(ScavengeIndex)
  1767  	// This is a bit lazy but we easily guarantee we'll be able
  1768  	// to reference all the relevant chunks. The worst-case
  1769  	// memory usage here is 512 MiB, but tests generally use
  1770  	// small offsets from BaseChunkIdx, which results in ~100s
  1771  	// of KiB in memory use.
  1772  	//
  1773  	// This may still be worth making better, at least by sharing
  1774  	// this fairly large array across calls with a sync.Pool or
  1775  	// something. Currently, when the tests are run serially,
  1776  	// it takes around 0.5s. Not all that much, but if we have
  1777  	// a lot of tests like this it could add up.
  1778  	s.i.chunks = make([]atomicScavChunkData, max)
  1779  	s.i.min.Store(uintptr(min))
  1780  	s.i.max.Store(uintptr(max))
  1781  	s.i.minHeapIdx.Store(uintptr(min))
  1782  	s.i.test = true
  1783  	return s
  1784  }
  1785  
  1786  func (s *ScavengeIndex) Find(force bool) (ChunkIdx, uint) {
  1787  	ci, off := s.i.find(force)
  1788  	return ChunkIdx(ci), off
  1789  }
  1790  
  1791  func (s *ScavengeIndex) AllocRange(base, limit uintptr) {
  1792  	sc, ec := chunkIndex(base), chunkIndex(limit-1)
  1793  	si, ei := chunkPageIndex(base), chunkPageIndex(limit-1)
  1794  
  1795  	if sc == ec {
  1796  		// The range doesn't cross any chunk boundaries.
  1797  		s.i.alloc(sc, ei+1-si)
  1798  	} else {
  1799  		// The range crosses at least one chunk boundary.
  1800  		s.i.alloc(sc, pallocChunkPages-si)
  1801  		for c := sc + 1; c < ec; c++ {
  1802  			s.i.alloc(c, pallocChunkPages)
  1803  		}
  1804  		s.i.alloc(ec, ei+1)
  1805  	}
  1806  }
  1807  
  1808  func (s *ScavengeIndex) FreeRange(base, limit uintptr) {
  1809  	sc, ec := chunkIndex(base), chunkIndex(limit-1)
  1810  	si, ei := chunkPageIndex(base), chunkPageIndex(limit-1)
  1811  
  1812  	if sc == ec {
  1813  		// The range doesn't cross any chunk boundaries.
  1814  		s.i.free(sc, si, ei+1-si)
  1815  	} else {
  1816  		// The range crosses at least one chunk boundary.
  1817  		s.i.free(sc, si, pallocChunkPages-si)
  1818  		for c := sc + 1; c < ec; c++ {
  1819  			s.i.free(c, 0, pallocChunkPages)
  1820  		}
  1821  		s.i.free(ec, 0, ei+1)
  1822  	}
  1823  }
  1824  
  1825  func (s *ScavengeIndex) ResetSearchAddrs() {
  1826  	for _, a := range []*atomicOffAddr{&s.i.searchAddrBg, &s.i.searchAddrForce} {
  1827  		addr, marked := a.Load()
  1828  		if marked {
  1829  			a.StoreUnmark(addr, addr)
  1830  		}
  1831  		a.Clear()
  1832  	}
  1833  	s.i.freeHWM = minOffAddr
  1834  }
  1835  
  1836  func (s *ScavengeIndex) NextGen() {
  1837  	s.i.nextGen()
  1838  }
  1839  
  1840  func (s *ScavengeIndex) SetEmpty(ci ChunkIdx) {
  1841  	s.i.setEmpty(chunkIdx(ci))
  1842  }
  1843  
  1844  func CheckPackScavChunkData(gen uint32, inUse, lastInUse uint16, flags uint8) bool {
  1845  	sc0 := scavChunkData{
  1846  		gen:            gen,
  1847  		inUse:          inUse,
  1848  		lastInUse:      lastInUse,
  1849  		scavChunkFlags: scavChunkFlags(flags),
  1850  	}
  1851  	scp := sc0.pack()
  1852  	sc1 := unpackScavChunkData(scp)
  1853  	return sc0 == sc1
  1854  }
  1855  
  1856  const GTrackingPeriod = gTrackingPeriod
  1857  
  1858  var ZeroBase = unsafe.Pointer(&zerobase)
  1859  
  1860  const UserArenaChunkBytes = userArenaChunkBytes
  1861  
  1862  type UserArena struct {
  1863  	arena *userArena
  1864  }
  1865  
  1866  func NewUserArena() *UserArena {
  1867  	return &UserArena{newUserArena()}
  1868  }
  1869  
  1870  func (a *UserArena) New(out *any) {
  1871  	i := efaceOf(out)
  1872  	typ := i._type
  1873  	if typ.Kind() != abi.Pointer {
  1874  		panic("new result of non-ptr type")
  1875  	}
  1876  	typ = (*ptrtype)(unsafe.Pointer(typ)).Elem
  1877  	i.data = a.arena.new(typ)
  1878  }
  1879  
  1880  func (a *UserArena) Slice(sl any, cap int) {
  1881  	a.arena.slice(sl, cap)
  1882  }
  1883  
  1884  func (a *UserArena) Free() {
  1885  	a.arena.free()
  1886  }
  1887  
  1888  func GlobalWaitingArenaChunks() int {
  1889  	n := 0
  1890  	systemstack(func() {
  1891  		lock(&mheap_.lock)
  1892  		for s := mheap_.userArena.quarantineList.first; s != nil; s = s.next {
  1893  			n++
  1894  		}
  1895  		unlock(&mheap_.lock)
  1896  	})
  1897  	return n
  1898  }
  1899  
  1900  func UserArenaClone[T any](s T) T {
  1901  	return arena_heapify(s).(T)
  1902  }
  1903  
  1904  var AlignUp = alignUp
  1905  
  1906  func BlockUntilEmptyFinalizerQueue(timeout int64) bool {
  1907  	return blockUntilEmptyFinalizerQueue(timeout)
  1908  }
  1909  
  1910  func BlockUntilEmptyCleanupQueue(timeout int64) bool {
  1911  	return gcCleanups.blockUntilEmpty(timeout)
  1912  }
  1913  
  1914  func FrameStartLine(f *Frame) int {
  1915  	return f.startLine
  1916  }
  1917  
  1918  // PersistentAlloc allocates some memory that lives outside the Go heap.
  1919  // This memory will never be freed; use sparingly.
  1920  func PersistentAlloc(n, align uintptr) unsafe.Pointer {
  1921  	return persistentalloc(n, align, &memstats.other_sys)
  1922  }
  1923  
  1924  const TagAlign = tagAlign
  1925  
  1926  // FPCallers works like Callers and uses frame pointer unwinding to populate
  1927  // pcBuf with the return addresses of the physical frames on the stack.
  1928  func FPCallers(pcBuf []uintptr) int {
  1929  	return fpTracebackPCs(unsafe.Pointer(getfp()), pcBuf)
  1930  }
  1931  
  1932  const FramePointerEnabled = framepointer_enabled
  1933  
  1934  var (
  1935  	IsPinned      = isPinned
  1936  	GetPinCounter = pinnerGetPinCounter
  1937  )
  1938  
  1939  func SetPinnerLeakPanic(f func()) {
  1940  	pinnerLeakPanic = f
  1941  }
  1942  func GetPinnerLeakPanic() func() {
  1943  	return pinnerLeakPanic
  1944  }
  1945  
  1946  var testUintptr uintptr
  1947  
  1948  func MyGenericFunc[T any]() {
  1949  	systemstack(func() {
  1950  		testUintptr = 4
  1951  	})
  1952  }
  1953  
  1954  func UnsafePoint(pc uintptr) bool {
  1955  	fi := findfunc(pc)
  1956  	v := pcdatavalue(fi, abi.PCDATA_UnsafePoint, pc)
  1957  	switch v {
  1958  	case abi.UnsafePointUnsafe:
  1959  		return true
  1960  	case abi.UnsafePointSafe:
  1961  		return false
  1962  	case abi.UnsafePointRestart1, abi.UnsafePointRestart2, abi.UnsafePointRestartAtEntry:
  1963  		// These are all interruptible, they just encode a nonstandard
  1964  		// way of recovering when interrupted.
  1965  		return false
  1966  	default:
  1967  		var buf [20]byte
  1968  		panic("invalid unsafe point code " + string(itoa(buf[:], uint64(v))))
  1969  	}
  1970  }
  1971  
  1972  type TraceMap struct {
  1973  	traceMap
  1974  }
  1975  
  1976  func (m *TraceMap) PutString(s string) (uint64, bool) {
  1977  	return m.traceMap.put(unsafe.Pointer(unsafe.StringData(s)), uintptr(len(s)))
  1978  }
  1979  
  1980  func (m *TraceMap) Reset() {
  1981  	m.traceMap.reset()
  1982  }
  1983  
  1984  func SetSpinInGCMarkDone(spin bool) {
  1985  	gcDebugMarkDone.spinAfterRaggedBarrier.Store(spin)
  1986  }
  1987  
  1988  func GCMarkDoneRestarted() bool {
  1989  	// Only read this outside of the GC. If we're running during a GC, just report false.
  1990  	mp := acquirem()
  1991  	if gcphase != _GCoff {
  1992  		releasem(mp)
  1993  		return false
  1994  	}
  1995  	restarted := gcDebugMarkDone.restartedDueTo27993
  1996  	releasem(mp)
  1997  	return restarted
  1998  }
  1999  
  2000  func GCMarkDoneResetRestartFlag() {
  2001  	mp := acquirem()
  2002  	for gcphase != _GCoff {
  2003  		releasem(mp)
  2004  		Gosched()
  2005  		mp = acquirem()
  2006  	}
  2007  	gcDebugMarkDone.restartedDueTo27993 = false
  2008  	releasem(mp)
  2009  }
  2010  
  2011  type BitCursor struct {
  2012  	b bitCursor
  2013  }
  2014  
  2015  func NewBitCursor(buf *byte) BitCursor {
  2016  	return BitCursor{b: bitCursor{ptr: buf, n: 0}}
  2017  }
  2018  
  2019  func (b BitCursor) Write(data *byte, cnt uintptr) {
  2020  	b.b.write(data, cnt)
  2021  }
  2022  func (b BitCursor) Offset(cnt uintptr) BitCursor {
  2023  	return BitCursor{b: b.b.offset(cnt)}
  2024  }
  2025  
  2026  const (
  2027  	BubbleAssocUnbubbled     = bubbleAssocUnbubbled
  2028  	BubbleAssocCurrentBubble = bubbleAssocCurrentBubble
  2029  	BubbleAssocOtherBubble   = bubbleAssocOtherBubble
  2030  )
  2031  
  2032  type TraceStackTable traceStackTable
  2033  
  2034  func (t *TraceStackTable) Reset() {
  2035  	t.tab.reset()
  2036  }
  2037  
  2038  func TraceStack(gp *G, tab *TraceStackTable) {
  2039  	traceStack(0, gp, (*traceStackTable)(tab))
  2040  }
  2041  
  2042  var X86HasAVX = &x86HasAVX
  2043  
  2044  var DebugDecorateMappings = &debug.decoratemappings
  2045  
  2046  func SetVMANameSupported() bool { return setVMANameSupported() }
  2047  
  2048  type ListHead struct {
  2049  	l listHead
  2050  }
  2051  
  2052  func (head *ListHead) Init(off uintptr) {
  2053  	head.l.init(off)
  2054  }
  2055  
  2056  type ListNode struct {
  2057  	l listNode
  2058  }
  2059  
  2060  func (head *ListHead) Push(p unsafe.Pointer) {
  2061  	head.l.push(p)
  2062  }
  2063  
  2064  func (head *ListHead) Pop() unsafe.Pointer {
  2065  	return head.l.pop()
  2066  }
  2067  
  2068  func (head *ListHead) Remove(p unsafe.Pointer) {
  2069  	head.l.remove(p)
  2070  }
  2071  
  2072  type ListHeadManual struct {
  2073  	l listHeadManual
  2074  }
  2075  
  2076  func (head *ListHeadManual) Init(off uintptr) {
  2077  	head.l.init(off)
  2078  }
  2079  
  2080  type ListNodeManual struct {
  2081  	l listNodeManual
  2082  }
  2083  
  2084  func (head *ListHeadManual) Push(p unsafe.Pointer) {
  2085  	head.l.push(p)
  2086  }
  2087  
  2088  func (head *ListHeadManual) Pop() unsafe.Pointer {
  2089  	return head.l.pop()
  2090  }
  2091  
  2092  func (head *ListHeadManual) Remove(p unsafe.Pointer) {
  2093  	head.l.remove(p)
  2094  }
  2095  
  2096  func Hexdumper(base uintptr, wordBytes int, mark func(addr uintptr, start func()), data ...[]byte) string {
  2097  	buf := make([]byte, 0, 2048)
  2098  	getg().writebuf = buf
  2099  	h := hexdumper{addr: base, addrBytes: 4, wordBytes: uint8(wordBytes)}
  2100  	if mark != nil {
  2101  		h.mark = func(addr uintptr, m hexdumpMarker) {
  2102  			mark(addr, m.start)
  2103  		}
  2104  	}
  2105  	for _, d := range data {
  2106  		h.write(d)
  2107  	}
  2108  	h.close()
  2109  	n := len(getg().writebuf)
  2110  	getg().writebuf = nil
  2111  	if n == cap(buf) {
  2112  		panic("Hexdumper buf too small")
  2113  	}
  2114  	return string(buf[:n])
  2115  }
  2116  
  2117  func HexdumpWords(p, bytes uintptr) string {
  2118  	buf := make([]byte, 0, 2048)
  2119  	getg().writebuf = buf
  2120  	hexdumpWords(p, bytes, nil)
  2121  	n := len(getg().writebuf)
  2122  	getg().writebuf = nil
  2123  	if n == cap(buf) {
  2124  		panic("HexdumpWords buf too small")
  2125  	}
  2126  	return string(buf[:n])
  2127  }
  2128  
  2129  // DumpPrintQuoted provides access to print(quoted()) for the tests in
  2130  // runtime/print_quoted_test.go, allowing us to test that implementation.
  2131  func DumpPrintQuoted(s string) string {
  2132  	gp := getg()
  2133  	gp.writebuf = make([]byte, 0, 1<<20)
  2134  	print(quoted(s))
  2135  	buf := gp.writebuf
  2136  	gp.writebuf = nil
  2137  
  2138  	return string(buf)
  2139  }
  2140  
  2141  // PrintBacklog returns a copy of the runtime's print backlog.
  2142  func PrintBacklog() []byte {
  2143  	b := make([]byte, len(printBacklog))
  2144  	printlock()
  2145  	copy(b, printBacklog[:])
  2146  	printunlock()
  2147  	return b
  2148  }
  2149  
  2150  // DumpPrint returns the output of print(v).
  2151  func DumpPrint[T any](v T) string {
  2152  	gp := getg()
  2153  	gp.writebuf = make([]byte, 0, 2048)
  2154  	print(v)
  2155  	buf := gp.writebuf
  2156  	gp.writebuf = nil
  2157  
  2158  	return string(buf)
  2159  }
  2160  
  2161  var (
  2162  	Float64Bytes    = float64Bytes
  2163  	Float32Bytes    = float32Bytes
  2164  	Complex128Bytes = complex128Bytes
  2165  	Complex64Bytes  = complex64Bytes
  2166  )
  2167  
  2168  func GetScanAlloc() uintptr {
  2169  	c := getMCache(getg().m)
  2170  	return c.scanAlloc
  2171  }
  2172  
  2173  func MallocGC(size uintptr, typ *abi.Type, needzero bool) unsafe.Pointer {
  2174  	return mallocgc(size, typ, needzero)
  2175  }
  2176  
  2177  // AllocSize returns how many bytes mallocgc uses for an object of the given
  2178  // size of a type that contains pointers: the size of its size class,
  2179  // including the malloc header if the object has one. A large object has no
  2180  // size class, and the result is its size rounded up to a page.
  2181  func AllocSize(size uintptr) uintptr {
  2182  	if size <= maxSmallSize-gc.MallocHeaderSize && !heapBitsInSpan(size) {
  2183  		// roundupsize subtracts the malloc header from the size class.
  2184  		return roundupsize(size, false) + gc.MallocHeaderSize
  2185  	}
  2186  	return roundupsize(size, false)
  2187  }
  2188  
  2189  func FuncNamePiecesForPrint(name string) (string, string, string, string, string) {
  2190  	return funcNamePiecesForPrint(name)
  2191  }
  2192  
  2193  var InHeapOrStack = inHeapOrStack
  2194
View as plain text