traceregion.go

     1  // Copyright 2023 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Simple not-in-heap bump-pointer traceRegion allocator.
     6  
     7  package runtime
     8  
     9  import (
    10  	"internal/runtime/atomic"
    11  	"internal/runtime/sys"
    12  	"unsafe"
    13  )
    14  
    15  // traceRegionAlloc is a thread-safe region allocator.
    16  // It holds a linked list of traceRegionAllocBlock.
    17  type traceRegionAlloc struct {
    18  	lock     mutex
    19  	dropping atomic.Bool          // For checking invariants.
    20  	current  atomic.UnsafePointer // *traceRegionAllocBlock
    21  	full     *traceRegionAllocBlock
    22  }
    23  
    24  // traceRegionAllocBlock is a block in traceRegionAlloc.
    25  //
    26  // traceRegionAllocBlock is allocated from non-GC'd memory, so it must not
    27  // contain heap pointers. Writes to pointers to traceRegionAllocBlocks do
    28  // not need write barriers.
    29  type traceRegionAllocBlock struct {
    30  	_ sys.NotInHeap
    31  	traceRegionAllocBlockHeader
    32  	data [traceRegionAllocBlockData]byte
    33  }
    34  
    35  type traceRegionAllocBlockHeader struct {
    36  	next *traceRegionAllocBlock
    37  	off  atomic.Uintptr
    38  }
    39  
    40  const traceRegionAllocBlockData = 64<<10 - unsafe.Sizeof(traceRegionAllocBlockHeader{})
    41  
    42  // alloc allocates n-byte block. The block is always aligned to 8 bytes, regardless of platform.
    43  func (a *traceRegionAlloc) alloc(n uintptr) *notInHeap {
    44  	n = alignUp(n, 8)
    45  	if n > traceRegionAllocBlockData {
    46  		throw("traceRegion: alloc too large")
    47  	}
    48  	if a.dropping.Load() {
    49  		throw("traceRegion: alloc with concurrent drop")
    50  	}
    51  
    52  	// Try to bump-pointer allocate into the current block.
    53  	block := (*traceRegionAllocBlock)(a.current.Load())
    54  	if block != nil {
    55  		r := block.off.Add(n)
    56  		if r <= uintptr(len(block.data)) {
    57  			return (*notInHeap)(unsafe.Pointer(&block.data[r-n]))
    58  		}
    59  	}
    60  
    61  	// Try to install a new block.
    62  	var x *notInHeap
    63  	systemstack(func() {
    64  		// Acquire a.lock on the systemstack to avoid stack growth
    65  		// and accidentally entering the tracer again.
    66  		lock(&a.lock)
    67  
    68  		// Check block again under the lock. Someone may
    69  		// have gotten here first.
    70  		block = (*traceRegionAllocBlock)(a.current.Load())
    71  		if block != nil {
    72  			r := block.off.Add(n)
    73  			if r <= uintptr(len(block.data)) {
    74  				unlock(&a.lock)
    75  				x = (*notInHeap)(unsafe.Pointer(&block.data[r-n]))
    76  				return
    77  			}
    78  
    79  			// Add the existing block to the full list.
    80  			block.next = a.full
    81  			a.full = block
    82  		}
    83  
    84  		// Allocate a new block.
    85  		block = (*traceRegionAllocBlock)(sysAlloc(unsafe.Sizeof(traceRegionAllocBlock{}), &memstats.other_sys, "trace arena alloc"))
    86  		if block == nil {
    87  			throw("traceRegion: out of memory")
    88  		}
    89  
    90  		// Allocate space for our current request, so we always make
    91  		// progress.
    92  		block.off.Store(n)
    93  		x = (*notInHeap)(unsafe.Pointer(&block.data[0]))
    94  
    95  		// Publish the new block.
    96  		a.current.Store(unsafe.Pointer(block))
    97  		unlock(&a.lock)
    98  	})
    99  	return x
   100  }
   101  
   102  // drop frees all previously allocated memory and resets the allocator.
   103  //
   104  // drop is not safe to call concurrently with other calls to drop or with calls to alloc. The caller
   105  // must ensure that it is not possible for anything else to be using the same structure.
   106  func (a *traceRegionAlloc) drop() {
   107  	a.dropping.Store(true)
   108  	for a.full != nil {
   109  		block := a.full
   110  		a.full = block.next
   111  		sysFree(unsafe.Pointer(block), unsafe.Sizeof(traceRegionAllocBlock{}), &memstats.other_sys)
   112  	}
   113  	if current := a.current.Load(); current != nil {
   114  		sysFree(current, unsafe.Sizeof(traceRegionAllocBlock{}), &memstats.other_sys)
   115  		a.current.Store(nil)
   116  	}
   117  	a.dropping.Store(false)
   118  }
   119
View as plain text