pinner.go

     1  // Copyright 2023 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/abi"
     9  	"internal/runtime/atomic"
    10  	"unsafe"
    11  )
    12  
    13  // A Pinner is a set of Go objects each pinned to a fixed location in memory. The
    14  // [Pinner.Pin] method pins one object, while [Pinner.Unpin] unpins all pinned
    15  // objects.
    16  //
    17  // The purpose of a Pinner is two-fold.
    18  // First, it allows C code to safely use Go pointers that have not been passed
    19  // explicitly to the C code via a cgo call.
    20  // For example, for safely interacting with a pointer stored inside of a struct
    21  // whose pointer is passed to a C function.
    22  // Second, it allows C memory to safely retain that Go pointer even after the
    23  // cgo call returns, provided the object remains pinned.
    24  //
    25  // A Pinner arranges for its objects to be automatically unpinned some time after
    26  // it becomes unreachable, so its referents will not leak. However, this means the
    27  // Pinner itself must be kept alive across a cgo call, or as long as C retains a
    28  // reference to the pinned Go pointers.
    29  //
    30  // Reusing a Pinner is safe, and in fact encouraged, to avoid the cost of
    31  // initializing new Pinners on first use.
    32  //
    33  // The zero value of Pinner is ready to use.
    34  type Pinner struct {
    35  	*pinner
    36  }
    37  
    38  // Pin pins a Go object, preventing it from being moved or freed by the garbage
    39  // collector until the [Pinner.Unpin] method has been called.
    40  //
    41  // A pointer to a pinned object can be directly stored in C memory or can be
    42  // contained in Go memory passed to C functions. If the pinned object itself
    43  // contains pointers to Go objects, these objects must be pinned separately if they
    44  // are going to be accessed from C code.
    45  //
    46  // The argument must be a pointer of any type or an [unsafe.Pointer].
    47  //
    48  // It's safe to call Pin on non-Go pointers, in which case Pin will do nothing.
    49  func (p *Pinner) Pin(pointer any) {
    50  	if p.pinner == nil {
    51  		// Check the pinner cache first.
    52  		mp := acquirem()
    53  		if pp := mp.p.ptr(); pp != nil {
    54  			p.pinner = pp.pinnerCache
    55  			pp.pinnerCache = nil
    56  		}
    57  		releasem(mp)
    58  
    59  		if p.pinner == nil {
    60  			// Didn't get anything from the pinner cache.
    61  			p.pinner = new(pinner)
    62  			p.refs = p.refStore[:0]
    63  
    64  			// We set this finalizer once and never clear it. Thus, if the
    65  			// pinner gets cached, we'll reuse it, along with its finalizer.
    66  			// This lets us avoid the relatively expensive SetFinalizer call
    67  			// when reusing from the cache. The finalizer however has to be
    68  			// resilient to an empty pinner being finalized, which is done
    69  			// by checking p.refs' length.
    70  			SetFinalizer(p.pinner, func(i *pinner) {
    71  				if len(i.refs) != 0 {
    72  					i.unpin() // only required to make the test idempotent
    73  					pinnerLeakPanic()
    74  				}
    75  			})
    76  		}
    77  	}
    78  	ptr := pinnerGetPtr(&pointer)
    79  	if setPinned(ptr) {
    80  		p.refs = append(p.refs, ptr)
    81  	}
    82  }
    83  
    84  // Unpin unpins all pinned objects of the [Pinner].
    85  // It's safe and encouraged to reuse a Pinner after calling Unpin.
    86  func (p *Pinner) Unpin() {
    87  	p.pinner.unpin()
    88  
    89  	mp := acquirem()
    90  	if pp := mp.p.ptr(); pp != nil && pp.pinnerCache == nil {
    91  		// Put the pinner back in the cache, but only if the
    92  		// cache is empty. If application code is reusing Pinners
    93  		// on its own, we want to leave the backing store in place
    94  		// so reuse is more efficient.
    95  		pp.pinnerCache = p.pinner
    96  		p.pinner = nil
    97  	}
    98  	releasem(mp)
    99  }
   100  
   101  const (
   102  	pinnerSize         = 64
   103  	pinnerRefStoreSize = (pinnerSize - unsafe.Sizeof([]unsafe.Pointer{})) / unsafe.Sizeof(unsafe.Pointer(nil))
   104  )
   105  
   106  type pinner struct {
   107  	refs     []unsafe.Pointer
   108  	refStore [pinnerRefStoreSize]unsafe.Pointer
   109  }
   110  
   111  func (p *pinner) unpin() {
   112  	if p == nil || p.refs == nil {
   113  		return
   114  	}
   115  	for refs := p.refs; len(refs) != 0; {
   116  		refs = refs[unpinObjects(refs):]
   117  	}
   118  	// The following two lines make all pointers to references
   119  	// in p.refs unreachable, either by deleting them or dropping
   120  	// p.refs' backing store (if it was not backed by refStore).
   121  	p.refStore = [pinnerRefStoreSize]unsafe.Pointer{}
   122  	p.refs = p.refStore[:0]
   123  }
   124  
   125  func pinnerGetPtr(i *any) unsafe.Pointer {
   126  	e := efaceOf(i)
   127  	etyp := e._type
   128  	if etyp == nil {
   129  		panic(errorString("runtime.Pinner: argument is nil"))
   130  	}
   131  	if kind := etyp.Kind(); kind != abi.Pointer && kind != abi.UnsafePointer {
   132  		panic(errorString("runtime.Pinner: argument is not a pointer: " + toRType(etyp).string()))
   133  	}
   134  	if inUserArenaChunk(uintptr(e.data)) {
   135  		// Arena-allocated objects are not eligible for pinning.
   136  		panic(errorString("runtime.Pinner: object was allocated into an arena"))
   137  	}
   138  	return e.data
   139  }
   140  
   141  // isPinned checks if a Go pointer is pinned.
   142  // nosplit, because it's called from nosplit code in cgocheck.
   143  //
   144  //go:nosplit
   145  func isPinned(ptr unsafe.Pointer) bool {
   146  	span := spanOfHeap(uintptr(ptr))
   147  	if span == nil {
   148  		// this code is only called for Go pointer, so this must be a
   149  		// linker-allocated global object.
   150  		return true
   151  	}
   152  	pinnerBits := span.getPinnerBits()
   153  	// these pinnerBits might get unlinked by a concurrently running sweep, but
   154  	// that's OK because gcBits don't get cleared until the following GC cycle
   155  	// (nextMarkBitArenaEpoch)
   156  	if pinnerBits == nil {
   157  		return false
   158  	}
   159  	objIndex := span.objIndex(uintptr(ptr))
   160  	pinState := pinnerBits.ofObject(objIndex)
   161  	KeepAlive(ptr) // make sure ptr is alive until we are done so the span can't be freed
   162  	return pinState.isPinned()
   163  }
   164  
   165  // setPinned marks a Go pointer as pinned. It returns false for non-Go pointers.
   166  func setPinned(ptr unsafe.Pointer) bool {
   167  	span := spanOfHeap(uintptr(ptr))
   168  	if span == nil {
   169  		// This is a linker-allocated, zero size object or other object,
   170  		// nothing to do, silently ignore it.
   171  		return false
   172  	}
   173  
   174  	// ensure that the span is swept, b/c sweeping accesses the specials list
   175  	// w/o locks.
   176  	mp := acquirem()
   177  	span.ensureSwept()
   178  	KeepAlive(ptr) // make sure ptr is still alive after span is swept
   179  
   180  	objIndex := span.objIndex(uintptr(ptr))
   181  
   182  	lock(&span.speciallock) // guard against concurrent pin and unpin operations
   183  
   184  	pinnerBits := span.getPinnerBits()
   185  	if pinnerBits == nil {
   186  		pinnerBits = span.newPinnerBits()
   187  		span.setPinnerBits(pinnerBits)
   188  	}
   189  	pinState := pinnerBits.ofObject(objIndex)
   190  	if pinState.isPinned() {
   191  		// multiple pins on same object, set multipin bit
   192  		pinState.setMultiPinned(true)
   193  		// and increase the pin counter
   194  		offset := objIndex * span.elemsize
   195  		span.incPinCounter(offset)
   196  	} else {
   197  		// set pin bit
   198  		pinState.setPinned(true)
   199  	}
   200  	unlock(&span.speciallock)
   201  	releasem(mp)
   202  	return true
   203  }
   204  
   205  // unpinObjects unpins a bounded prefix of refs in the same span and returns
   206  // the number of references processed. refs must not be empty.
   207  func unpinObjects(refs []unsafe.Pointer) int {
   208  	span := spanOfHeap(uintptr(refs[0]))
   209  	if span == nil {
   210  		panic(errorString("tried to unpin non-Go pointer"))
   211  	}
   212  	mp := acquirem()
   213  	span.ensureSwept()
   214  	lock(&span.speciallock)
   215  	pinnerBits := span.getPinnerBits()
   216  	if pinnerBits == nil {
   217  		throw("runtime.Pinner: object already unpinned")
   218  	}
   219  
   220  	// Bound the work done with preemption disabled, even for repeated pins
   221  	// of the same object. The caller can be preempted between batches.
   222  	const maxBatch = 64
   223  	base, limit := span.base(), span.limit
   224  	n := 0
   225  	for n < min(len(refs), maxBatch) {
   226  		ptr := uintptr(refs[n])
   227  		if ptr < base || ptr >= limit {
   228  			break
   229  		}
   230  		objIndex := span.objIndex(ptr)
   231  		// Reload the byte for each reference: previous updates may have
   232  		// changed another object's bits in the same byte.
   233  		pinState := pinnerBits.ofObject(objIndex)
   234  		if pinState.isPinned() {
   235  			if pinState.isMultiPinned() {
   236  				offset := objIndex * span.elemsize
   237  				if !span.decPinCounter(offset) {
   238  					// counter is 0, clear multipin bit
   239  					pinState.setMultiPinned(false)
   240  				}
   241  			} else {
   242  				// no multipins recorded. unpin object.
   243  				pinState.setPinned(false)
   244  			}
   245  		} else {
   246  			// unpinning unpinned object, bail out
   247  			throw("runtime.Pinner: object already unpinned")
   248  		}
   249  		n++
   250  	}
   251  	unlock(&span.speciallock)
   252  	releasem(mp)
   253  	KeepAlive(refs)
   254  	return n
   255  }
   256  
   257  type pinState struct {
   258  	bytep   *uint8
   259  	byteVal uint8
   260  	mask    uint8
   261  }
   262  
   263  // nosplit, because it's called by isPinned, which is nosplit
   264  //
   265  //go:nosplit
   266  func (v *pinState) isPinned() bool {
   267  	return (v.byteVal & v.mask) != 0
   268  }
   269  
   270  func (v *pinState) isMultiPinned() bool {
   271  	return (v.byteVal & (v.mask << 1)) != 0
   272  }
   273  
   274  func (v *pinState) setPinned(val bool) {
   275  	v.set(val, false)
   276  }
   277  
   278  func (v *pinState) setMultiPinned(val bool) {
   279  	v.set(val, true)
   280  }
   281  
   282  // set sets the pin bit of the pinState to val. If multipin is true, it
   283  // sets/unsets the multipin bit instead.
   284  //
   285  // The caller must hold the span's speciallock from the call to ofObject
   286  // through the call to set, since set updates the whole byte.
   287  func (v *pinState) set(val bool, multipin bool) {
   288  	mask := v.mask
   289  	if multipin {
   290  		mask <<= 1
   291  	}
   292  	if (v.byteVal&mask != 0) == val {
   293  		return
   294  	}
   295  	if val {
   296  		v.byteVal |= mask
   297  	} else {
   298  		v.byteVal &^= mask
   299  	}
   300  	atomic.Store8(v.bytep, v.byteVal)
   301  }
   302  
   303  // pinnerBits is the same type as gcBits but has different methods.
   304  type pinnerBits gcBits
   305  
   306  // ofObject returns the pinState of the n'th object.
   307  // nosplit, because it's called by isPinned, which is nosplit
   308  //
   309  //go:nosplit
   310  func (p *pinnerBits) ofObject(n uintptr) pinState {
   311  	bytep, mask := (*gcBits)(p).bitp(n * 2)
   312  	byteVal := atomic.Load8(bytep)
   313  	return pinState{bytep, byteVal, mask}
   314  }
   315  
   316  func (s *mspan) pinnerBitSize() uintptr {
   317  	return divRoundUp(uintptr(s.nelems)*2, 8)
   318  }
   319  
   320  // newPinnerBits returns a pointer to 8 byte aligned bytes to be used for this
   321  // span's pinner bits. newPinnerBits is used to mark objects that are pinned.
   322  // They are copied when the span is swept.
   323  func (s *mspan) newPinnerBits() *pinnerBits {
   324  	return (*pinnerBits)(newMarkBits(uintptr(s.nelems) * 2))
   325  }
   326  
   327  // nosplit, because it's called by isPinned, which is nosplit
   328  //
   329  //go:nosplit
   330  func (s *mspan) getPinnerBits() *pinnerBits {
   331  	return (*pinnerBits)(atomic.Loadp(unsafe.Pointer(&s.pinnerBits)))
   332  }
   333  
   334  func (s *mspan) setPinnerBits(p *pinnerBits) {
   335  	atomicstorep(unsafe.Pointer(&s.pinnerBits), unsafe.Pointer(p))
   336  }
   337  
   338  // refreshPinnerBits replaces pinnerBits with a fresh copy in the arenas for the
   339  // next GC cycle. If it does not contain any pinned objects, pinnerBits of the
   340  // span is set to nil.
   341  func (s *mspan) refreshPinnerBits() {
   342  	p := s.getPinnerBits()
   343  	if p == nil {
   344  		return
   345  	}
   346  
   347  	hasPins := false
   348  	bytes := alignUp(s.pinnerBitSize(), 8)
   349  
   350  	// Iterate over each 8-byte chunk and check for pins. Note that
   351  	// newPinnerBits guarantees that pinnerBits will be 8-byte aligned, so we
   352  	// don't have to worry about edge cases, irrelevant bits will simply be
   353  	// zero.
   354  	for _, x := range unsafe.Slice((*uint64)(unsafe.Pointer(&p.x)), bytes/8) {
   355  		if x != 0 {
   356  			hasPins = true
   357  			break
   358  		}
   359  	}
   360  
   361  	if hasPins {
   362  		newPinnerBits := s.newPinnerBits()
   363  		memmove(unsafe.Pointer(&newPinnerBits.x), unsafe.Pointer(&p.x), bytes)
   364  		s.setPinnerBits(newPinnerBits)
   365  	} else {
   366  		s.setPinnerBits(nil)
   367  	}
   368  }
   369  
   370  // incPinCounter is only called for multiple pins of the same object and records
   371  // the _additional_ pins.
   372  //
   373  // The caller must ensure span is swept and hold span.speciallock with
   374  // preemption disabled.
   375  func (span *mspan) incPinCounter(offset uintptr) {
   376  	var rec *specialPinCounter
   377  	ref, exists := span.specialFindSplicePoint(offset, _KindSpecialPinCounter)
   378  	if !exists {
   379  		if pp := getg().m.p.ptr(); pp != nil && pp.pinCounterCache != nil {
   380  			rec = pp.pinCounterCache
   381  			pp.pinCounterCache = nil
   382  		} else {
   383  			lock(&mheap_.speciallock)
   384  			rec = (*specialPinCounter)(mheap_.specialPinCounterAlloc.alloc())
   385  			unlock(&mheap_.speciallock)
   386  		}
   387  		// splice in record, fill in offset.
   388  		rec.special.offset = offset
   389  		rec.special.kind = _KindSpecialPinCounter
   390  		rec.special.next = *ref
   391  		rec.counter = 1
   392  		*ref = (*special)(unsafe.Pointer(rec))
   393  		spanHasSpecials(span)
   394  	} else {
   395  		rec = (*specialPinCounter)(unsafe.Pointer(*ref))
   396  		rec.counter++
   397  	}
   398  }
   399  
   400  // decPinCounter decreases the counter. If the counter reaches 0, the counter
   401  // special is deleted and false is returned. Otherwise true is returned.
   402  //
   403  // The caller must ensure span is swept and hold span.speciallock with
   404  // preemption disabled.
   405  func (span *mspan) decPinCounter(offset uintptr) bool {
   406  	ref, exists := span.specialFindSplicePoint(offset, _KindSpecialPinCounter)
   407  	if !exists {
   408  		throw("runtime.Pinner: decreased non-existing pin counter")
   409  	}
   410  	counter := (*specialPinCounter)(unsafe.Pointer(*ref))
   411  	counter.counter--
   412  	if counter.counter == 0 {
   413  		*ref = counter.special.next
   414  		if span.specials == nil {
   415  			spanHasNoSpecials(span)
   416  		}
   417  		if pp := getg().m.p.ptr(); pp != nil && pp.pinCounterCache == nil {
   418  			counter.special.next = nil
   419  			pp.pinCounterCache = counter
   420  		} else {
   421  			lock(&mheap_.speciallock)
   422  			mheap_.specialPinCounterAlloc.free(unsafe.Pointer(counter))
   423  			unlock(&mheap_.speciallock)
   424  		}
   425  		return false
   426  	}
   427  	return true
   428  }
   429  
   430  // only for tests
   431  func pinnerGetPinCounter(addr unsafe.Pointer) *uintptr {
   432  	_, span, objIndex := findObject(uintptr(addr), 0, 0)
   433  	offset := objIndex * span.elemsize
   434  	t, exists := span.specialFindSplicePoint(offset, _KindSpecialPinCounter)
   435  	if !exists {
   436  		return nil
   437  	}
   438  	counter := (*specialPinCounter)(unsafe.Pointer(*t))
   439  	return &counter.counter
   440  }
   441  
   442  // to be able to test that the GC panics when a pinned pointer is leaking, this
   443  // panic function is a variable, that can be overwritten by a test.
   444  var pinnerLeakPanic = func() {
   445  	panic(errorString("runtime.Pinner: found leaking pinned pointer; forgot to call Unpin()?"))
   446  }
   447
View as plain text