Source file src/runtime/atomic_pointer.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/goexperiment"
     9  	"internal/runtime/atomic"
    10  	"unsafe"
    11  )
    12  
    13  // These functions cannot have go:noescape annotations,
    14  // because while ptr does not escape, new does.
    15  // If new is marked as not escaping, the compiler will make incorrect
    16  // escape analysis decisions about the pointer value being stored.
    17  
    18  // atomicwb performs a write barrier before an atomic pointer write.
    19  // The caller should guard the call with "if writeBarrier.enabled".
    20  //
    21  // atomicwb should be an internal detail,
    22  // but widely used packages access it using linkname.
    23  // Notable members of the hall of shame include:
    24  //   - github.com/bytedance/gopkg
    25  //   - github.com/songzhibin97/gkit
    26  //
    27  // Do not remove or change the type signature.
    28  // See go.dev/issue/67401.
    29  //
    30  //go:linkname atomicwb
    31  //go:nosplit
    32  func atomicwb(ptr *unsafe.Pointer, new unsafe.Pointer) {
    33  	slot := (*uintptr)(unsafe.Pointer(ptr))
    34  	buf := getg().m.p.ptr().wbBuf.get2()
    35  	buf[0] = *slot
    36  	buf[1] = uintptr(new)
    37  }
    38  
    39  // atomicstorep performs *ptr = new atomically and invokes a write barrier.
    40  //
    41  //go:nosplit
    42  func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) {
    43  	if writeBarrier.enabled {
    44  		atomicwb((*unsafe.Pointer)(ptr), new)
    45  	}
    46  	if goexperiment.CgoCheck2 {
    47  		cgoCheckPtrWrite((*unsafe.Pointer)(ptr), new)
    48  	}
    49  	atomic.StorepNoWB(noescape(ptr), new)
    50  }
    51  
    52  // atomic_storePointer is the implementation of runtime/internal/UnsafePointer.Store
    53  // (like StoreNoWB but with the write barrier).
    54  //
    55  //go:nosplit
    56  //go:linkname atomic_storePointer internal/runtime/atomic.storePointer
    57  func atomic_storePointer(ptr *unsafe.Pointer, new unsafe.Pointer) {
    58  	atomicstorep(unsafe.Pointer(ptr), new)
    59  }
    60  
    61  // atomic_casPointer is the implementation of runtime/internal/UnsafePointer.CompareAndSwap
    62  // (like CompareAndSwapNoWB but with the write barrier).
    63  //
    64  //go:nosplit
    65  //go:linkname atomic_casPointer internal/runtime/atomic.casPointer
    66  func atomic_casPointer(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool {
    67  	if writeBarrier.enabled {
    68  		atomicwb(ptr, new)
    69  	}
    70  	if goexperiment.CgoCheck2 {
    71  		cgoCheckPtrWrite(ptr, new)
    72  	}
    73  	return atomic.Casp1(ptr, old, new)
    74  }
    75  
    76  // Like above, but implement in terms of sync/atomic's uintptr operations.
    77  // We cannot just call the runtime routines, because the race detector expects
    78  // to be able to intercept the sync/atomic forms but not the runtime forms.
    79  
    80  //go:linkname sync_atomic_StoreUintptr sync/atomic.StoreUintptr
    81  func sync_atomic_StoreUintptr(ptr *uintptr, new uintptr)
    82  
    83  //go:linkname sync_atomic_StorePointer sync/atomic.StorePointer
    84  //go:nosplit
    85  func sync_atomic_StorePointer(ptr *unsafe.Pointer, new unsafe.Pointer) {
    86  	if writeBarrier.enabled {
    87  		atomicwb(ptr, new)
    88  	}
    89  	if goexperiment.CgoCheck2 {
    90  		cgoCheckPtrWrite(ptr, new)
    91  	}
    92  	sync_atomic_StoreUintptr((*uintptr)(unsafe.Pointer(ptr)), uintptr(new))
    93  }
    94  
    95  //go:linkname sync_atomic_SwapUintptr sync/atomic.SwapUintptr
    96  func sync_atomic_SwapUintptr(ptr *uintptr, new uintptr) uintptr
    97  
    98  //go:linkname sync_atomic_SwapPointer sync/atomic.SwapPointer
    99  //go:nosplit
   100  func sync_atomic_SwapPointer(ptr *unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer {
   101  	if writeBarrier.enabled {
   102  		atomicwb(ptr, new)
   103  	}
   104  	if goexperiment.CgoCheck2 {
   105  		cgoCheckPtrWrite(ptr, new)
   106  	}
   107  	old := unsafe.Pointer(sync_atomic_SwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(new)))
   108  	return old
   109  }
   110  
   111  //go:linkname sync_atomic_CompareAndSwapUintptr sync/atomic.CompareAndSwapUintptr
   112  func sync_atomic_CompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool
   113  
   114  //go:linkname sync_atomic_CompareAndSwapPointer sync/atomic.CompareAndSwapPointer
   115  //go:nosplit
   116  func sync_atomic_CompareAndSwapPointer(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool {
   117  	if writeBarrier.enabled {
   118  		atomicwb(ptr, new)
   119  	}
   120  	if goexperiment.CgoCheck2 {
   121  		cgoCheckPtrWrite(ptr, new)
   122  	}
   123  	return sync_atomic_CompareAndSwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(old), uintptr(new))
   124  }
   125  

View as plain text