atomic_arm.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build arm
     6  
     7  package atomic
     8  
     9  import (
    10  	"internal/cpu"
    11  	"unsafe"
    12  )
    13  
    14  const (
    15  	offsetARMHasV7Atomics = unsafe.Offsetof(cpu.ARM.HasV7Atomics)
    16  )
    17  
    18  type spinlock struct {
    19  	v uint32
    20  }
    21  
    22  //go:nosplit
    23  func (l *spinlock) lock() {
    24  	for {
    25  		if Cas(&l.v, 0, 1) {
    26  			return
    27  		}
    28  	}
    29  }
    30  
    31  //go:nosplit
    32  func (l *spinlock) unlock() {
    33  	Store(&l.v, 0)
    34  }
    35  
    36  var locktab [57]struct {
    37  	l   spinlock
    38  	pad [cpu.CacheLinePadSize - unsafe.Sizeof(spinlock{})]byte
    39  }
    40  
    41  func addrLock(addr *uint64) *spinlock {
    42  	return &locktab[(uintptr(unsafe.Pointer(addr))>>3)%uintptr(len(locktab))].l
    43  }
    44  
    45  // Atomic add and return new value.
    46  //
    47  //go:nosplit
    48  func Xadd(val *uint32, delta int32) uint32 {
    49  	for {
    50  		oval := *val
    51  		nval := oval + uint32(delta)
    52  		if Cas(val, oval, nval) {
    53  			return nval
    54  		}
    55  	}
    56  }
    57  
    58  //go:noescape
    59  func Xadduintptr(ptr *uintptr, delta uintptr) uintptr
    60  
    61  //go:nosplit
    62  func Xchg(addr *uint32, v uint32) uint32 {
    63  	for {
    64  		old := *addr
    65  		if Cas(addr, old, v) {
    66  			return old
    67  		}
    68  	}
    69  }
    70  
    71  //go:noescape
    72  func Xchg8(addr *uint8, v uint8) uint8
    73  
    74  //go:nosplit
    75  func Xchguintptr(addr *uintptr, v uintptr) uintptr {
    76  	return uintptr(Xchg((*uint32)(unsafe.Pointer(addr)), uint32(v)))
    77  }
    78  
    79  // Not noescape -- it installs a pointer to addr.
    80  func StorepNoWB(addr unsafe.Pointer, v unsafe.Pointer)
    81  
    82  //go:noescape
    83  func Store(addr *uint32, v uint32)
    84  
    85  //go:noescape
    86  func StoreRel(addr *uint32, v uint32)
    87  
    88  //go:noescape
    89  func StoreReluintptr(addr *uintptr, v uintptr)
    90  
    91  //go:nosplit
    92  func goCas64(addr *uint64, old, new uint64) bool {
    93  	if uintptr(unsafe.Pointer(addr))&7 != 0 {
    94  		*(*int)(nil) = 0 // crash on unaligned uint64
    95  	}
    96  	_ = *addr // if nil, fault before taking the lock
    97  	var ok bool
    98  	addrLock(addr).lock()
    99  	if *addr == old {
   100  		*addr = new
   101  		ok = true
   102  	}
   103  	addrLock(addr).unlock()
   104  	return ok
   105  }
   106  
   107  //go:nosplit
   108  func goXadd64(addr *uint64, delta int64) uint64 {
   109  	if uintptr(unsafe.Pointer(addr))&7 != 0 {
   110  		*(*int)(nil) = 0 // crash on unaligned uint64
   111  	}
   112  	_ = *addr // if nil, fault before taking the lock
   113  	var r uint64
   114  	addrLock(addr).lock()
   115  	r = *addr + uint64(delta)
   116  	*addr = r
   117  	addrLock(addr).unlock()
   118  	return r
   119  }
   120  
   121  //go:nosplit
   122  func goXchg64(addr *uint64, v uint64) uint64 {
   123  	if uintptr(unsafe.Pointer(addr))&7 != 0 {
   124  		*(*int)(nil) = 0 // crash on unaligned uint64
   125  	}
   126  	_ = *addr // if nil, fault before taking the lock
   127  	var r uint64
   128  	addrLock(addr).lock()
   129  	r = *addr
   130  	*addr = v
   131  	addrLock(addr).unlock()
   132  	return r
   133  }
   134  
   135  //go:nosplit
   136  func goLoad64(addr *uint64) uint64 {
   137  	if uintptr(unsafe.Pointer(addr))&7 != 0 {
   138  		*(*int)(nil) = 0 // crash on unaligned uint64
   139  	}
   140  	_ = *addr // if nil, fault before taking the lock
   141  	var r uint64
   142  	addrLock(addr).lock()
   143  	r = *addr
   144  	addrLock(addr).unlock()
   145  	return r
   146  }
   147  
   148  //go:nosplit
   149  func goStore64(addr *uint64, v uint64) {
   150  	if uintptr(unsafe.Pointer(addr))&7 != 0 {
   151  		*(*int)(nil) = 0 // crash on unaligned uint64
   152  	}
   153  	_ = *addr // if nil, fault before taking the lock
   154  	addrLock(addr).lock()
   155  	*addr = v
   156  	addrLock(addr).unlock()
   157  }
   158  
   159  //go:noescape
   160  func Or8(addr *uint8, v uint8)
   161  
   162  //go:nosplit
   163  func goOr8(addr *uint8, v uint8) {
   164  	// Align down to 4 bytes and use 32-bit CAS.
   165  	addr32 := (*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(addr)) &^ 3))
   166  	word := uint32(v) << ((uintptr(unsafe.Pointer(addr)) & 3) * 8) // little endian
   167  	for {
   168  		old := *addr32
   169  		if Cas(addr32, old, old|word) {
   170  			return
   171  		}
   172  	}
   173  }
   174  
   175  //go:noescape
   176  func And8(addr *uint8, v uint8)
   177  
   178  //go:nosplit
   179  func goAnd8(addr *uint8, v uint8) {
   180  	// Align down to 4 bytes and use 32-bit CAS.
   181  	addr32 := (*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(addr)) &^ 3))
   182  	word := uint32(v) << ((uintptr(unsafe.Pointer(addr)) & 3) * 8)    // little endian
   183  	mask := uint32(0xFF) << ((uintptr(unsafe.Pointer(addr)) & 3) * 8) // little endian
   184  	word |= ^mask
   185  	for {
   186  		old := *addr32
   187  		if Cas(addr32, old, old&word) {
   188  			return
   189  		}
   190  	}
   191  }
   192  
   193  //go:nosplit
   194  func Or(addr *uint32, v uint32) {
   195  	for {
   196  		old := *addr
   197  		if Cas(addr, old, old|v) {
   198  			return
   199  		}
   200  	}
   201  }
   202  
   203  //go:nosplit
   204  func And(addr *uint32, v uint32) {
   205  	for {
   206  		old := *addr
   207  		if Cas(addr, old, old&v) {
   208  			return
   209  		}
   210  	}
   211  }
   212  
   213  //go:nosplit
   214  func armcas(ptr *uint32, old, new uint32) bool
   215  
   216  //go:noescape
   217  func Load(addr *uint32) uint32
   218  
   219  // NO go:noescape annotation; *addr escapes if result escapes (#31525)
   220  func Loadp(addr unsafe.Pointer) unsafe.Pointer
   221  
   222  //go:noescape
   223  func Load8(addr *uint8) uint8
   224  
   225  //go:noescape
   226  func LoadAcq(addr *uint32) uint32
   227  
   228  //go:noescape
   229  func LoadAcquintptr(ptr *uintptr) uintptr
   230  
   231  //go:noescape
   232  func Cas64(addr *uint64, old, new uint64) bool
   233  
   234  //go:noescape
   235  func CasRel(addr *uint32, old, new uint32) bool
   236  
   237  //go:noescape
   238  func Xadd64(addr *uint64, delta int64) uint64
   239  
   240  //go:noescape
   241  func Xchg64(addr *uint64, v uint64) uint64
   242  
   243  //go:noescape
   244  func Load64(addr *uint64) uint64
   245  
   246  //go:noescape
   247  func Store8(addr *uint8, v uint8)
   248  
   249  //go:noescape
   250  func Store64(addr *uint64, v uint64)
   251
View as plain text