Source file src/runtime/stubs.go
1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/abi" 9 "unsafe" 10 ) 11 12 // Should be a built-in for unsafe.Pointer? 13 // 14 // add should be an internal detail, 15 // but widely used packages access it using linkname. 16 // Notable members of the hall of shame include: 17 // - fortio.org/log 18 // 19 // Do not remove or change the type signature. 20 // See go.dev/issue/67401. 21 // 22 //go:linkname add 23 //go:nosplit 24 func add(p unsafe.Pointer, x uintptr) unsafe.Pointer { 25 return unsafe.Pointer(uintptr(p) + x) 26 } 27 28 // getg returns the pointer to the current g. 29 // The compiler rewrites calls to this function into instructions 30 // that fetch the g directly (from TLS or from the dedicated register). 31 func getg() *g 32 33 // mcall switches from the g to the g0 stack and invokes fn(g), 34 // where g is the goroutine that made the call. 35 // mcall saves g's current PC/SP in g->sched so that it can be restored later. 36 // It is up to fn to arrange for that later execution, typically by recording 37 // g in a data structure, causing something to call ready(g) later. 38 // mcall returns to the original goroutine g later, when g has been rescheduled. 39 // fn must not return at all; typically it ends by calling schedule, to let the m 40 // run other goroutines. 41 // 42 // mcall can only be called from g stacks (not g0, not gsignal). 43 // 44 // This must NOT be go:noescape: if fn is a stack-allocated closure, 45 // fn puts g on a run queue, and g executes before fn returns, the 46 // closure will be invalidated while it is still executing. 47 func mcall(fn func(*g)) 48 49 // systemstack runs fn on a system stack. 50 // If systemstack is called from the per-OS-thread (g0) stack, or 51 // if systemstack is called from the signal handling (gsignal) stack, 52 // systemstack calls fn directly and returns. 53 // Otherwise, systemstack is being called from the limited stack 54 // of an ordinary goroutine. In this case, systemstack switches 55 // to the per-OS-thread stack, calls fn, and switches back. 56 // It is common to use a func literal as the argument, in order 57 // to share inputs and outputs with the code around the call 58 // to system stack: 59 // 60 // ... set up y ... 61 // systemstack(func() { 62 // x = bigcall(y) 63 // }) 64 // ... use x ... 65 // 66 //go:noescape 67 func systemstack(fn func()) 68 69 //go:nosplit 70 //go:nowritebarrierrec 71 func badsystemstack() { 72 writeErrStr("fatal: systemstack called from unexpected goroutine") 73 } 74 75 // memclrNoHeapPointers clears n bytes starting at ptr. 76 // 77 // Usually you should use typedmemclr. memclrNoHeapPointers should be 78 // used only when the caller knows that *ptr contains no heap pointers 79 // because either: 80 // 81 // *ptr is initialized memory and its type is pointer-free, or 82 // 83 // *ptr is uninitialized memory (e.g., memory that's being reused 84 // for a new allocation) and hence contains only "junk". 85 // 86 // memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n 87 // is a multiple of the pointer size, then any pointer-aligned, 88 // pointer-sized portion is cleared atomically. Despite the function 89 // name, this is necessary because this function is the underlying 90 // implementation of typedmemclr and memclrHasPointers. See the doc of 91 // memmove for more details. 92 // 93 // The (CPU-specific) implementations of this function are in memclr_*.s. 94 // 95 // memclrNoHeapPointers should be an internal detail, 96 // but widely used packages access it using linkname. 97 // Notable members of the hall of shame include: 98 // - github.com/bytedance/sonic 99 // - github.com/chenzhuoyu/iasm 100 // - github.com/cloudwego/frugal 101 // - github.com/dgraph-io/ristretto 102 // - github.com/outcaste-io/ristretto 103 // 104 // Do not remove or change the type signature. 105 // See go.dev/issue/67401. 106 // 107 //go:linkname memclrNoHeapPointers 108 //go:noescape 109 func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) 110 111 //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers 112 func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) { 113 memclrNoHeapPointers(ptr, n) 114 } 115 116 // memmove copies n bytes from "from" to "to". 117 // 118 // memmove ensures that any pointer in "from" is written to "to" with 119 // an indivisible write, so that racy reads cannot observe a 120 // half-written pointer. This is necessary to prevent the garbage 121 // collector from observing invalid pointers, and differs from memmove 122 // in unmanaged languages. However, memmove is only required to do 123 // this if "from" and "to" may contain pointers, which can only be the 124 // case if "from", "to", and "n" are all be word-aligned. 125 // 126 // Implementations are in memmove_*.s. 127 // 128 // Outside assembly calls memmove. 129 // 130 // memmove should be an internal detail, 131 // but widely used packages access it using linkname. 132 // Notable members of the hall of shame include: 133 // - github.com/bytedance/sonic 134 // - github.com/cloudwego/dynamicgo 135 // - github.com/cloudwego/frugal 136 // - github.com/ebitengine/purego 137 // - github.com/tetratelabs/wazero 138 // - github.com/ugorji/go/codec 139 // - gvisor.dev/gvisor 140 // - github.com/sagernet/gvisor 141 // 142 // Do not remove or change the type signature. 143 // See go.dev/issue/67401. 144 // 145 //go:linkname memmove 146 //go:noescape 147 func memmove(to, from unsafe.Pointer, n uintptr) 148 149 //go:linkname reflect_memmove reflect.memmove 150 func reflect_memmove(to, from unsafe.Pointer, n uintptr) { 151 memmove(to, from, n) 152 } 153 154 // exported value for testing 155 const hashLoad = float32(loadFactorNum) / float32(loadFactorDen) 156 157 // in internal/bytealg/equal_*.s 158 // 159 // memequal should be an internal detail, 160 // but widely used packages access it using linkname. 161 // Notable members of the hall of shame include: 162 // - github.com/bytedance/sonic 163 // 164 // Do not remove or change the type signature. 165 // See go.dev/issue/67401. 166 // 167 //go:linkname memequal 168 //go:noescape 169 func memequal(a, b unsafe.Pointer, size uintptr) bool 170 171 // noescape hides a pointer from escape analysis. noescape is 172 // the identity function but escape analysis doesn't think the 173 // output depends on the input. noescape is inlined and currently 174 // compiles down to zero instructions. 175 // USE CAREFULLY! 176 // 177 // noescape should be an internal detail, 178 // but widely used packages access it using linkname. 179 // Notable members of the hall of shame include: 180 // - github.com/bytedance/gopkg 181 // - github.com/ebitengine/purego 182 // - github.com/hamba/avro/v2 183 // - github.com/puzpuzpuz/xsync/v3 184 // - github.com/songzhibin97/gkit 185 // 186 // Do not remove or change the type signature. 187 // See go.dev/issue/67401. 188 // 189 //go:linkname noescape 190 //go:nosplit 191 func noescape(p unsafe.Pointer) unsafe.Pointer { 192 x := uintptr(p) 193 return unsafe.Pointer(x ^ 0) 194 } 195 196 // noEscapePtr hides a pointer from escape analysis. See noescape. 197 // USE CAREFULLY! 198 // 199 //go:nosplit 200 func noEscapePtr[T any](p *T) *T { 201 x := uintptr(unsafe.Pointer(p)) 202 return (*T)(unsafe.Pointer(x ^ 0)) 203 } 204 205 // Not all cgocallback frames are actually cgocallback, 206 // so not all have these arguments. Mark them uintptr so that the GC 207 // does not misinterpret memory when the arguments are not present. 208 // cgocallback is not called from Go, only from crosscall2. 209 // This in turn calls cgocallbackg, which is where we'll find 210 // pointer-declared arguments. 211 // 212 // When fn is nil (frame is saved g), call dropm instead, 213 // this is used when the C thread is exiting. 214 func cgocallback(fn, frame, ctxt uintptr) 215 216 func gogo(buf *gobuf) 217 218 func asminit() 219 func setg(gg *g) 220 func breakpoint() 221 222 // reflectcall calls fn with arguments described by stackArgs, stackArgsSize, 223 // frameSize, and regArgs. 224 // 225 // Arguments passed on the stack and space for return values passed on the stack 226 // must be laid out at the space pointed to by stackArgs (with total length 227 // stackArgsSize) according to the ABI. 228 // 229 // stackRetOffset must be some value <= stackArgsSize that indicates the 230 // offset within stackArgs where the return value space begins. 231 // 232 // frameSize is the total size of the argument frame at stackArgs and must 233 // therefore be >= stackArgsSize. It must include additional space for spilling 234 // register arguments for stack growth and preemption. 235 // 236 // TODO(mknyszek): Once we don't need the additional spill space, remove frameSize, 237 // since frameSize will be redundant with stackArgsSize. 238 // 239 // Arguments passed in registers must be laid out in regArgs according to the ABI. 240 // regArgs will hold any return values passed in registers after the call. 241 // 242 // reflectcall copies stack arguments from stackArgs to the goroutine stack, and 243 // then copies back stackArgsSize-stackRetOffset bytes back to the return space 244 // in stackArgs once fn has completed. It also "unspills" argument registers from 245 // regArgs before calling fn, and spills them back into regArgs immediately 246 // following the call to fn. If there are results being returned on the stack, 247 // the caller should pass the argument frame type as stackArgsType so that 248 // reflectcall can execute appropriate write barriers during the copy. 249 // 250 // reflectcall expects regArgs.ReturnIsPtr to be populated indicating which 251 // registers on the return path will contain Go pointers. It will then store 252 // these pointers in regArgs.Ptrs such that they are visible to the GC. 253 // 254 // Package reflect passes a frame type. In package runtime, there is only 255 // one call that copies results back, in callbackWrap in syscall_windows.go, and it 256 // does NOT pass a frame type, meaning there are no write barriers invoked. See that 257 // call site for justification. 258 // 259 // Package reflect accesses this symbol through a linkname. 260 // 261 // Arguments passed through to reflectcall do not escape. The type is used 262 // only in a very limited callee of reflectcall, the stackArgs are copied, and 263 // regArgs is only used in the reflectcall frame. 264 // 265 //go:noescape 266 func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 267 268 // procyield should be an internal detail, 269 // but widely used packages access it using linkname. 270 // Notable members of the hall of shame include: 271 // - github.com/sagernet/sing-tun 272 // - github.com/slackhq/nebula 273 // - github.com/tailscale/wireguard-go 274 // 275 // Do not remove or change the type signature. 276 // See go.dev/issue/67401. 277 // 278 //go:linkname procyield 279 func procyield(cycles uint32) 280 281 type neverCallThisFunction struct{} 282 283 // goexit is the return stub at the top of every goroutine call stack. 284 // Each goroutine stack is constructed as if goexit called the 285 // goroutine's entry point function, so that when the entry point 286 // function returns, it will return to goexit, which will call goexit1 287 // to perform the actual exit. 288 // 289 // This function must never be called directly. Call goexit1 instead. 290 // gentraceback assumes that goexit terminates the stack. A direct 291 // call on the stack will cause gentraceback to stop walking the stack 292 // prematurely and if there is leftover state it may panic. 293 func goexit(neverCallThisFunction) 294 295 // publicationBarrier performs a store/store barrier (a "publication" 296 // or "export" barrier). Some form of synchronization is required 297 // between initializing an object and making that object accessible to 298 // another processor. Without synchronization, the initialization 299 // writes and the "publication" write may be reordered, allowing the 300 // other processor to follow the pointer and observe an uninitialized 301 // object. In general, higher-level synchronization should be used, 302 // such as locking or an atomic pointer write. publicationBarrier is 303 // for when those aren't an option, such as in the implementation of 304 // the memory manager. 305 // 306 // There's no corresponding barrier for the read side because the read 307 // side naturally has a data dependency order. All architectures that 308 // Go supports or seems likely to ever support automatically enforce 309 // data dependency ordering. 310 func publicationBarrier() 311 312 // getcallerpc returns the program counter (PC) of its caller's caller. 313 // getcallersp returns the stack pointer (SP) of its caller's caller. 314 // The implementation may be a compiler intrinsic; there is not 315 // necessarily code implementing this on every platform. 316 // 317 // For example: 318 // 319 // func f(arg1, arg2, arg3 int) { 320 // pc := getcallerpc() 321 // sp := getcallersp() 322 // } 323 // 324 // These two lines find the PC and SP immediately following 325 // the call to f (where f will return). 326 // 327 // The call to getcallerpc and getcallersp must be done in the 328 // frame being asked about. 329 // 330 // The result of getcallersp is correct at the time of the return, 331 // but it may be invalidated by any subsequent call to a function 332 // that might relocate the stack in order to grow or shrink it. 333 // A general rule is that the result of getcallersp should be used 334 // immediately and can only be passed to nosplit functions. 335 336 //go:noescape 337 func getcallerpc() uintptr 338 339 //go:noescape 340 func getcallersp() uintptr // implemented as an intrinsic on all platforms 341 342 // getclosureptr returns the pointer to the current closure. 343 // getclosureptr can only be used in an assignment statement 344 // at the entry of a function. Moreover, go:nosplit directive 345 // must be specified at the declaration of caller function, 346 // so that the function prolog does not clobber the closure register. 347 // for example: 348 // 349 // //go:nosplit 350 // func f(arg1, arg2, arg3 int) { 351 // dx := getclosureptr() 352 // } 353 // 354 // The compiler rewrites calls to this function into instructions that fetch the 355 // pointer from a well-known register (DX on x86 architecture, etc.) directly. 356 // 357 // WARNING: PGO-based devirtualization cannot detect that caller of 358 // getclosureptr require closure context, and thus must maintain a list of 359 // these functions, which is in 360 // cmd/compile/internal/devirtualize/pgo.maybeDevirtualizeFunctionCall. 361 func getclosureptr() uintptr 362 363 //go:noescape 364 func asmcgocall(fn, arg unsafe.Pointer) int32 365 366 func morestack() 367 368 // morestack_noctxt should be an internal detail, 369 // but widely used packages access it using linkname. 370 // Notable members of the hall of shame include: 371 // - github.com/cloudwego/frugal 372 // 373 // Do not remove or change the type signature. 374 // See go.dev/issue/67401. 375 // 376 //go:linkname morestack_noctxt 377 func morestack_noctxt() 378 379 func rt0_go() 380 381 // return0 is a stub used to return 0 from deferproc. 382 // It is called at the very end of deferproc to signal 383 // the calling Go function that it should not jump 384 // to deferreturn. 385 // in asm_*.s 386 func return0() 387 388 // in asm_*.s 389 // not called directly; definitions here supply type information for traceback. 390 // These must have the same signature (arg pointer map) as reflectcall. 391 func call16(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 392 func call32(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 393 func call64(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 394 func call128(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 395 func call256(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 396 func call512(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 397 func call1024(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 398 func call2048(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 399 func call4096(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 400 func call8192(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 401 func call16384(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 402 func call32768(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 403 func call65536(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 404 func call131072(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 405 func call262144(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 406 func call524288(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 407 func call1048576(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 408 func call2097152(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 409 func call4194304(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 410 func call8388608(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 411 func call16777216(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 412 func call33554432(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 413 func call67108864(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 414 func call134217728(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 415 func call268435456(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 416 func call536870912(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 417 func call1073741824(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 418 419 func systemstack_switch() 420 421 // alignUp rounds n up to a multiple of a. a must be a power of 2. 422 // 423 //go:nosplit 424 func alignUp(n, a uintptr) uintptr { 425 return (n + a - 1) &^ (a - 1) 426 } 427 428 // alignDown rounds n down to a multiple of a. a must be a power of 2. 429 // 430 //go:nosplit 431 func alignDown(n, a uintptr) uintptr { 432 return n &^ (a - 1) 433 } 434 435 // divRoundUp returns ceil(n / a). 436 func divRoundUp(n, a uintptr) uintptr { 437 // a is generally a power of two. This will get inlined and 438 // the compiler will optimize the division. 439 return (n + a - 1) / a 440 } 441 442 // checkASM reports whether assembly runtime checks have passed. 443 func checkASM() bool 444 445 func memequal_varlen(a, b unsafe.Pointer) bool 446 447 // bool2int returns 0 if x is false or 1 if x is true. 448 func bool2int(x bool) int { 449 // Avoid branches. In the SSA compiler, this compiles to 450 // exactly what you would want it to. 451 return int(*(*uint8)(unsafe.Pointer(&x))) 452 } 453 454 // abort crashes the runtime in situations where even throw might not 455 // work. In general it should do something a debugger will recognize 456 // (e.g., an INT3 on x86). A crash in abort is recognized by the 457 // signal handler, which will attempt to tear down the runtime 458 // immediately. 459 func abort() 460 461 // Called from compiled code; declared for vet; do NOT call from Go. 462 func gcWriteBarrier1() 463 464 // gcWriteBarrier2 should be an internal detail, 465 // but widely used packages access it using linkname. 466 // Notable members of the hall of shame include: 467 // - github.com/bytedance/sonic 468 // - github.com/cloudwego/frugal 469 // 470 // Do not remove or change the type signature. 471 // See go.dev/issue/67401. 472 // 473 //go:linkname gcWriteBarrier2 474 func gcWriteBarrier2() 475 476 func gcWriteBarrier3() 477 func gcWriteBarrier4() 478 func gcWriteBarrier5() 479 func gcWriteBarrier6() 480 func gcWriteBarrier7() 481 func gcWriteBarrier8() 482 func duffzero() 483 func duffcopy() 484 485 // Called from linker-generated .initarray; declared for go vet; do NOT call from Go. 486 func addmoduledata() 487 488 // Injected by the signal handler for panicking signals. 489 // Initializes any registers that have fixed meaning at calls but 490 // are scratch in bodies and calls sigpanic. 491 // On many platforms it just jumps to sigpanic. 492 func sigpanic0() 493 494 // intArgRegs is used by the various register assignment 495 // algorithm implementations in the runtime. These include:. 496 // - Finalizers (mfinal.go) 497 // - Windows callbacks (syscall_windows.go) 498 // 499 // Both are stripped-down versions of the algorithm since they 500 // only have to deal with a subset of cases (finalizers only 501 // take a pointer or interface argument, Go Windows callbacks 502 // don't support floating point). 503 // 504 // It should be modified with care and are generally only 505 // modified when testing this package. 506 // 507 // It should never be set higher than its internal/abi 508 // constant counterparts, because the system relies on a 509 // structure that is at least large enough to hold the 510 // registers the system supports. 511 // 512 // Protected by finlock. 513 var intArgRegs = abi.IntArgRegs 514