Source file src/runtime/runtime.go
1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/abi" 9 "internal/runtime/atomic" 10 "unsafe" 11 ) 12 13 //go:generate go run wincallback.go 14 //go:generate go run mkduff.go 15 //go:generate go run mkfastlog2table.go 16 //go:generate go run mklockrank.go -o lockrank.go 17 18 var ticks ticksType 19 20 type ticksType struct { 21 // lock protects access to start* and val. 22 lock mutex 23 startTicks int64 24 startTime int64 25 val atomic.Int64 26 } 27 28 // init initializes ticks to maximize the chance that we have a good ticksPerSecond reference. 29 // 30 // Must not run concurrently with ticksPerSecond. 31 func (t *ticksType) init() { 32 lock(&ticks.lock) 33 t.startTime = nanotime() 34 t.startTicks = cputicks() 35 unlock(&ticks.lock) 36 } 37 38 // minTimeForTicksPerSecond is the minimum elapsed time we require to consider our ticksPerSecond 39 // measurement to be of decent enough quality for profiling. 40 // 41 // There's a linear relationship here between minimum time and error from the true value. 42 // The error from the true ticks-per-second in a linux/amd64 VM seems to be: 43 // - 1 ms -> ~0.02% error 44 // - 5 ms -> ~0.004% error 45 // - 10 ms -> ~0.002% error 46 // - 50 ms -> ~0.0003% error 47 // - 100 ms -> ~0.0001% error 48 // 49 // We're willing to take 0.004% error here, because ticksPerSecond is intended to be used for 50 // converting durations, not timestamps. Durations are usually going to be much larger, and so 51 // the tiny error doesn't matter. The error is definitely going to be a problem when trying to 52 // use this for timestamps, as it'll make those timestamps much less likely to line up. 53 const minTimeForTicksPerSecond = 5_000_000*(1-osHasLowResClockInt) + 100_000_000*osHasLowResClockInt 54 55 // ticksPerSecond returns a conversion rate between the cputicks clock and the nanotime clock. 56 // 57 // Note: Clocks are hard. Using this as an actual conversion rate for timestamps is ill-advised 58 // and should be avoided when possible. Use only for durations, where a tiny error term isn't going 59 // to make a meaningful difference in even a 1ms duration. If an accurate timestamp is needed, 60 // use nanotime instead. (The entire Windows platform is a broad exception to this rule, where nanotime 61 // produces timestamps on such a coarse granularity that the error from this conversion is actually 62 // preferable.) 63 // 64 // The strategy for computing the conversion rate is to write down nanotime and cputicks as 65 // early in process startup as possible. From then, we just need to wait until we get values 66 // from nanotime that we can use (some platforms have a really coarse system time granularity). 67 // We require some amount of time to pass to ensure that the conversion rate is fairly accurate 68 // in aggregate. But because we compute this rate lazily, there's a pretty good chance a decent 69 // amount of time has passed by the time we get here. 70 // 71 // Must be called from a normal goroutine context (running regular goroutine with a P). 72 // 73 // Called by runtime/pprof in addition to runtime code. 74 // 75 // TODO(mknyszek): This doesn't account for things like CPU frequency scaling. Consider 76 // a more sophisticated and general approach in the future. 77 func ticksPerSecond() int64 { 78 // Get the conversion rate if we've already computed it. 79 r := ticks.val.Load() 80 if r != 0 { 81 return r 82 } 83 84 // Compute the conversion rate. 85 for { 86 lock(&ticks.lock) 87 r = ticks.val.Load() 88 if r != 0 { 89 unlock(&ticks.lock) 90 return r 91 } 92 93 // Grab the current time in both clocks. 94 nowTime := nanotime() 95 nowTicks := cputicks() 96 97 // See if we can use these times. 98 if nowTicks > ticks.startTicks && nowTime-ticks.startTime > minTimeForTicksPerSecond { 99 // Perform the calculation with floats. We don't want to risk overflow. 100 r = int64(float64(nowTicks-ticks.startTicks) * 1e9 / float64(nowTime-ticks.startTime)) 101 if r == 0 { 102 // Zero is both a sentinel value and it would be bad if callers used this as 103 // a divisor. We tried out best, so just make it 1. 104 r++ 105 } 106 ticks.val.Store(r) 107 unlock(&ticks.lock) 108 break 109 } 110 unlock(&ticks.lock) 111 112 // Sleep in one millisecond increments until we have a reliable time. 113 timeSleep(1_000_000) 114 } 115 return r 116 } 117 118 var envs []string 119 var argslice []string 120 121 //go:linkname syscall_runtime_envs syscall.runtime_envs 122 func syscall_runtime_envs() []string { return append([]string{}, envs...) } 123 124 //go:linkname syscall_Getpagesize syscall.Getpagesize 125 func syscall_Getpagesize() int { return int(physPageSize) } 126 127 //go:linkname os_runtime_args os.runtime_args 128 func os_runtime_args() []string { return append([]string{}, argslice...) } 129 130 //go:linkname syscall_Exit syscall.Exit 131 //go:nosplit 132 func syscall_Exit(code int) { 133 exit(int32(code)) 134 } 135 136 var godebugDefault string 137 var godebugUpdate atomic.Pointer[func(string, string)] 138 var godebugEnv atomic.Pointer[string] // set by parsedebugvars 139 var godebugNewIncNonDefault atomic.Pointer[func(string) func()] 140 141 //go:linkname godebug_setUpdate internal/godebug.setUpdate 142 func godebug_setUpdate(update func(string, string)) { 143 p := new(func(string, string)) 144 *p = update 145 godebugUpdate.Store(p) 146 godebugNotify(false) 147 } 148 149 //go:linkname godebug_setNewIncNonDefault internal/godebug.setNewIncNonDefault 150 func godebug_setNewIncNonDefault(newIncNonDefault func(string) func()) { 151 p := new(func(string) func()) 152 *p = newIncNonDefault 153 godebugNewIncNonDefault.Store(p) 154 } 155 156 // A godebugInc provides access to internal/godebug's IncNonDefault function 157 // for a given GODEBUG setting. 158 // Calls before internal/godebug registers itself are dropped on the floor. 159 type godebugInc struct { 160 name string 161 inc atomic.Pointer[func()] 162 } 163 164 func (g *godebugInc) IncNonDefault() { 165 inc := g.inc.Load() 166 if inc == nil { 167 newInc := godebugNewIncNonDefault.Load() 168 if newInc == nil { 169 return 170 } 171 inc = new(func()) 172 *inc = (*newInc)(g.name) 173 if raceenabled { 174 racereleasemerge(unsafe.Pointer(&g.inc)) 175 } 176 if !g.inc.CompareAndSwap(nil, inc) { 177 inc = g.inc.Load() 178 } 179 } 180 if raceenabled { 181 raceacquire(unsafe.Pointer(&g.inc)) 182 } 183 (*inc)() 184 } 185 186 func godebugNotify(envChanged bool) { 187 update := godebugUpdate.Load() 188 var env string 189 if p := godebugEnv.Load(); p != nil { 190 env = *p 191 } 192 if envChanged { 193 reparsedebugvars(env) 194 } 195 if update != nil { 196 (*update)(godebugDefault, env) 197 } 198 } 199 200 //go:linkname syscall_runtimeSetenv syscall.runtimeSetenv 201 func syscall_runtimeSetenv(key, value string) { 202 setenv_c(key, value) 203 if key == "GODEBUG" { 204 p := new(string) 205 *p = value 206 godebugEnv.Store(p) 207 godebugNotify(true) 208 } 209 } 210 211 //go:linkname syscall_runtimeUnsetenv syscall.runtimeUnsetenv 212 func syscall_runtimeUnsetenv(key string) { 213 unsetenv_c(key) 214 if key == "GODEBUG" { 215 godebugEnv.Store(nil) 216 godebugNotify(true) 217 } 218 } 219 220 // writeErrStr writes a string to descriptor 2. 221 // If SetCrashOutput(f) was called, it also writes to f. 222 // 223 //go:nosplit 224 func writeErrStr(s string) { 225 writeErrData(unsafe.StringData(s), int32(len(s))) 226 } 227 228 // writeErrData is the common parts of writeErr{,Str}. 229 // 230 //go:nosplit 231 func writeErrData(data *byte, n int32) { 232 write(2, unsafe.Pointer(data), n) 233 234 // If crashing, print a copy to the SetCrashOutput fd. 235 gp := getg() 236 if gp != nil && gp.m.dying > 0 || 237 gp == nil && panicking.Load() > 0 { 238 if fd := crashFD.Load(); fd != ^uintptr(0) { 239 write(fd, unsafe.Pointer(data), n) 240 } 241 } 242 } 243 244 // crashFD is an optional file descriptor to use for fatal panics, as 245 // set by debug.SetCrashOutput (see #42888). If it is a valid fd (not 246 // all ones), writeErr and related functions write to it in addition 247 // to standard error. 248 // 249 // Initialized to -1 in schedinit. 250 var crashFD atomic.Uintptr 251 252 //go:linkname setCrashFD 253 func setCrashFD(fd uintptr) uintptr { 254 // Don't change the crash FD if a crash is already in progress. 255 // 256 // Unlike the case below, this is not required for correctness, but it 257 // is generally nicer to have all of the crash output go to the same 258 // place rather than getting split across two different FDs. 259 if panicking.Load() > 0 { 260 return ^uintptr(0) 261 } 262 263 old := crashFD.Swap(fd) 264 265 // If we are panicking, don't return the old FD to runtime/debug for 266 // closing. writeErrData may have already read the old FD from crashFD 267 // before the swap and closing it would cause the write to be lost [1]. 268 // The old FD will never be closed, but we are about to crash anyway. 269 // 270 // On the writeErrData thread, panicking.Add(1) happens-before 271 // crashFD.Load() [2]. 272 // 273 // On this thread, swapping old FD for new in crashFD happens-before 274 // panicking.Load() > 0. 275 // 276 // Therefore, if panicking.Load() == 0 here (old FD will be closed), it 277 // is impossible for the writeErrData thread to observe 278 // crashFD.Load() == old FD. 279 // 280 // [1] Or, if really unlucky, another concurrent open could reuse the 281 // FD, sending the write into an unrelated file. 282 // 283 // [2] If gp != nil, it occurs when incrementing gp.m.dying in 284 // startpanic_m. If gp == nil, we read panicking.Load() > 0, so an Add 285 // must have happened-before. 286 if panicking.Load() > 0 { 287 return ^uintptr(0) 288 } 289 return old 290 } 291 292 // auxv is populated on relevant platforms but defined here for all platforms 293 // so x/sys/cpu can assume the getAuxv symbol exists without keeping its list 294 // of auxv-using GOOS build tags in sync. 295 // 296 // It contains an even number of elements, (tag, value) pairs. 297 var auxv []uintptr 298 299 // golang.org/x/sys/cpu uses getAuxv via linkname. 300 // Do not remove or change the type signature. 301 // (See go.dev/issue/57336.) 302 // 303 // getAuxv should be an internal detail, 304 // but widely used packages access it using linkname. 305 // Notable members of the hall of shame include: 306 // - github.com/cilium/ebpf 307 // 308 // Do not remove or change the type signature. 309 // See go.dev/issue/67401. 310 // 311 //go:linkname getAuxv 312 func getAuxv() []uintptr { return auxv } 313 314 // zeroVal is used by reflect via linkname. 315 // 316 // zeroVal should be an internal detail, 317 // but widely used packages access it using linkname. 318 // Notable members of the hall of shame include: 319 // - github.com/ugorji/go/codec 320 // 321 // Do not remove or change the type signature. 322 // See go.dev/issue/67401. 323 // 324 //go:linkname zeroVal 325 var zeroVal [abi.ZeroValSize]byte 326