Source file src/runtime/mcleanup.go
1 // Copyright 2024 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/abi" 9 "internal/cpu" 10 "internal/goarch" 11 "internal/runtime/atomic" 12 "internal/runtime/math" 13 "internal/runtime/sys" 14 "unsafe" 15 ) 16 17 // AddCleanup attaches a cleanup function to ptr. Some time after ptr is no longer 18 // reachable, the runtime will call cleanup(arg) in a separate goroutine. 19 // 20 // A typical use is that ptr is an object wrapping an underlying resource (e.g., 21 // a File object wrapping an OS file descriptor), arg is the underlying resource 22 // (e.g., the OS file descriptor), and the cleanup function releases the underlying 23 // resource (e.g., by calling the close system call). 24 // 25 // There are few constraints on ptr. In particular, multiple cleanups may be 26 // attached to the same pointer, or to different pointers within the same 27 // allocation. 28 // 29 // If ptr is reachable from cleanup or arg, ptr will never be collected 30 // and the cleanup will never run. As a protection against simple cases of this, 31 // AddCleanup panics if arg is equal to ptr. 32 // 33 // There is no specified order in which cleanups will run. 34 // In particular, if several objects point to each other and all become 35 // unreachable at the same time, their cleanups all become eligible to run 36 // and can run in any order. This is true even if the objects form a cycle. 37 // 38 // Cleanups run concurrently with any user-created goroutines. 39 // Cleanups may also run concurrently with one another (unlike finalizers). 40 // If a cleanup function must run for a long time, it should create a new goroutine 41 // to avoid blocking the execution of other cleanups. 42 // 43 // If ptr has both a cleanup and a finalizer, the cleanup will only run once 44 // it has been finalized and becomes unreachable without an associated finalizer. 45 // 46 // The cleanup(arg) call is not always guaranteed to run; in particular it is not 47 // guaranteed to run before program exit. 48 // 49 // Cleanups are not guaranteed to run if the size of T is zero bytes, because 50 // it may share same address with other zero-size objects in memory. See 51 // https://go.dev/ref/spec#Size_and_alignment_guarantees. 52 // 53 // It is not guaranteed that a cleanup will run for objects allocated 54 // in initializers for package-level variables. Such objects may be 55 // linker-allocated, not heap-allocated. 56 // 57 // Note that because cleanups may execute arbitrarily far into the future 58 // after an object is no longer referenced, the runtime is allowed to perform 59 // a space-saving optimization that batches objects together in a single 60 // allocation slot. The cleanup for an unreferenced object in such an 61 // allocation may never run if it always exists in the same batch as a 62 // referenced object. Typically, this batching only happens for tiny 63 // (on the order of 16 bytes or less) and pointer-free objects. 64 // 65 // A cleanup may run as soon as an object becomes unreachable. 66 // In order to use cleanups correctly, the program must ensure that 67 // the object is reachable until it is safe to run its cleanup. 68 // Objects stored in global variables, or that can be found by tracing 69 // pointers from a global variable, are reachable. A function argument or 70 // receiver may become unreachable at the last point where the function 71 // mentions it. To ensure a cleanup does not get called prematurely, 72 // pass the object to the [KeepAlive] function after the last point 73 // where the object must remain reachable. 74 // 75 //go:nocheckptr 76 func AddCleanup[T, S any](ptr *T, cleanup func(S), arg S) Cleanup { 77 // This is marked nocheckptr because checkptr doesn't understand the 78 // pointer manipulation done when looking at closure pointers. 79 // Similar code in mbitmap.go works because the functions are 80 // go:nosplit, which implies go:nocheckptr (CL 202158). 81 82 // Explicitly force ptr and cleanup to escape to the heap. 83 ptr = abi.Escape(ptr) 84 cleanup = abi.Escape(cleanup) 85 86 // The pointer to the object must be valid. 87 if ptr == nil { 88 panic("runtime.AddCleanup: ptr is nil") 89 } 90 usptr := uintptr(unsafe.Pointer(ptr)) 91 92 // Check that arg is not equal to ptr. 93 argType := abi.TypeOf(arg) 94 kind := argType.Kind() 95 if kind == abi.Pointer || kind == abi.UnsafePointer { 96 if unsafe.Pointer(ptr) == *((*unsafe.Pointer)(unsafe.Pointer(&arg))) { 97 panic("runtime.AddCleanup: ptr is equal to arg, cleanup will never run") 98 } 99 } 100 if inUserArenaChunk(usptr) { 101 // Arena-allocated objects are not eligible for cleanup. 102 panic("runtime.AddCleanup: ptr is arena-allocated") 103 } 104 if debug.sbrk != 0 { 105 // debug.sbrk never frees memory, so no cleanup will ever run 106 // (and we don't have the data structures to record them). 107 // Return a noop cleanup. 108 return Cleanup{} 109 } 110 111 // Create new storage for the argument. 112 var argv *S 113 if size := unsafe.Sizeof(arg); size < maxTinySize && argType.PtrBytes == 0 { 114 // Side-step the tiny allocator to avoid liveness issues, since this box 115 // will be treated like a root by the GC. We model the box as an array of 116 // uintptrs to guarantee maximum allocator alignment. 117 // 118 // TODO(mknyszek): Consider just making space in cleanupFn for this. The 119 // unfortunate part of this is it would grow specialCleanup by 16 bytes, so 120 // while there wouldn't be an allocation, *every* cleanup would take the 121 // memory overhead hit. 122 box := new([maxTinySize / goarch.PtrSize]uintptr) 123 argv = (*S)(unsafe.Pointer(box)) 124 } else { 125 argv = new(S) 126 } 127 *argv = arg 128 129 // Find the containing object. 130 base, span, _ := findObject(usptr, 0, 0) 131 if base == 0 { 132 if isGoPointerWithoutSpan(unsafe.Pointer(ptr)) { 133 // Cleanup is a noop. 134 return Cleanup{} 135 } 136 panic("runtime.AddCleanup: ptr not in allocated block") 137 } 138 139 // Check that arg is not within ptr. 140 if kind == abi.Pointer || kind == abi.UnsafePointer { 141 argPtr := uintptr(*(*unsafe.Pointer)(unsafe.Pointer(&arg))) 142 if argPtr >= base && argPtr < base+span.elemsize { 143 // It's possible that both pointers are separate 144 // parts of a tiny allocation, which is OK. 145 // We side-stepped the tiny allocator above for 146 // the allocation for the cleanup, 147 // but the argument itself can still overlap 148 // with the value to which the cleanup is attached. 149 if span.spanclass != tinySpanClass { 150 panic("runtime.AddCleanup: ptr is within arg, cleanup will never run") 151 } 152 } 153 } 154 155 // Check that the cleanup function doesn't close over the pointer. 156 cleanupFV := unsafe.Pointer(*(**funcval)(unsafe.Pointer(&cleanup))) 157 cBase, cSpan, _ := findObject(uintptr(cleanupFV), 0, 0) 158 if cBase != 0 { 159 tp := cSpan.typePointersOfUnchecked(cBase) 160 for { 161 var addr uintptr 162 if tp, addr = tp.next(cBase + cSpan.elemsize); addr == 0 { 163 break 164 } 165 ptr := *(*uintptr)(unsafe.Pointer(addr)) 166 if ptr >= base && ptr < base+span.elemsize { 167 panic("runtime.AddCleanup: cleanup function closes over ptr, cleanup will never run") 168 } 169 } 170 } 171 172 // Create another G if necessary. 173 if gcCleanups.needG() { 174 gcCleanups.createGs() 175 } 176 177 id := addCleanup(unsafe.Pointer(ptr), cleanupFn{ 178 // Instantiate a caller function to call the cleanup, that is cleanup(*argv). 179 // 180 // TODO(mknyszek): This allocates because the generic dictionary argument 181 // gets closed over, but callCleanup doesn't even use the dictionary argument, 182 // so theoretically that could be removed, eliminating an allocation. 183 call: callCleanup[S], 184 fn: *(**funcval)(unsafe.Pointer(&cleanup)), 185 arg: unsafe.Pointer(argv), 186 }) 187 if debug.checkfinalizers != 0 { 188 cleanupFn := *(**funcval)(unsafe.Pointer(&cleanup)) 189 setCleanupContext(unsafe.Pointer(ptr), abi.TypeFor[T](), sys.GetCallerPC(), cleanupFn.fn, id) 190 } 191 return Cleanup{ 192 id: id, 193 ptr: usptr, 194 } 195 } 196 197 // callCleanup is a helper for calling cleanups in a polymorphic way. 198 // 199 // In practice, all it does is call fn(*arg). arg must be a *T. 200 // 201 //go:noinline 202 func callCleanup[T any](fn *funcval, arg unsafe.Pointer) { 203 cleanup := *(*func(T))(unsafe.Pointer(&fn)) 204 cleanup(*(*T)(arg)) 205 } 206 207 // Cleanup is a handle to a cleanup call for a specific object. 208 type Cleanup struct { 209 // id is the unique identifier for the cleanup within the arena. 210 id uint64 211 // ptr contains the pointer to the object. 212 ptr uintptr 213 } 214 215 // Stop cancels the cleanup call. Stop will have no effect if the cleanup call 216 // has already been queued for execution (because ptr became unreachable). 217 // To guarantee that Stop removes the cleanup function, the caller must ensure 218 // that the pointer that was passed to AddCleanup is reachable across the call to Stop. 219 func (c Cleanup) Stop() { 220 if c.id == 0 { 221 // id is set to zero when the cleanup is a noop. 222 return 223 } 224 225 // The following block removes the Special record of type cleanup for the object c.ptr. 226 span := spanOfHeap(c.ptr) 227 if span == nil { 228 return 229 } 230 // Ensure that the span is swept. 231 // Sweeping accesses the specials list w/o locks, so we have 232 // to synchronize with it. And it's just much safer. 233 mp := acquirem() 234 span.ensureSwept() 235 236 offset := c.ptr - span.base() 237 238 var found *special 239 lock(&span.speciallock) 240 241 iter, exists := span.specialFindSplicePoint(offset, _KindSpecialCleanup) 242 if exists { 243 for { 244 s := *iter 245 if s == nil { 246 // Reached the end of the linked list. Stop searching at this point. 247 break 248 } 249 if offset == s.offset && _KindSpecialCleanup == s.kind && 250 (*specialCleanup)(unsafe.Pointer(s)).id == c.id { 251 // The special is a cleanup and contains a matching cleanup id. 252 *iter = s.next 253 found = s 254 break 255 } 256 if offset < s.offset || (offset == s.offset && _KindSpecialCleanup < s.kind) { 257 // The special is outside the region specified for that kind of 258 // special. The specials are sorted by kind. 259 break 260 } 261 // Try the next special. 262 iter = &s.next 263 } 264 } 265 if span.specials == nil { 266 spanHasNoSpecials(span) 267 } 268 unlock(&span.speciallock) 269 releasem(mp) 270 271 if found == nil { 272 return 273 } 274 lock(&mheap_.speciallock) 275 mheap_.specialCleanupAlloc.free(unsafe.Pointer(found)) 276 unlock(&mheap_.speciallock) 277 278 if debug.checkfinalizers != 0 { 279 clearCleanupContext(c.ptr, c.id) 280 } 281 } 282 283 const cleanupBlockSize = 512 284 285 // cleanupBlock is an block of cleanups to be executed. 286 // 287 // cleanupBlock is allocated from non-GC'd memory, so any heap pointers 288 // must be specially handled. The GC and cleanup queue currently assume 289 // that the cleanup queue does not grow during marking (but it can shrink). 290 type cleanupBlock struct { 291 cleanupBlockHeader 292 cleanups [(cleanupBlockSize - unsafe.Sizeof(cleanupBlockHeader{})) / unsafe.Sizeof(cleanupFn{})]cleanupFn 293 } 294 295 var cleanupFnPtrMask = [...]uint8{0b111} 296 297 // cleanupFn represents a cleanup function with it's argument, yet to be called. 298 type cleanupFn struct { 299 // call is an adapter function that understands how to safely call fn(*arg). 300 call func(*funcval, unsafe.Pointer) 301 fn *funcval // cleanup function passed to AddCleanup. 302 arg unsafe.Pointer // pointer to argument to pass to cleanup function. 303 } 304 305 var cleanupBlockPtrMask [cleanupBlockSize / goarch.PtrSize / 8]byte 306 307 type cleanupBlockHeader struct { 308 _ sys.NotInHeap 309 lfnode 310 alllink *cleanupBlock 311 312 // n is sometimes accessed atomically. 313 // 314 // The invariant depends on what phase the garbage collector is in. 315 // During the sweep phase (gcphase == _GCoff), each block has exactly 316 // one owner, so it's always safe to update this without atomics. 317 // But if this *could* be updated during the mark phase, it must be 318 // updated atomically to synchronize with the garbage collector 319 // scanning the block as a root. 320 n uint32 321 } 322 323 // enqueue pushes a single cleanup function into the block. 324 // 325 // Returns if this enqueue call filled the block. This is odd, 326 // but we want to flush full blocks eagerly to get cleanups 327 // running as soon as possible. 328 // 329 // Must only be called if the GC is in the sweep phase (gcphase == _GCoff), 330 // because it does not synchronize with the garbage collector. 331 func (b *cleanupBlock) enqueue(c cleanupFn) bool { 332 b.cleanups[b.n] = c 333 b.n++ 334 return b.full() 335 } 336 337 // full returns true if the cleanup block is full. 338 func (b *cleanupBlock) full() bool { 339 return b.n == uint32(len(b.cleanups)) 340 } 341 342 // empty returns true if the cleanup block is empty. 343 func (b *cleanupBlock) empty() bool { 344 return b.n == 0 345 } 346 347 // take moves as many cleanups as possible from b into a. 348 func (a *cleanupBlock) take(b *cleanupBlock) { 349 dst := a.cleanups[a.n:] 350 if uint32(len(dst)) >= b.n { 351 // Take all. 352 copy(dst, b.cleanups[:]) 353 a.n += b.n 354 b.n = 0 355 } else { 356 // Partial take. Copy from the tail to avoid having 357 // to move more memory around. 358 copy(dst, b.cleanups[b.n-uint32(len(dst)):b.n]) 359 a.n = uint32(len(a.cleanups)) 360 b.n -= uint32(len(dst)) 361 } 362 } 363 364 // cleanupQueue is a queue of ready-to-run cleanup functions. 365 type cleanupQueue struct { 366 // Stack of full cleanup blocks. 367 full lfstack 368 workUnits atomic.Uint64 // length of full; decrement before pop from full, increment after push to full 369 _ [cpu.CacheLinePadSize - unsafe.Sizeof(lfstack(0)) - unsafe.Sizeof(atomic.Uint64{})]byte 370 371 // Stack of free cleanup blocks. 372 free lfstack 373 374 // flushed indicates whether all local cleanupBlocks have been 375 // flushed, and we're in a period of time where this condition is 376 // stable (after the last sweeper, before the next sweep phase 377 // begins). 378 flushed atomic.Bool // Next to free because frequently accessed together. 379 380 _ [cpu.CacheLinePadSize - unsafe.Sizeof(lfstack(0)) - 1]byte 381 382 // Linked list of all cleanup blocks. 383 all atomic.UnsafePointer // *cleanupBlock 384 _ [cpu.CacheLinePadSize - unsafe.Sizeof(atomic.UnsafePointer{})]byte 385 386 // Goroutine block state. 387 lock mutex 388 389 // sleeping is the list of sleeping cleanup goroutines. 390 // 391 // Protected by lock. 392 sleeping gList 393 394 // asleep is the number of cleanup goroutines sleeping. 395 // 396 // Read without lock, written only with the lock held. 397 // When the lock is held, the lock holder may only observe 398 // asleep.Load() == sleeping.n. 399 // 400 // To make reading without the lock safe as a signal to wake up 401 // a goroutine and handle new work, it must always be greater 402 // than or equal to sleeping.n. In the periods of time that it 403 // is strictly greater, it may cause spurious calls to wake. 404 asleep atomic.Uint32 405 406 // running indicates the number of cleanup goroutines actively 407 // executing user cleanup functions at any point in time. 408 // 409 // Read and written to without lock. 410 running atomic.Uint32 411 412 // ng is the number of cleanup goroutines. 413 // 414 // Read without lock, written only with lock held. 415 ng atomic.Uint32 416 417 // needg is the number of new cleanup goroutines that 418 // need to be created. 419 // 420 // Read without lock, written only with lock held. 421 needg atomic.Uint32 422 423 // Cleanup queue stats. 424 425 // queued represents a monotonic count of queued cleanups. This is sharded across 426 // Ps via the field cleanupsQueued in each p, so reading just this value is insufficient. 427 // In practice, this value only includes the queued count of dead Ps. 428 // 429 // Writes are protected by STW. 430 queued uint64 431 432 // executed is a monotonic count of executed cleanups. 433 // 434 // Read and updated atomically. 435 executed atomic.Uint64 436 } 437 438 // addWork indicates that n units of parallelizable work have been added to the queue. 439 func (q *cleanupQueue) addWork(n int) { 440 q.workUnits.Add(int64(n)) 441 } 442 443 // tryTakeWork is an attempt to dequeue some work by a cleanup goroutine. 444 // This might fail if there's no work to do. 445 func (q *cleanupQueue) tryTakeWork() bool { 446 for { 447 wu := q.workUnits.Load() 448 if wu == 0 { 449 return false 450 } 451 // CAS to prevent us from going negative. 452 if q.workUnits.CompareAndSwap(wu, wu-1) { 453 return true 454 } 455 } 456 } 457 458 // enqueue queues a single cleanup for execution. 459 // 460 // Called by the sweeper, and only the sweeper. 461 func (q *cleanupQueue) enqueue(c cleanupFn) { 462 mp := acquirem() 463 pp := mp.p.ptr() 464 b := pp.cleanups 465 if b == nil { 466 if q.flushed.Load() { 467 q.flushed.Store(false) 468 } 469 b = (*cleanupBlock)(q.free.pop()) 470 if b == nil { 471 b = (*cleanupBlock)(persistentalloc(cleanupBlockSize, tagAlign, &memstats.gcMiscSys)) 472 for { 473 next := (*cleanupBlock)(q.all.Load()) 474 b.alllink = next 475 if q.all.CompareAndSwap(unsafe.Pointer(next), unsafe.Pointer(b)) { 476 break 477 } 478 } 479 } 480 pp.cleanups = b 481 } 482 if full := b.enqueue(c); full { 483 q.full.push(&b.lfnode) 484 pp.cleanups = nil 485 q.addWork(1) 486 } 487 pp.cleanupsQueued++ 488 releasem(mp) 489 } 490 491 // dequeue pops a block of cleanups from the queue. Blocks until one is available 492 // and never returns nil. 493 func (q *cleanupQueue) dequeue() *cleanupBlock { 494 for { 495 if q.tryTakeWork() { 496 // Guaranteed to be non-nil. 497 return (*cleanupBlock)(q.full.pop()) 498 } 499 lock(&q.lock) 500 // Increment asleep first. We may have to undo this if we abort the sleep. 501 // We must update asleep first because the scheduler might not try to wake 502 // us up when work comes in between the last check of workUnits and when we 503 // go to sleep. (It may see asleep as 0.) By incrementing it here, we guarantee 504 // after this point that if new work comes in, someone will try to grab the 505 // lock and wake us. However, this also means that if we back out, we may cause 506 // someone to spuriously grab the lock and try to wake us up, only to fail. 507 // This should be very rare because the window here is incredibly small: the 508 // window between now and when we decrement q.asleep below. 509 q.asleep.Add(1) 510 511 // Re-check workUnits under the lock and with asleep updated. If it's still zero, 512 // then no new work came in, and it's safe for us to go to sleep. If new work 513 // comes in after this point, then the scheduler will notice that we're sleeping 514 // and wake us up. 515 if q.workUnits.Load() > 0 { 516 // Undo the q.asleep update and try to take work again. 517 q.asleep.Add(-1) 518 unlock(&q.lock) 519 continue 520 } 521 q.sleeping.push(getg()) 522 goparkunlock(&q.lock, waitReasonCleanupWait, traceBlockSystemGoroutine, 1) 523 } 524 } 525 526 // flush pushes all active cleanup blocks to the full list and wakes up cleanup 527 // goroutines to handle them. 528 // 529 // Must only be called at a point when we can guarantee that no more cleanups 530 // are being queued, such as after the final sweeper for the cycle is done 531 // but before the next mark phase. 532 func (q *cleanupQueue) flush() { 533 mp := acquirem() 534 flushed := 0 535 emptied := 0 536 missing := 0 537 538 // Coalesce the partially-filled blocks to present a more accurate picture of demand. 539 // We use the number of coalesced blocks to process as a signal for demand to create 540 // new cleanup goroutines. 541 var cb *cleanupBlock 542 for _, pp := range allp { 543 if pp == nil { 544 // This function is reachable via mallocgc in the 545 // middle of procresize, when allp has been resized, 546 // but the new Ps not allocated yet. 547 missing++ 548 continue 549 } 550 b := pp.cleanups 551 if b == nil { 552 missing++ 553 continue 554 } 555 pp.cleanups = nil 556 if cb == nil { 557 cb = b 558 continue 559 } 560 // N.B. After take, either cb is full, b is empty, or both. 561 cb.take(b) 562 if cb.full() { 563 q.full.push(&cb.lfnode) 564 flushed++ 565 cb = b 566 b = nil 567 } 568 if b != nil && b.empty() { 569 q.free.push(&b.lfnode) 570 emptied++ 571 } 572 } 573 if cb != nil { 574 q.full.push(&cb.lfnode) 575 flushed++ 576 } 577 if flushed != 0 { 578 q.addWork(flushed) 579 } 580 if flushed+emptied+missing != len(allp) { 581 throw("failed to correctly flush all P-owned cleanup blocks") 582 } 583 q.flushed.Store(true) 584 releasem(mp) 585 } 586 587 // needsWake returns true if cleanup goroutines may need to be awoken or created to handle cleanup load. 588 func (q *cleanupQueue) needsWake() bool { 589 return q.workUnits.Load() > 0 && (q.asleep.Load() > 0 || q.ng.Load() < maxCleanupGs()) 590 } 591 592 // wake wakes up one or more goroutines to process the cleanup queue. If there aren't 593 // enough sleeping goroutines to handle the demand, wake will arrange for new goroutines 594 // to be created. 595 func (q *cleanupQueue) wake() { 596 lock(&q.lock) 597 598 // Figure out how many goroutines to wake, and how many extra goroutines to create. 599 // Wake one goroutine for each work unit. 600 var wake, extra uint32 601 work := q.workUnits.Load() 602 asleep := uint64(q.asleep.Load()) 603 if work > asleep { 604 wake = uint32(asleep) 605 if work > uint64(math.MaxUint32) { 606 // Protect against overflow. 607 extra = math.MaxUint32 608 } else { 609 extra = uint32(work - asleep) 610 } 611 } else { 612 wake = uint32(work) 613 extra = 0 614 } 615 if extra != 0 { 616 // Signal that we should create new goroutines, one for each extra work unit, 617 // up to maxCleanupGs. 618 newg := min(extra, maxCleanupGs()-q.ng.Load()) 619 if newg > 0 { 620 q.needg.Add(int32(newg)) 621 } 622 } 623 if wake == 0 { 624 // Nothing to do. 625 unlock(&q.lock) 626 return 627 } 628 629 // Take ownership of waking 'wake' goroutines. 630 // 631 // Nobody else will wake up these goroutines, so they're guaranteed 632 // to be sitting on q.sleeping, waiting for us to wake them. 633 q.asleep.Add(-int32(wake)) 634 635 // Collect them and schedule them. 636 var list gList 637 for range wake { 638 list.push(q.sleeping.pop()) 639 } 640 unlock(&q.lock) 641 642 injectglist(&list) 643 return 644 } 645 646 func (q *cleanupQueue) needG() bool { 647 have := q.ng.Load() 648 if have >= maxCleanupGs() { 649 return false 650 } 651 if have == 0 { 652 // Make sure we have at least one. 653 return true 654 } 655 return q.needg.Load() > 0 656 } 657 658 func (q *cleanupQueue) createGs() { 659 lock(&q.lock) 660 have := q.ng.Load() 661 need := min(q.needg.Swap(0), maxCleanupGs()-have) 662 if have == 0 && need == 0 { 663 // Make sure we have at least one. 664 need = 1 665 } 666 if need > 0 { 667 q.ng.Add(int32(need)) 668 } 669 unlock(&q.lock) 670 671 for range need { 672 go runCleanups() 673 } 674 } 675 676 func (q *cleanupQueue) beginRunningCleanups() { 677 // Update runningCleanups and running atomically with respect 678 // to goroutine profiles by disabling preemption. 679 mp := acquirem() 680 getg().runningCleanups.Store(true) 681 q.running.Add(1) 682 releasem(mp) 683 } 684 685 func (q *cleanupQueue) endRunningCleanups() { 686 // Update runningCleanups and running atomically with respect 687 // to goroutine profiles by disabling preemption. 688 mp := acquirem() 689 getg().runningCleanups.Store(false) 690 q.running.Add(-1) 691 releasem(mp) 692 } 693 694 func (q *cleanupQueue) readQueueStats() (queued, executed uint64) { 695 executed = q.executed.Load() 696 queued = q.queued 697 698 // N.B. This is inconsistent, but that's intentional. It's just an estimate. 699 // Read this _after_ reading executed to decrease the chance that we observe 700 // an inconsistency in the statistics (executed > queued). 701 for _, pp := range allp { 702 queued += pp.cleanupsQueued 703 } 704 return 705 } 706 707 func maxCleanupGs() uint32 { 708 // N.B. Left as a function to make changing the policy easier. 709 return uint32(max(gomaxprocs/4, 1)) 710 } 711 712 // gcCleanups is the global cleanup queue. 713 var gcCleanups cleanupQueue 714 715 // runCleanups is the entrypoint for all cleanup-running goroutines. 716 func runCleanups() { 717 for { 718 b := gcCleanups.dequeue() 719 if raceenabled { 720 // Approximately: adds a happens-before edge between the cleanup 721 // argument being mutated and the call to the cleanup below. 722 racefingo() 723 } 724 725 gcCleanups.beginRunningCleanups() 726 for i := 0; i < int(b.n); i++ { 727 c := b.cleanups[i] 728 b.cleanups[i] = cleanupFn{} 729 730 var racectx uintptr 731 if raceenabled { 732 // Enter a new race context so the race detector can catch 733 // potential races between cleanups, even if they execute on 734 // the same goroutine. 735 // 736 // Synchronize on fn. This would fail to find races on the 737 // closed-over values in fn (suppose arg is passed to multiple 738 // AddCleanup calls) if arg was not unique, but it is. 739 racerelease(unsafe.Pointer(c.arg)) 740 racectx = raceEnterNewCtx() 741 raceacquire(unsafe.Pointer(c.arg)) 742 } 743 744 // Execute the next cleanup. 745 c.call(c.fn, c.arg) 746 747 if raceenabled { 748 // Restore the old context. 749 raceRestoreCtx(racectx) 750 } 751 } 752 gcCleanups.endRunningCleanups() 753 gcCleanups.executed.Add(int64(b.n)) 754 755 atomic.Store(&b.n, 0) // Synchronize with markroot. See comment in cleanupBlockHeader. 756 gcCleanups.free.push(&b.lfnode) 757 } 758 } 759 760 // blockUntilEmpty blocks until either the cleanup queue is emptied 761 // and the cleanups have been executed, or the timeout is reached. 762 // Returns true if the cleanup queue was emptied. 763 // This is used by the sync and unique tests. 764 func (q *cleanupQueue) blockUntilEmpty(timeout int64) bool { 765 start := nanotime() 766 for nanotime()-start < timeout { 767 lock(&q.lock) 768 // The queue is empty when there's no work left to do *and* all the cleanup goroutines 769 // are asleep. If they're not asleep, they may be actively working on a block. 770 if q.flushed.Load() && q.full.empty() && uint32(q.sleeping.size) == q.ng.Load() { 771 unlock(&q.lock) 772 return true 773 } 774 unlock(&q.lock) 775 Gosched() 776 } 777 return false 778 } 779 780 //go:linkname unique_runtime_blockUntilEmptyCleanupQueue unique.runtime_blockUntilEmptyCleanupQueue 781 func unique_runtime_blockUntilEmptyCleanupQueue(timeout int64) bool { 782 return gcCleanups.blockUntilEmpty(timeout) 783 } 784 785 //go:linkname sync_test_runtime_blockUntilEmptyCleanupQueue sync_test.runtime_blockUntilEmptyCleanupQueue 786 func sync_test_runtime_blockUntilEmptyCleanupQueue(timeout int64) bool { 787 return gcCleanups.blockUntilEmpty(timeout) 788 } 789 790 // raceEnterNewCtx creates a new racectx and switches the current 791 // goroutine to it. Returns the old racectx. 792 // 793 // Must be running on a user goroutine. nosplit to match other race 794 // instrumentation. 795 // 796 //go:nosplit 797 func raceEnterNewCtx() uintptr { 798 // We use the existing ctx as the spawn context, but gp.gopc 799 // as the spawn PC to make the error output a little nicer 800 // (pointing to AddCleanup, where the goroutines are created). 801 // 802 // We also need to carefully indicate to the race detector 803 // that the goroutine stack will only be accessed by the new 804 // race context, to avoid false positives on stack locations. 805 // We do this by marking the stack as free in the first context 806 // and then re-marking it as allocated in the second. Crucially, 807 // there must be (1) no race operations and (2) no stack changes 808 // in between. (1) is easy to avoid because we're in the runtime 809 // so there's no implicit race instrumentation. To avoid (2) we 810 // defensively become non-preemptible so the GC can't stop us, 811 // and rely on the fact that racemalloc, racefreem, and racectx 812 // are nosplit. 813 mp := acquirem() 814 gp := getg() 815 ctx := getg().racectx 816 racefree(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo) 817 getg().racectx = racectxstart(gp.gopc, ctx) 818 racemalloc(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo) 819 releasem(mp) 820 return ctx 821 } 822 823 // raceRestoreCtx restores ctx on the goroutine. It is the inverse of 824 // raceenternewctx and must be called with its result. 825 // 826 // Must be running on a user goroutine. nosplit to match other race 827 // instrumentation. 828 // 829 //go:nosplit 830 func raceRestoreCtx(ctx uintptr) { 831 mp := acquirem() 832 gp := getg() 833 racefree(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo) 834 racectxend(getg().racectx) 835 racemalloc(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo) 836 getg().racectx = ctx 837 releasem(mp) 838 } 839