traceruntime.go

     1  // Copyright 2023 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Runtime -> tracer API.
     6  
     7  package runtime
     8  
     9  import (
    10  	"internal/runtime/atomic"
    11  	"internal/trace/tracev2"
    12  	_ "unsafe" // for go:linkname
    13  )
    14  
    15  // gTraceState is per-G state for the tracer.
    16  type gTraceState struct {
    17  	traceSchedResourceState
    18  }
    19  
    20  // reset resets the gTraceState for a new goroutine.
    21  func (s *gTraceState) reset() {
    22  	s.seq = [2]uint64{}
    23  	// N.B. s.statusTraced is managed and cleared separately.
    24  }
    25  
    26  // mTraceState is per-M state for the tracer.
    27  type mTraceState struct {
    28  	writing       atomic.Bool                          // flag indicating that this M is writing to a trace buffer.
    29  	buf           [2][tracev2.NumExperiments]*traceBuf // Per-M traceBuf for writing. Indexed by trace.gen%2.
    30  	link          *m                                   // Snapshot of alllink or freelink.
    31  	reentered     uint32                               // Whether we've reentered tracing from within tracing.
    32  	entryGen      uintptr                              // The generation value on first entry.
    33  	oldthrowsplit bool                                 // gp.throwsplit upon calling traceLocker.writer. For debugging.
    34  }
    35  
    36  // pTraceState is per-P state for the tracer.
    37  type pTraceState struct {
    38  	traceSchedResourceState
    39  
    40  	// mSyscallID is the ID of the M this was bound to before entering a syscall.
    41  	mSyscallID int64
    42  
    43  	// maySweep indicates the sweep events should be traced.
    44  	// This is used to defer the sweep start event until a span
    45  	// has actually been swept.
    46  	maySweep bool
    47  
    48  	// inSweep indicates that at least one sweep event has been traced.
    49  	inSweep bool
    50  
    51  	// swept and reclaimed track the number of bytes swept and reclaimed
    52  	// by sweeping in the current sweep loop (while maySweep was true).
    53  	swept, reclaimed uintptr
    54  }
    55  
    56  // traceLockInit initializes global trace locks.
    57  func traceLockInit() {
    58  	// Sharing a lock rank here is fine because they should never be accessed
    59  	// together. If they are, we want to find out immediately.
    60  	lockInit(&trace.stringTab[0].lock, lockRankTraceStrings)
    61  	lockInit(&trace.stringTab[0].tab.mem.lock, lockRankTraceStrings)
    62  	lockInit(&trace.stringTab[1].lock, lockRankTraceStrings)
    63  	lockInit(&trace.stringTab[1].tab.mem.lock, lockRankTraceStrings)
    64  	lockInit(&trace.stackTab[0].tab.mem.lock, lockRankTraceStackTab)
    65  	lockInit(&trace.stackTab[1].tab.mem.lock, lockRankTraceStackTab)
    66  	lockInit(&trace.typeTab[0].tab.mem.lock, lockRankTraceTypeTab)
    67  	lockInit(&trace.typeTab[1].tab.mem.lock, lockRankTraceTypeTab)
    68  	lockInit(&trace.lock, lockRankTrace)
    69  }
    70  
    71  // lockRankMayTraceFlush records the lock ranking effects of a
    72  // potential call to traceFlush.
    73  //
    74  // nosplit because traceAcquire is nosplit.
    75  //
    76  //go:nosplit
    77  func lockRankMayTraceFlush() {
    78  	lockWithRankMayAcquire(&trace.lock, getLockRank(&trace.lock))
    79  }
    80  
    81  // traceBlockReason is an enumeration of reasons a goroutine might block.
    82  // This is the interface the rest of the runtime uses to tell the
    83  // tracer why a goroutine blocked. The tracer then propagates this information
    84  // into the trace however it sees fit.
    85  //
    86  // Note that traceBlockReasons should not be compared, since reasons that are
    87  // distinct by name may *not* be distinct by value.
    88  type traceBlockReason uint8
    89  
    90  const (
    91  	traceBlockGeneric traceBlockReason = iota
    92  	traceBlockForever
    93  	traceBlockNet
    94  	traceBlockSelect
    95  	traceBlockCondWait
    96  	traceBlockSync
    97  	traceBlockChanSend
    98  	traceBlockChanRecv
    99  	traceBlockGCMarkAssist
   100  	traceBlockGCSweep
   101  	traceBlockSystemGoroutine
   102  	traceBlockPreempted
   103  	traceBlockDebugCall
   104  	traceBlockUntilGCEnds
   105  	traceBlockSleep
   106  	traceBlockGCWeakToStrongWait
   107  	traceBlockSynctest
   108  )
   109  
   110  var traceBlockReasonStrings = [...]string{
   111  	traceBlockGeneric:            "unspecified",
   112  	traceBlockForever:            "forever",
   113  	traceBlockNet:                "network",
   114  	traceBlockSelect:             "select",
   115  	traceBlockCondWait:           "sync.(*Cond).Wait",
   116  	traceBlockSync:               "sync",
   117  	traceBlockChanSend:           "chan send",
   118  	traceBlockChanRecv:           "chan receive",
   119  	traceBlockGCMarkAssist:       "GC mark assist wait for work",
   120  	traceBlockGCSweep:            "GC background sweeper wait",
   121  	traceBlockSystemGoroutine:    "system goroutine wait",
   122  	traceBlockPreempted:          "preempted",
   123  	traceBlockDebugCall:          "wait for debug call",
   124  	traceBlockUntilGCEnds:        "wait until GC ends",
   125  	traceBlockSleep:              "sleep",
   126  	traceBlockGCWeakToStrongWait: "GC weak to strong wait",
   127  	traceBlockSynctest:           "synctest",
   128  }
   129  
   130  // traceGoStopReason is an enumeration of reasons a goroutine might yield.
   131  //
   132  // Note that traceGoStopReasons should not be compared, since reasons that are
   133  // distinct by name may *not* be distinct by value.
   134  type traceGoStopReason uint8
   135  
   136  const (
   137  	traceGoStopGeneric traceGoStopReason = iota
   138  	traceGoStopGoSched
   139  	traceGoStopPreempted
   140  )
   141  
   142  var traceGoStopReasonStrings = [...]string{
   143  	traceGoStopGeneric:   "unspecified",
   144  	traceGoStopGoSched:   "runtime.Gosched",
   145  	traceGoStopPreempted: "preempted",
   146  }
   147  
   148  // traceEnabled returns true if the trace is currently enabled.
   149  //
   150  //go:nosplit
   151  func traceEnabled() bool {
   152  	return trace.enabled
   153  }
   154  
   155  // traceAllocFreeEnabled returns true if the trace is currently enabled
   156  // and alloc/free events are also enabled.
   157  //
   158  //go:nosplit
   159  func traceAllocFreeEnabled() bool {
   160  	return trace.enabledWithAllocFree
   161  }
   162  
   163  // traceShuttingDown returns true if the trace is currently shutting down.
   164  func traceShuttingDown() bool {
   165  	return trace.shutdown.Load()
   166  }
   167  
   168  // traceLocker represents an M writing trace events. While a traceLocker value
   169  // is valid, the tracer observes all operations on the G/M/P or trace events being
   170  // written as happening atomically.
   171  type traceLocker struct {
   172  	mp  *m
   173  	gen uintptr
   174  }
   175  
   176  // debugTraceReentrancy checks if the trace is reentrant.
   177  //
   178  // This is optional because throwing in a function makes it instantly
   179  // not inlineable, and we want traceAcquire to be inlineable for
   180  // low overhead when the trace is disabled.
   181  const debugTraceReentrancy = false
   182  
   183  // traceAcquire prepares this M for writing one or more trace events.
   184  //
   185  // nosplit because it's called on the syscall path when stack movement is forbidden.
   186  //
   187  //go:nosplit
   188  func traceAcquire() traceLocker {
   189  	if !traceEnabled() {
   190  		return traceLocker{}
   191  	}
   192  	return traceAcquireEnabled()
   193  }
   194  
   195  // traceAcquireEnabled is the traceEnabled path for traceAcquire. It's explicitly
   196  // broken out to make traceAcquire inlineable to keep the overhead of the tracer
   197  // when it's disabled low.
   198  //
   199  // nosplit because it's called by traceAcquire, which is nosplit.
   200  //
   201  //go:nosplit
   202  func traceAcquireEnabled() traceLocker {
   203  	// Any time we acquire a traceLocker, we may flush a trace buffer. But
   204  	// buffer flushes are rare. Record the lock edge even if it doesn't happen
   205  	// this time.
   206  	lockRankMayTraceFlush()
   207  
   208  	// Prevent preemption.
   209  	mp := acquirem()
   210  
   211  	// Check if we're already tracing. It's safe to be reentrant in general,
   212  	// because this function (and the invariants of traceLocker.writer) ensure
   213  	// that it is.
   214  	if mp.trace.writing.Load() {
   215  		mp.trace.reentered++
   216  		return traceLocker{mp, mp.trace.entryGen}
   217  	}
   218  
   219  	// Set the write flag. This prevents traceAdvance from moving forward
   220  	// until all Ms are observed to be outside of a write critical section.
   221  	//
   222  	// Note: The write flag is mutated here and also in traceCPUSample. If you update
   223  	// usage of the write flag here, make sure to also look at what traceCPUSample is
   224  	// doing.
   225  	mp.trace.writing.Store(true)
   226  
   227  	// N.B. This load of gen appears redundant with the one in traceEnabled.
   228  	// However, it's very important that the gen we use for writing to the trace
   229  	// is acquired under a traceLocker so traceAdvance can make sure no stale
   230  	// gen values are being used.
   231  	//
   232  	// Because we're doing this load again, it also means that the trace
   233  	// might end up being disabled when we load it. In that case we need to undo
   234  	// what we did and bail.
   235  	gen := trace.gen.Load()
   236  	if gen == 0 {
   237  		mp.trace.writing.Store(false)
   238  		releasem(mp)
   239  		return traceLocker{}
   240  	}
   241  	mp.trace.entryGen = gen
   242  	return traceLocker{mp, gen}
   243  }
   244  
   245  // ok returns true if the traceLocker is valid (i.e. tracing is enabled).
   246  //
   247  // nosplit because it's called on the syscall path when stack movement is forbidden.
   248  //
   249  //go:nosplit
   250  func (tl traceLocker) ok() bool {
   251  	return tl.gen != 0
   252  }
   253  
   254  // traceRelease indicates that this M is done writing trace events.
   255  //
   256  // nosplit because it's called on the syscall path when stack movement is forbidden.
   257  //
   258  //go:nosplit
   259  func traceRelease(tl traceLocker) {
   260  	if tl.mp.trace.reentered > 0 {
   261  		tl.mp.trace.reentered--
   262  	} else {
   263  		tl.mp.trace.writing.Store(false)
   264  	}
   265  	releasem(tl.mp)
   266  }
   267  
   268  // traceExitingSyscall marks a goroutine as exiting the syscall slow path.
   269  //
   270  // Must be paired with a traceExitedSyscall call.
   271  func traceExitingSyscall() {
   272  	trace.exitingSyscall.Add(1)
   273  }
   274  
   275  // traceExitedSyscall marks a goroutine as having exited the syscall slow path.
   276  func traceExitedSyscall() {
   277  	trace.exitingSyscall.Add(-1)
   278  }
   279  
   280  // Gomaxprocs emits a ProcsChange event.
   281  func (tl traceLocker) Gomaxprocs(procs int32) {
   282  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvProcsChange, traceArg(procs), tl.stack(1))
   283  }
   284  
   285  // ProcStart traces a ProcStart event.
   286  //
   287  // Must be called with a valid P.
   288  func (tl traceLocker) ProcStart() {
   289  	pp := tl.mp.p.ptr()
   290  	// Procs are typically started within the scheduler when there is no user goroutine. If there is a user goroutine,
   291  	// it must be in _Gsyscall because the only time a goroutine is allowed to have its Proc moved around from under it
   292  	// is during a syscall.
   293  	tl.eventWriter(tracev2.GoSyscall, tracev2.ProcIdle).event(tracev2.EvProcStart, traceArg(pp.id), pp.trace.nextSeq(tl.gen))
   294  }
   295  
   296  // ProcStop traces a ProcStop event.
   297  func (tl traceLocker) ProcStop(pp *p) {
   298  	// The only time a goroutine is allowed to have its Proc moved around
   299  	// from under it is during a syscall.
   300  	tl.eventWriter(tracev2.GoSyscall, tracev2.ProcRunning).event(tracev2.EvProcStop)
   301  }
   302  
   303  // GCActive traces a GCActive event.
   304  //
   305  // Must be emitted by an actively running goroutine on an active P. This restriction can be changed
   306  // easily and only depends on where it's currently called.
   307  func (tl traceLocker) GCActive() {
   308  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCActive, traceArg(trace.seqGC))
   309  	// N.B. Only one GC can be running at a time, so this is naturally
   310  	// serialized by the caller.
   311  	trace.seqGC++
   312  }
   313  
   314  // GCStart traces a GCBegin event.
   315  //
   316  // Must be emitted by an actively running goroutine on an active P. This restriction can be changed
   317  // easily and only depends on where it's currently called.
   318  func (tl traceLocker) GCStart() {
   319  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCBegin, traceArg(trace.seqGC), tl.stack(3))
   320  	// N.B. Only one GC can be running at a time, so this is naturally
   321  	// serialized by the caller.
   322  	trace.seqGC++
   323  }
   324  
   325  // GCDone traces a GCEnd event.
   326  //
   327  // Must be emitted by an actively running goroutine on an active P. This restriction can be changed
   328  // easily and only depends on where it's currently called.
   329  func (tl traceLocker) GCDone() {
   330  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCEnd, traceArg(trace.seqGC))
   331  	// N.B. Only one GC can be running at a time, so this is naturally
   332  	// serialized by the caller.
   333  	trace.seqGC++
   334  }
   335  
   336  // STWStart traces a STWBegin event.
   337  func (tl traceLocker) STWStart(reason stwReason) {
   338  	// Although the current P may be in _Pgcstop here, we model the P as running during the STW. This deviates from the
   339  	// runtime's state tracking, but it's more accurate and doesn't result in any loss of information.
   340  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvSTWBegin, tl.string(reason.String()), tl.stack(2))
   341  }
   342  
   343  // STWDone traces a STWEnd event.
   344  func (tl traceLocker) STWDone() {
   345  	// Although the current P may be in _Pgcstop here, we model the P as running during the STW. This deviates from the
   346  	// runtime's state tracking, but it's more accurate and doesn't result in any loss of information.
   347  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvSTWEnd)
   348  }
   349  
   350  // GCSweepStart prepares to trace a sweep loop. This does not
   351  // emit any events until traceGCSweepSpan is called.
   352  //
   353  // GCSweepStart must be paired with traceGCSweepDone and there
   354  // must be no preemption points between these two calls.
   355  //
   356  // Must be called with a valid P.
   357  func (tl traceLocker) GCSweepStart() {
   358  	// Delay the actual GCSweepBegin event until the first span
   359  	// sweep. If we don't sweep anything, don't emit any events.
   360  	pp := tl.mp.p.ptr()
   361  	if pp.trace.maySweep {
   362  		throw("double traceGCSweepStart")
   363  	}
   364  	pp.trace.maySweep, pp.trace.swept, pp.trace.reclaimed = true, 0, 0
   365  }
   366  
   367  // GCSweepSpan traces the sweep of a single span. If this is
   368  // the first span swept since traceGCSweepStart was called, this
   369  // will emit a GCSweepBegin event.
   370  //
   371  // This may be called outside a traceGCSweepStart/traceGCSweepDone
   372  // pair; however, it will not emit any trace events in this case.
   373  //
   374  // Must be called with a valid P.
   375  func (tl traceLocker) GCSweepSpan(bytesSwept uintptr) {
   376  	pp := tl.mp.p.ptr()
   377  	if pp.trace.maySweep {
   378  		if pp.trace.swept == 0 {
   379  			tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCSweepBegin, tl.stack(1))
   380  			pp.trace.inSweep = true
   381  		}
   382  		pp.trace.swept += bytesSwept
   383  	}
   384  }
   385  
   386  // GCSweepDone finishes tracing a sweep loop. If any memory was
   387  // swept (i.e. traceGCSweepSpan emitted an event) then this will emit
   388  // a GCSweepEnd event.
   389  //
   390  // Must be called with a valid P.
   391  func (tl traceLocker) GCSweepDone() {
   392  	pp := tl.mp.p.ptr()
   393  	if !pp.trace.maySweep {
   394  		throw("missing traceGCSweepStart")
   395  	}
   396  	if pp.trace.inSweep {
   397  		tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCSweepEnd, traceArg(pp.trace.swept), traceArg(pp.trace.reclaimed))
   398  		pp.trace.inSweep = false
   399  	}
   400  	pp.trace.maySweep = false
   401  }
   402  
   403  // GCMarkAssistStart emits a MarkAssistBegin event.
   404  func (tl traceLocker) GCMarkAssistStart() {
   405  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCMarkAssistBegin, tl.stack(1))
   406  }
   407  
   408  // GCMarkAssistDone emits a MarkAssistEnd event.
   409  func (tl traceLocker) GCMarkAssistDone() {
   410  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGCMarkAssistEnd)
   411  }
   412  
   413  // GoCreate emits a GoCreate event.
   414  func (tl traceLocker) GoCreate(newg *g, pc uintptr, blocked bool) {
   415  	newg.trace.setStatusTraced(tl.gen)
   416  	ev := tracev2.EvGoCreate
   417  	if blocked {
   418  		ev = tracev2.EvGoCreateBlocked
   419  	}
   420  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(ev, traceArg(newg.goid), tl.startPC(pc), tl.stack(2))
   421  }
   422  
   423  // GoStart emits a GoStart event.
   424  //
   425  // Must be called with a valid P.
   426  func (tl traceLocker) GoStart() {
   427  	gp := getg().m.curg
   428  	pp := gp.m.p
   429  	w := tl.eventWriter(tracev2.GoRunnable, tracev2.ProcRunning)
   430  	w.event(tracev2.EvGoStart, traceArg(gp.goid), gp.trace.nextSeq(tl.gen))
   431  	if pp.ptr().gcMarkWorkerMode != gcMarkWorkerNotWorker {
   432  		w.event(tracev2.EvGoLabel, trace.markWorkerLabels[tl.gen%2][pp.ptr().gcMarkWorkerMode])
   433  	}
   434  }
   435  
   436  // GoEnd emits a GoDestroy event.
   437  //
   438  // TODO(mknyszek): Rename this to GoDestroy.
   439  func (tl traceLocker) GoEnd() {
   440  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoDestroy)
   441  }
   442  
   443  // GoSched emits a GoStop event with a GoSched reason.
   444  func (tl traceLocker) GoSched() {
   445  	tl.GoStop(traceGoStopGoSched)
   446  }
   447  
   448  // GoPreempt emits a GoStop event with a GoPreempted reason.
   449  func (tl traceLocker) GoPreempt() {
   450  	tl.GoStop(traceGoStopPreempted)
   451  }
   452  
   453  // GoStop emits a GoStop event with the provided reason.
   454  func (tl traceLocker) GoStop(reason traceGoStopReason) {
   455  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoStop, trace.goStopReasons[tl.gen%2][reason], tl.stack(0))
   456  }
   457  
   458  // GoPark emits a GoBlock event with the provided reason.
   459  //
   460  // TODO(mknyszek): Replace traceBlockReason with waitReason. It's silly
   461  // that we have both, and waitReason is way more descriptive.
   462  func (tl traceLocker) GoPark(reason traceBlockReason, skip int) {
   463  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoBlock, trace.goBlockReasons[tl.gen%2][reason], tl.stack(skip))
   464  }
   465  
   466  // GoUnpark emits a GoUnblock event.
   467  func (tl traceLocker) GoUnpark(gp *g, skip int) {
   468  	// Emit a GoWaiting status if necessary for the unblocked goroutine.
   469  	tl.emitUnblockStatus(gp, tl.gen)
   470  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoUnblock, traceArg(gp.goid), gp.trace.nextSeq(tl.gen), tl.stack(skip))
   471  }
   472  
   473  // GoSwitch emits a GoSwitch event. If destroy is true, the calling goroutine
   474  // is simultaneously being destroyed.
   475  func (tl traceLocker) GoSwitch(nextg *g, destroy bool) {
   476  	// Emit a GoWaiting status if necessary for the unblocked goroutine.
   477  	tl.emitUnblockStatus(nextg, tl.gen)
   478  	w := tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning)
   479  	ev := tracev2.EvGoSwitch
   480  	if destroy {
   481  		ev = tracev2.EvGoSwitchDestroy
   482  	}
   483  	w.event(ev, traceArg(nextg.goid), nextg.trace.nextSeq(tl.gen))
   484  }
   485  
   486  // emitUnblockStatus emits a GoStatus GoWaiting event for a goroutine about to be
   487  // unblocked to the trace writer.
   488  func (tl traceLocker) emitUnblockStatus(gp *g, gen uintptr) {
   489  	if !gp.trace.statusWasTraced(gen) && gp.trace.acquireStatus(gen) {
   490  		// TODO(go.dev/issue/65634): Although it would be nice to add a stack trace here of gp,
   491  		// we cannot safely do so. gp is in _Gwaiting and so we don't have ownership of its stack.
   492  		// We can fix this by acquiring the goroutine's scan bit.
   493  		tl.writer().writeGoStatus(gp.goid, -1, tracev2.GoWaiting, gp.inMarkAssist, 0).end()
   494  	}
   495  }
   496  
   497  // GoSysCall emits a GoSyscallBegin event.
   498  //
   499  // Must be called with a valid P.
   500  func (tl traceLocker) GoSysCall() {
   501  	// Scribble down the M that the P is currently attached to.
   502  	pp := tl.mp.p.ptr()
   503  	pp.trace.mSyscallID = int64(tl.mp.procid)
   504  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvGoSyscallBegin, pp.trace.nextSeq(tl.gen), tl.stack(1))
   505  }
   506  
   507  // GoSysExit emits a GoSyscallEnd event, possibly along with a GoSyscallBlocked event
   508  // if lostP is true.
   509  //
   510  // lostP must be true in all cases that a goroutine loses its P during a syscall.
   511  // This means it's not sufficient to check if it has no P. In particular, it needs to be
   512  // true in the following cases:
   513  // - The goroutine lost its P, it ran some other code, and then got it back. It's now running with that P.
   514  // - The goroutine lost its P and was unable to reacquire it, and is now running without a P.
   515  // - The goroutine lost its P and acquired a different one, and is now running with that P.
   516  func (tl traceLocker) GoSysExit(lostP bool) {
   517  	ev := tracev2.EvGoSyscallEnd
   518  	procStatus := tracev2.ProcSyscall // Procs implicitly enter tracev2.ProcSyscall on GoSyscallBegin.
   519  	if lostP {
   520  		ev = tracev2.EvGoSyscallEndBlocked
   521  		procStatus = tracev2.ProcRunning // If a G has a P when emitting this event, it reacquired a P and is indeed running.
   522  	} else {
   523  		tl.mp.p.ptr().trace.mSyscallID = -1
   524  	}
   525  	tl.eventWriter(tracev2.GoSyscall, procStatus).event(ev)
   526  }
   527  
   528  // ProcSteal indicates that our current M stole a P from another M.
   529  //
   530  // The caller must have ownership of pp.
   531  func (tl traceLocker) ProcSteal(pp *p) {
   532  	// Grab the M ID we stole from.
   533  	mStolenFrom := pp.trace.mSyscallID
   534  	pp.trace.mSyscallID = -1
   535  
   536  	// Emit the status of the P we're stealing. We may be just about to do this when creating the event
   537  	// writer but it's not guaranteed, even if we're stealing from a syscall. Although it might seem like
   538  	// from a syscall context we're always stealing a P for ourselves, we may have not wired it up yet (so
   539  	// it wouldn't be visible to eventWriter) or we may not even intend to wire it up to ourselves
   540  	// at all and plan to hand it back to the runtime.
   541  	if !pp.trace.statusWasTraced(tl.gen) && pp.trace.acquireStatus(tl.gen) {
   542  		// Careful: don't use the event writer. We never want status or in-progress events
   543  		// to trigger more in-progress events.
   544  		tl.writer().writeProcStatus(uint64(pp.id), tracev2.ProcSyscallAbandoned, pp.trace.inSweep).end()
   545  	}
   546  
   547  	// The status of the proc and goroutine, if we need to emit one here, is not evident from the
   548  	// context of just emitting this event alone. There are two cases. Either we're trying to steal
   549  	// the P just to get its attention (e.g. STW or sysmon retake) or we're trying to steal a P for
   550  	// ourselves specifically to keep running. The two contexts look different, but can be summarized
   551  	// fairly succinctly. In the former, we're a regular running goroutine and proc, if we have either.
   552  	// In the latter, we're a goroutine in a syscall.
   553  	goStatus := tracev2.GoRunning
   554  	procStatus := tracev2.ProcRunning
   555  	if tl.mp.curg != nil && tl.mp.curg.syscallsp != 0 {
   556  		goStatus = tracev2.GoSyscall
   557  		procStatus = tracev2.ProcSyscallAbandoned
   558  	}
   559  	tl.eventWriter(goStatus, procStatus).event(tracev2.EvProcSteal, traceArg(pp.id), pp.trace.nextSeq(tl.gen), traceArg(mStolenFrom))
   560  }
   561  
   562  // HeapAlloc emits a HeapAlloc event.
   563  func (tl traceLocker) HeapAlloc(live uint64) {
   564  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvHeapAlloc, traceArg(live))
   565  }
   566  
   567  // HeapGoal reads the current heap goal and emits a HeapGoal event.
   568  func (tl traceLocker) HeapGoal() {
   569  	heapGoal := gcController.heapGoal()
   570  	// The heapGoal calculations will result in strange numbers if the GC if off. See go.dev/issue/63864.
   571  	// Check gcPercent before using the heapGoal in the trace.
   572  	if heapGoal == ^uint64(0) || gcController.gcPercent.Load() < 0 {
   573  		// Heap-based triggering is disabled.
   574  		heapGoal = 0
   575  	}
   576  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvHeapGoal, traceArg(heapGoal))
   577  }
   578  
   579  // GoCreateSyscall indicates that a goroutine has transitioned from dead to GoSyscall.
   580  //
   581  // Unlike GoCreate, the caller must be running on gp.
   582  //
   583  // This occurs when C code calls into Go. On pthread platforms it occurs only when
   584  // a C thread calls into Go code for the first time.
   585  func (tl traceLocker) GoCreateSyscall(gp *g) {
   586  	// N.B. We should never trace a status for this goroutine (which we're currently running on),
   587  	// since we want this to appear like goroutine creation.
   588  	gp.trace.setStatusTraced(tl.gen)
   589  
   590  	// We might have a P left over on the thread from the last cgo callback,
   591  	// but in a syscall context, it is NOT ours. Act as if we do not have a P,
   592  	// and don't record a status.
   593  	tl.rawEventWriter().event(tracev2.EvGoCreateSyscall, traceArg(gp.goid))
   594  }
   595  
   596  // GoDestroySyscall indicates that a goroutine has transitioned from GoSyscall to dead.
   597  //
   598  // This occurs when Go code returns back to C. On pthread platforms it occurs only when
   599  // the C thread is destroyed.
   600  func (tl traceLocker) GoDestroySyscall() {
   601  	// Write the status for the goroutine if necessary.
   602  	if gp := tl.mp.curg; gp != nil && !gp.trace.statusWasTraced(tl.gen) && gp.trace.acquireStatus(tl.gen) {
   603  		tl.writer().writeGoStatus(gp.goid, int64(tl.mp.procid), tracev2.GoSyscall, false, 0 /* no stack */).end()
   604  	}
   605  
   606  	// We might have a P left over on the thread from the last cgo callback,
   607  	// but in a syscall context, it is NOT ours. Act as if we do not have a P,
   608  	// and don't record a status.
   609  	tl.rawEventWriter().event(tracev2.EvGoDestroySyscall)
   610  }
   611  
   612  // To access runtime functions from runtime/trace.
   613  // See runtime/trace/annotation.go
   614  
   615  // trace_userTaskCreate emits a UserTaskCreate event.
   616  //
   617  //go:linkname trace_userTaskCreate runtime/trace.userTaskCreate
   618  func trace_userTaskCreate(id, parentID uint64, taskType string) {
   619  	tl := traceAcquire()
   620  	if !tl.ok() {
   621  		// Need to do this check because the caller won't have it.
   622  		return
   623  	}
   624  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvUserTaskBegin, traceArg(id), traceArg(parentID), tl.string(taskType), tl.stack(3))
   625  	traceRelease(tl)
   626  }
   627  
   628  // trace_userTaskEnd emits a UserTaskEnd event.
   629  //
   630  //go:linkname trace_userTaskEnd runtime/trace.userTaskEnd
   631  func trace_userTaskEnd(id uint64) {
   632  	tl := traceAcquire()
   633  	if !tl.ok() {
   634  		// Need to do this check because the caller won't have it.
   635  		return
   636  	}
   637  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvUserTaskEnd, traceArg(id), tl.stack(2))
   638  	traceRelease(tl)
   639  }
   640  
   641  // trace_userRegion emits a UserRegionBegin or UserRegionEnd event,
   642  // depending on mode (0 == Begin, 1 == End).
   643  //
   644  // TODO(mknyszek): Just make this two functions.
   645  //
   646  //go:linkname trace_userRegion runtime/trace.userRegion
   647  func trace_userRegion(id, mode uint64, name string) {
   648  	tl := traceAcquire()
   649  	if !tl.ok() {
   650  		// Need to do this check because the caller won't have it.
   651  		return
   652  	}
   653  	var ev tracev2.EventType
   654  	switch mode {
   655  	case 0:
   656  		ev = tracev2.EvUserRegionBegin
   657  	case 1:
   658  		ev = tracev2.EvUserRegionEnd
   659  	default:
   660  		return
   661  	}
   662  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(ev, traceArg(id), tl.string(name), tl.stack(3))
   663  	traceRelease(tl)
   664  }
   665  
   666  // trace_userLog emits a UserRegionBegin or UserRegionEnd event.
   667  //
   668  //go:linkname trace_userLog runtime/trace.userLog
   669  func trace_userLog(id uint64, category, message string) {
   670  	tl := traceAcquire()
   671  	if !tl.ok() {
   672  		// Need to do this check because the caller won't have it.
   673  		return
   674  	}
   675  	tl.eventWriter(tracev2.GoRunning, tracev2.ProcRunning).event(tracev2.EvUserLog, traceArg(id), tl.string(category), tl.uniqueString(message), tl.stack(3))
   676  	traceRelease(tl)
   677  }
   678  
   679  // traceThreadDestroy is called when a thread is removed from
   680  // sched.freem.
   681  //
   682  // mp must not be able to emit trace events anymore.
   683  //
   684  // sched.lock must be held to synchronize with traceAdvance.
   685  func traceThreadDestroy(mp *m) {
   686  	assertLockHeld(&sched.lock)
   687  
   688  	// Flush all outstanding buffers to maintain the invariant
   689  	// that an M only has active buffers while on sched.freem
   690  	// or allm.
   691  	//
   692  	// Perform a traceAcquire/traceRelease on behalf of mp to
   693  	// synchronize with the tracer trying to flush our buffer
   694  	// as well.
   695  	if debugTraceReentrancy && mp.trace.writing.Load() {
   696  		throw("bad use of trace.writing")
   697  	}
   698  	mp.trace.writing.Store(true)
   699  	systemstack(func() {
   700  		lock(&trace.lock)
   701  		for i := range mp.trace.buf {
   702  			for exp, buf := range mp.trace.buf[i] {
   703  				if buf != nil {
   704  					// N.B. traceBufFlush accepts a generation, but it
   705  					// really just cares about gen%2.
   706  					traceBufFlush(buf, uintptr(i))
   707  					mp.trace.buf[i][exp] = nil
   708  				}
   709  			}
   710  		}
   711  		unlock(&trace.lock)
   712  	})
   713  	mp.trace.writing.Store(false)
   714  }
   715
View as plain text