github.com/comwrg/go/src@v0.0.0-20220319063731-c238d0440370/runtime/trace.go (about) 1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Go execution tracer. 6 // The tracer captures a wide range of execution events like goroutine 7 // creation/blocking/unblocking, syscall enter/exit/block, GC-related events, 8 // changes of heap size, processor start/stop, etc and writes them to a buffer 9 // in a compact form. A precise nanosecond-precision timestamp and a stack 10 // trace is captured for most events. 11 // See https://golang.org/s/go15trace for more info. 12 13 package runtime 14 15 import ( 16 "runtime/internal/atomic" 17 "runtime/internal/sys" 18 "unsafe" 19 ) 20 21 // Event types in the trace, args are given in square brackets. 22 const ( 23 traceEvNone = 0 // unused 24 traceEvBatch = 1 // start of per-P batch of events [pid, timestamp] 25 traceEvFrequency = 2 // contains tracer timer frequency [frequency (ticks per second)] 26 traceEvStack = 3 // stack [stack id, number of PCs, array of {PC, func string ID, file string ID, line}] 27 traceEvGomaxprocs = 4 // current value of GOMAXPROCS [timestamp, GOMAXPROCS, stack id] 28 traceEvProcStart = 5 // start of P [timestamp, thread id] 29 traceEvProcStop = 6 // stop of P [timestamp] 30 traceEvGCStart = 7 // GC start [timestamp, seq, stack id] 31 traceEvGCDone = 8 // GC done [timestamp] 32 traceEvGCSTWStart = 9 // GC STW start [timestamp, kind] 33 traceEvGCSTWDone = 10 // GC STW done [timestamp] 34 traceEvGCSweepStart = 11 // GC sweep start [timestamp, stack id] 35 traceEvGCSweepDone = 12 // GC sweep done [timestamp, swept, reclaimed] 36 traceEvGoCreate = 13 // goroutine creation [timestamp, new goroutine id, new stack id, stack id] 37 traceEvGoStart = 14 // goroutine starts running [timestamp, goroutine id, seq] 38 traceEvGoEnd = 15 // goroutine ends [timestamp] 39 traceEvGoStop = 16 // goroutine stops (like in select{}) [timestamp, stack] 40 traceEvGoSched = 17 // goroutine calls Gosched [timestamp, stack] 41 traceEvGoPreempt = 18 // goroutine is preempted [timestamp, stack] 42 traceEvGoSleep = 19 // goroutine calls Sleep [timestamp, stack] 43 traceEvGoBlock = 20 // goroutine blocks [timestamp, stack] 44 traceEvGoUnblock = 21 // goroutine is unblocked [timestamp, goroutine id, seq, stack] 45 traceEvGoBlockSend = 22 // goroutine blocks on chan send [timestamp, stack] 46 traceEvGoBlockRecv = 23 // goroutine blocks on chan recv [timestamp, stack] 47 traceEvGoBlockSelect = 24 // goroutine blocks on select [timestamp, stack] 48 traceEvGoBlockSync = 25 // goroutine blocks on Mutex/RWMutex [timestamp, stack] 49 traceEvGoBlockCond = 26 // goroutine blocks on Cond [timestamp, stack] 50 traceEvGoBlockNet = 27 // goroutine blocks on network [timestamp, stack] 51 traceEvGoSysCall = 28 // syscall enter [timestamp, stack] 52 traceEvGoSysExit = 29 // syscall exit [timestamp, goroutine id, seq, real timestamp] 53 traceEvGoSysBlock = 30 // syscall blocks [timestamp] 54 traceEvGoWaiting = 31 // denotes that goroutine is blocked when tracing starts [timestamp, goroutine id] 55 traceEvGoInSyscall = 32 // denotes that goroutine is in syscall when tracing starts [timestamp, goroutine id] 56 traceEvHeapAlloc = 33 // gcController.heapLive change [timestamp, heap_alloc] 57 traceEvHeapGoal = 34 // gcController.heapGoal (formerly next_gc) change [timestamp, heap goal in bytes] 58 traceEvTimerGoroutine = 35 // not currently used; previously denoted timer goroutine [timer goroutine id] 59 traceEvFutileWakeup = 36 // denotes that the previous wakeup of this goroutine was futile [timestamp] 60 traceEvString = 37 // string dictionary entry [ID, length, string] 61 traceEvGoStartLocal = 38 // goroutine starts running on the same P as the last event [timestamp, goroutine id] 62 traceEvGoUnblockLocal = 39 // goroutine is unblocked on the same P as the last event [timestamp, goroutine id, stack] 63 traceEvGoSysExitLocal = 40 // syscall exit on the same P as the last event [timestamp, goroutine id, real timestamp] 64 traceEvGoStartLabel = 41 // goroutine starts running with label [timestamp, goroutine id, seq, label string id] 65 traceEvGoBlockGC = 42 // goroutine blocks on GC assist [timestamp, stack] 66 traceEvGCMarkAssistStart = 43 // GC mark assist start [timestamp, stack] 67 traceEvGCMarkAssistDone = 44 // GC mark assist done [timestamp] 68 traceEvUserTaskCreate = 45 // trace.NewContext [timestamp, internal task id, internal parent task id, stack, name string] 69 traceEvUserTaskEnd = 46 // end of a task [timestamp, internal task id, stack] 70 traceEvUserRegion = 47 // trace.WithRegion [timestamp, internal task id, mode(0:start, 1:end), stack, name string] 71 traceEvUserLog = 48 // trace.Log [timestamp, internal task id, key string id, stack, value string] 72 traceEvCount = 49 73 // Byte is used but only 6 bits are available for event type. 74 // The remaining 2 bits are used to specify the number of arguments. 75 // That means, the max event type value is 63. 76 ) 77 78 const ( 79 // Timestamps in trace are cputicks/traceTickDiv. 80 // This makes absolute values of timestamp diffs smaller, 81 // and so they are encoded in less number of bytes. 82 // 64 on x86 is somewhat arbitrary (one tick is ~20ns on a 3GHz machine). 83 // The suggested increment frequency for PowerPC's time base register is 84 // 512 MHz according to Power ISA v2.07 section 6.2, so we use 16 on ppc64 85 // and ppc64le. 86 // Tracing won't work reliably for architectures where cputicks is emulated 87 // by nanotime, so the value doesn't matter for those architectures. 88 traceTickDiv = 16 + 48*(sys.Goarch386|sys.GoarchAmd64) 89 // Maximum number of PCs in a single stack trace. 90 // Since events contain only stack id rather than whole stack trace, 91 // we can allow quite large values here. 92 traceStackSize = 128 93 // Identifier of a fake P that is used when we trace without a real P. 94 traceGlobProc = -1 95 // Maximum number of bytes to encode uint64 in base-128. 96 traceBytesPerNumber = 10 97 // Shift of the number of arguments in the first event byte. 98 traceArgCountShift = 6 99 // Flag passed to traceGoPark to denote that the previous wakeup of this 100 // goroutine was futile. For example, a goroutine was unblocked on a mutex, 101 // but another goroutine got ahead and acquired the mutex before the first 102 // goroutine is scheduled, so the first goroutine has to block again. 103 // Such wakeups happen on buffered channels and sync.Mutex, 104 // but are generally not interesting for end user. 105 traceFutileWakeup byte = 128 106 ) 107 108 // trace is global tracing context. 109 var trace struct { 110 lock mutex // protects the following members 111 lockOwner *g // to avoid deadlocks during recursive lock locks 112 enabled bool // when set runtime traces events 113 shutdown bool // set when we are waiting for trace reader to finish after setting enabled to false 114 headerWritten bool // whether ReadTrace has emitted trace header 115 footerWritten bool // whether ReadTrace has emitted trace footer 116 shutdownSema uint32 // used to wait for ReadTrace completion 117 seqStart uint64 // sequence number when tracing was started 118 ticksStart int64 // cputicks when tracing was started 119 ticksEnd int64 // cputicks when tracing was stopped 120 timeStart int64 // nanotime when tracing was started 121 timeEnd int64 // nanotime when tracing was stopped 122 seqGC uint64 // GC start/done sequencer 123 reading traceBufPtr // buffer currently handed off to user 124 empty traceBufPtr // stack of empty buffers 125 fullHead traceBufPtr // queue of full buffers 126 fullTail traceBufPtr 127 reader guintptr // goroutine that called ReadTrace, or nil 128 stackTab traceStackTable // maps stack traces to unique ids 129 130 // Dictionary for traceEvString. 131 // 132 // TODO: central lock to access the map is not ideal. 133 // option: pre-assign ids to all user annotation region names and tags 134 // option: per-P cache 135 // option: sync.Map like data structure 136 stringsLock mutex 137 strings map[string]uint64 138 stringSeq uint64 139 140 // markWorkerLabels maps gcMarkWorkerMode to string ID. 141 markWorkerLabels [len(gcMarkWorkerModeStrings)]uint64 142 143 bufLock mutex // protects buf 144 buf traceBufPtr // global trace buffer, used when running without a p 145 } 146 147 // traceBufHeader is per-P tracing buffer. 148 type traceBufHeader struct { 149 link traceBufPtr // in trace.empty/full 150 lastTicks uint64 // when we wrote the last event 151 pos int // next write offset in arr 152 stk [traceStackSize]uintptr // scratch buffer for traceback 153 } 154 155 // traceBuf is per-P tracing buffer. 156 // 157 //go:notinheap 158 type traceBuf struct { 159 traceBufHeader 160 arr [64<<10 - unsafe.Sizeof(traceBufHeader{})]byte // underlying buffer for traceBufHeader.buf 161 } 162 163 // traceBufPtr is a *traceBuf that is not traced by the garbage 164 // collector and doesn't have write barriers. traceBufs are not 165 // allocated from the GC'd heap, so this is safe, and are often 166 // manipulated in contexts where write barriers are not allowed, so 167 // this is necessary. 168 // 169 // TODO: Since traceBuf is now go:notinheap, this isn't necessary. 170 type traceBufPtr uintptr 171 172 func (tp traceBufPtr) ptr() *traceBuf { return (*traceBuf)(unsafe.Pointer(tp)) } 173 func (tp *traceBufPtr) set(b *traceBuf) { *tp = traceBufPtr(unsafe.Pointer(b)) } 174 func traceBufPtrOf(b *traceBuf) traceBufPtr { 175 return traceBufPtr(unsafe.Pointer(b)) 176 } 177 178 // StartTrace enables tracing for the current process. 179 // While tracing, the data will be buffered and available via ReadTrace. 180 // StartTrace returns an error if tracing is already enabled. 181 // Most clients should use the runtime/trace package or the testing package's 182 // -test.trace flag instead of calling StartTrace directly. 183 func StartTrace() error { 184 // Stop the world so that we can take a consistent snapshot 185 // of all goroutines at the beginning of the trace. 186 // Do not stop the world during GC so we ensure we always see 187 // a consistent view of GC-related events (e.g. a start is always 188 // paired with an end). 189 stopTheWorldGC("start tracing") 190 191 // Prevent sysmon from running any code that could generate events. 192 lock(&sched.sysmonlock) 193 194 // We are in stop-the-world, but syscalls can finish and write to trace concurrently. 195 // Exitsyscall could check trace.enabled long before and then suddenly wake up 196 // and decide to write to trace at a random point in time. 197 // However, such syscall will use the global trace.buf buffer, because we've 198 // acquired all p's by doing stop-the-world. So this protects us from such races. 199 lock(&trace.bufLock) 200 201 if trace.enabled || trace.shutdown { 202 unlock(&trace.bufLock) 203 unlock(&sched.sysmonlock) 204 startTheWorldGC() 205 return errorString("tracing is already enabled") 206 } 207 208 // Can't set trace.enabled yet. While the world is stopped, exitsyscall could 209 // already emit a delayed event (see exitTicks in exitsyscall) if we set trace.enabled here. 210 // That would lead to an inconsistent trace: 211 // - either GoSysExit appears before EvGoInSyscall, 212 // - or GoSysExit appears for a goroutine for which we don't emit EvGoInSyscall below. 213 // To instruct traceEvent that it must not ignore events below, we set startingtrace. 214 // trace.enabled is set afterwards once we have emitted all preliminary events. 215 _g_ := getg() 216 _g_.m.startingtrace = true 217 218 // Obtain current stack ID to use in all traceEvGoCreate events below. 219 mp := acquirem() 220 stkBuf := make([]uintptr, traceStackSize) 221 stackID := traceStackID(mp, stkBuf, 2) 222 releasem(mp) 223 224 // World is stopped, no need to lock. 225 forEachGRace(func(gp *g) { 226 status := readgstatus(gp) 227 if status != _Gdead { 228 gp.traceseq = 0 229 gp.tracelastp = getg().m.p 230 // +PCQuantum because traceFrameForPC expects return PCs and subtracts PCQuantum. 231 id := trace.stackTab.put([]uintptr{gp.startpc + sys.PCQuantum}) 232 traceEvent(traceEvGoCreate, -1, uint64(gp.goid), uint64(id), stackID) 233 } 234 if status == _Gwaiting { 235 // traceEvGoWaiting is implied to have seq=1. 236 gp.traceseq++ 237 traceEvent(traceEvGoWaiting, -1, uint64(gp.goid)) 238 } 239 if status == _Gsyscall { 240 gp.traceseq++ 241 traceEvent(traceEvGoInSyscall, -1, uint64(gp.goid)) 242 } else { 243 gp.sysblocktraced = false 244 } 245 }) 246 traceProcStart() 247 traceGoStart() 248 // Note: ticksStart needs to be set after we emit traceEvGoInSyscall events. 249 // If we do it the other way around, it is possible that exitsyscall will 250 // query sysexitticks after ticksStart but before traceEvGoInSyscall timestamp. 251 // It will lead to a false conclusion that cputicks is broken. 252 trace.ticksStart = cputicks() 253 trace.timeStart = nanotime() 254 trace.headerWritten = false 255 trace.footerWritten = false 256 257 // string to id mapping 258 // 0 : reserved for an empty string 259 // remaining: other strings registered by traceString 260 trace.stringSeq = 0 261 trace.strings = make(map[string]uint64) 262 263 trace.seqGC = 0 264 _g_.m.startingtrace = false 265 trace.enabled = true 266 267 // Register runtime goroutine labels. 268 _, pid, bufp := traceAcquireBuffer() 269 for i, label := range gcMarkWorkerModeStrings[:] { 270 trace.markWorkerLabels[i], bufp = traceString(bufp, pid, label) 271 } 272 traceReleaseBuffer(pid) 273 274 unlock(&trace.bufLock) 275 276 unlock(&sched.sysmonlock) 277 278 startTheWorldGC() 279 return nil 280 } 281 282 // StopTrace stops tracing, if it was previously enabled. 283 // StopTrace only returns after all the reads for the trace have completed. 284 func StopTrace() { 285 // Stop the world so that we can collect the trace buffers from all p's below, 286 // and also to avoid races with traceEvent. 287 stopTheWorldGC("stop tracing") 288 289 // See the comment in StartTrace. 290 lock(&sched.sysmonlock) 291 292 // See the comment in StartTrace. 293 lock(&trace.bufLock) 294 295 if !trace.enabled { 296 unlock(&trace.bufLock) 297 unlock(&sched.sysmonlock) 298 startTheWorldGC() 299 return 300 } 301 302 traceGoSched() 303 304 // Loop over all allocated Ps because dead Ps may still have 305 // trace buffers. 306 for _, p := range allp[:cap(allp)] { 307 buf := p.tracebuf 308 if buf != 0 { 309 traceFullQueue(buf) 310 p.tracebuf = 0 311 } 312 } 313 if trace.buf != 0 { 314 buf := trace.buf 315 trace.buf = 0 316 if buf.ptr().pos != 0 { 317 traceFullQueue(buf) 318 } 319 } 320 321 for { 322 trace.ticksEnd = cputicks() 323 trace.timeEnd = nanotime() 324 // Windows time can tick only every 15ms, wait for at least one tick. 325 if trace.timeEnd != trace.timeStart { 326 break 327 } 328 osyield() 329 } 330 331 trace.enabled = false 332 trace.shutdown = true 333 unlock(&trace.bufLock) 334 335 unlock(&sched.sysmonlock) 336 337 startTheWorldGC() 338 339 // The world is started but we've set trace.shutdown, so new tracing can't start. 340 // Wait for the trace reader to flush pending buffers and stop. 341 semacquire(&trace.shutdownSema) 342 if raceenabled { 343 raceacquire(unsafe.Pointer(&trace.shutdownSema)) 344 } 345 346 // The lock protects us from races with StartTrace/StopTrace because they do stop-the-world. 347 lock(&trace.lock) 348 for _, p := range allp[:cap(allp)] { 349 if p.tracebuf != 0 { 350 throw("trace: non-empty trace buffer in proc") 351 } 352 } 353 if trace.buf != 0 { 354 throw("trace: non-empty global trace buffer") 355 } 356 if trace.fullHead != 0 || trace.fullTail != 0 { 357 throw("trace: non-empty full trace buffer") 358 } 359 if trace.reading != 0 || trace.reader != 0 { 360 throw("trace: reading after shutdown") 361 } 362 for trace.empty != 0 { 363 buf := trace.empty 364 trace.empty = buf.ptr().link 365 sysFree(unsafe.Pointer(buf), unsafe.Sizeof(*buf.ptr()), &memstats.other_sys) 366 } 367 trace.strings = nil 368 trace.shutdown = false 369 unlock(&trace.lock) 370 } 371 372 // ReadTrace returns the next chunk of binary tracing data, blocking until data 373 // is available. If tracing is turned off and all the data accumulated while it 374 // was on has been returned, ReadTrace returns nil. The caller must copy the 375 // returned data before calling ReadTrace again. 376 // ReadTrace must be called from one goroutine at a time. 377 func ReadTrace() []byte { 378 // This function may need to lock trace.lock recursively 379 // (goparkunlock -> traceGoPark -> traceEvent -> traceFlush). 380 // To allow this we use trace.lockOwner. 381 // Also this function must not allocate while holding trace.lock: 382 // allocation can call heap allocate, which will try to emit a trace 383 // event while holding heap lock. 384 lock(&trace.lock) 385 trace.lockOwner = getg() 386 387 if trace.reader != 0 { 388 // More than one goroutine reads trace. This is bad. 389 // But we rather do not crash the program because of tracing, 390 // because tracing can be enabled at runtime on prod servers. 391 trace.lockOwner = nil 392 unlock(&trace.lock) 393 println("runtime: ReadTrace called from multiple goroutines simultaneously") 394 return nil 395 } 396 // Recycle the old buffer. 397 if buf := trace.reading; buf != 0 { 398 buf.ptr().link = trace.empty 399 trace.empty = buf 400 trace.reading = 0 401 } 402 // Write trace header. 403 if !trace.headerWritten { 404 trace.headerWritten = true 405 trace.lockOwner = nil 406 unlock(&trace.lock) 407 return []byte("go 1.11 trace\x00\x00\x00") 408 } 409 // Wait for new data. 410 if trace.fullHead == 0 && !trace.shutdown { 411 trace.reader.set(getg()) 412 goparkunlock(&trace.lock, waitReasonTraceReaderBlocked, traceEvGoBlock, 2) 413 lock(&trace.lock) 414 } 415 // Write a buffer. 416 if trace.fullHead != 0 { 417 buf := traceFullDequeue() 418 trace.reading = buf 419 trace.lockOwner = nil 420 unlock(&trace.lock) 421 return buf.ptr().arr[:buf.ptr().pos] 422 } 423 // Write footer with timer frequency. 424 if !trace.footerWritten { 425 trace.footerWritten = true 426 // Use float64 because (trace.ticksEnd - trace.ticksStart) * 1e9 can overflow int64. 427 freq := float64(trace.ticksEnd-trace.ticksStart) * 1e9 / float64(trace.timeEnd-trace.timeStart) / traceTickDiv 428 trace.lockOwner = nil 429 unlock(&trace.lock) 430 var data []byte 431 data = append(data, traceEvFrequency|0<<traceArgCountShift) 432 data = traceAppend(data, uint64(freq)) 433 // This will emit a bunch of full buffers, we will pick them up 434 // on the next iteration. 435 trace.stackTab.dump() 436 return data 437 } 438 // Done. 439 if trace.shutdown { 440 trace.lockOwner = nil 441 unlock(&trace.lock) 442 if raceenabled { 443 // Model synchronization on trace.shutdownSema, which race 444 // detector does not see. This is required to avoid false 445 // race reports on writer passed to trace.Start. 446 racerelease(unsafe.Pointer(&trace.shutdownSema)) 447 } 448 // trace.enabled is already reset, so can call traceable functions. 449 semrelease(&trace.shutdownSema) 450 return nil 451 } 452 // Also bad, but see the comment above. 453 trace.lockOwner = nil 454 unlock(&trace.lock) 455 println("runtime: spurious wakeup of trace reader") 456 return nil 457 } 458 459 // traceReader returns the trace reader that should be woken up, if any. 460 func traceReader() *g { 461 if trace.reader == 0 || (trace.fullHead == 0 && !trace.shutdown) { 462 return nil 463 } 464 lock(&trace.lock) 465 if trace.reader == 0 || (trace.fullHead == 0 && !trace.shutdown) { 466 unlock(&trace.lock) 467 return nil 468 } 469 gp := trace.reader.ptr() 470 trace.reader.set(nil) 471 unlock(&trace.lock) 472 return gp 473 } 474 475 // traceProcFree frees trace buffer associated with pp. 476 func traceProcFree(pp *p) { 477 buf := pp.tracebuf 478 pp.tracebuf = 0 479 if buf == 0 { 480 return 481 } 482 lock(&trace.lock) 483 traceFullQueue(buf) 484 unlock(&trace.lock) 485 } 486 487 // traceFullQueue queues buf into queue of full buffers. 488 func traceFullQueue(buf traceBufPtr) { 489 buf.ptr().link = 0 490 if trace.fullHead == 0 { 491 trace.fullHead = buf 492 } else { 493 trace.fullTail.ptr().link = buf 494 } 495 trace.fullTail = buf 496 } 497 498 // traceFullDequeue dequeues from queue of full buffers. 499 func traceFullDequeue() traceBufPtr { 500 buf := trace.fullHead 501 if buf == 0 { 502 return 0 503 } 504 trace.fullHead = buf.ptr().link 505 if trace.fullHead == 0 { 506 trace.fullTail = 0 507 } 508 buf.ptr().link = 0 509 return buf 510 } 511 512 // traceEvent writes a single event to trace buffer, flushing the buffer if necessary. 513 // ev is event type. 514 // If skip > 0, write current stack id as the last argument (skipping skip top frames). 515 // If skip = 0, this event type should contain a stack, but we don't want 516 // to collect and remember it for this particular call. 517 func traceEvent(ev byte, skip int, args ...uint64) { 518 mp, pid, bufp := traceAcquireBuffer() 519 // Double-check trace.enabled now that we've done m.locks++ and acquired bufLock. 520 // This protects from races between traceEvent and StartTrace/StopTrace. 521 522 // The caller checked that trace.enabled == true, but trace.enabled might have been 523 // turned off between the check and now. Check again. traceLockBuffer did mp.locks++, 524 // StopTrace does stopTheWorld, and stopTheWorld waits for mp.locks to go back to zero, 525 // so if we see trace.enabled == true now, we know it's true for the rest of the function. 526 // Exitsyscall can run even during stopTheWorld. The race with StartTrace/StopTrace 527 // during tracing in exitsyscall is resolved by locking trace.bufLock in traceLockBuffer. 528 // 529 // Note trace_userTaskCreate runs the same check. 530 if !trace.enabled && !mp.startingtrace { 531 traceReleaseBuffer(pid) 532 return 533 } 534 535 if skip > 0 { 536 if getg() == mp.curg { 537 skip++ // +1 because stack is captured in traceEventLocked. 538 } 539 } 540 traceEventLocked(0, mp, pid, bufp, ev, skip, args...) 541 traceReleaseBuffer(pid) 542 } 543 544 func traceEventLocked(extraBytes int, mp *m, pid int32, bufp *traceBufPtr, ev byte, skip int, args ...uint64) { 545 buf := bufp.ptr() 546 // TODO: test on non-zero extraBytes param. 547 maxSize := 2 + 5*traceBytesPerNumber + extraBytes // event type, length, sequence, timestamp, stack id and two add params 548 if buf == nil || len(buf.arr)-buf.pos < maxSize { 549 buf = traceFlush(traceBufPtrOf(buf), pid).ptr() 550 bufp.set(buf) 551 } 552 553 ticks := uint64(cputicks()) / traceTickDiv 554 tickDiff := ticks - buf.lastTicks 555 buf.lastTicks = ticks 556 narg := byte(len(args)) 557 if skip >= 0 { 558 narg++ 559 } 560 // We have only 2 bits for number of arguments. 561 // If number is >= 3, then the event type is followed by event length in bytes. 562 if narg > 3 { 563 narg = 3 564 } 565 startPos := buf.pos 566 buf.byte(ev | narg<<traceArgCountShift) 567 var lenp *byte 568 if narg == 3 { 569 // Reserve the byte for length assuming that length < 128. 570 buf.varint(0) 571 lenp = &buf.arr[buf.pos-1] 572 } 573 buf.varint(tickDiff) 574 for _, a := range args { 575 buf.varint(a) 576 } 577 if skip == 0 { 578 buf.varint(0) 579 } else if skip > 0 { 580 buf.varint(traceStackID(mp, buf.stk[:], skip)) 581 } 582 evSize := buf.pos - startPos 583 if evSize > maxSize { 584 throw("invalid length of trace event") 585 } 586 if lenp != nil { 587 // Fill in actual length. 588 *lenp = byte(evSize - 2) 589 } 590 } 591 592 func traceStackID(mp *m, buf []uintptr, skip int) uint64 { 593 _g_ := getg() 594 gp := mp.curg 595 var nstk int 596 if gp == _g_ { 597 nstk = callers(skip+1, buf) 598 } else if gp != nil { 599 gp = mp.curg 600 nstk = gcallers(gp, skip, buf) 601 } 602 if nstk > 0 { 603 nstk-- // skip runtime.goexit 604 } 605 if nstk > 0 && gp.goid == 1 { 606 nstk-- // skip runtime.main 607 } 608 id := trace.stackTab.put(buf[:nstk]) 609 return uint64(id) 610 } 611 612 // traceAcquireBuffer returns trace buffer to use and, if necessary, locks it. 613 func traceAcquireBuffer() (mp *m, pid int32, bufp *traceBufPtr) { 614 mp = acquirem() 615 if p := mp.p.ptr(); p != nil { 616 return mp, p.id, &p.tracebuf 617 } 618 lock(&trace.bufLock) 619 return mp, traceGlobProc, &trace.buf 620 } 621 622 // traceReleaseBuffer releases a buffer previously acquired with traceAcquireBuffer. 623 func traceReleaseBuffer(pid int32) { 624 if pid == traceGlobProc { 625 unlock(&trace.bufLock) 626 } 627 releasem(getg().m) 628 } 629 630 // traceFlush puts buf onto stack of full buffers and returns an empty buffer. 631 func traceFlush(buf traceBufPtr, pid int32) traceBufPtr { 632 owner := trace.lockOwner 633 dolock := owner == nil || owner != getg().m.curg 634 if dolock { 635 lock(&trace.lock) 636 } 637 if buf != 0 { 638 traceFullQueue(buf) 639 } 640 if trace.empty != 0 { 641 buf = trace.empty 642 trace.empty = buf.ptr().link 643 } else { 644 buf = traceBufPtr(sysAlloc(unsafe.Sizeof(traceBuf{}), &memstats.other_sys)) 645 if buf == 0 { 646 throw("trace: out of memory") 647 } 648 } 649 bufp := buf.ptr() 650 bufp.link.set(nil) 651 bufp.pos = 0 652 653 // initialize the buffer for a new batch 654 ticks := uint64(cputicks()) / traceTickDiv 655 bufp.lastTicks = ticks 656 bufp.byte(traceEvBatch | 1<<traceArgCountShift) 657 bufp.varint(uint64(pid)) 658 bufp.varint(ticks) 659 660 if dolock { 661 unlock(&trace.lock) 662 } 663 return buf 664 } 665 666 // traceString adds a string to the trace.strings and returns the id. 667 func traceString(bufp *traceBufPtr, pid int32, s string) (uint64, *traceBufPtr) { 668 if s == "" { 669 return 0, bufp 670 } 671 672 lock(&trace.stringsLock) 673 if raceenabled { 674 // raceacquire is necessary because the map access 675 // below is race annotated. 676 raceacquire(unsafe.Pointer(&trace.stringsLock)) 677 } 678 679 if id, ok := trace.strings[s]; ok { 680 if raceenabled { 681 racerelease(unsafe.Pointer(&trace.stringsLock)) 682 } 683 unlock(&trace.stringsLock) 684 685 return id, bufp 686 } 687 688 trace.stringSeq++ 689 id := trace.stringSeq 690 trace.strings[s] = id 691 692 if raceenabled { 693 racerelease(unsafe.Pointer(&trace.stringsLock)) 694 } 695 unlock(&trace.stringsLock) 696 697 // memory allocation in above may trigger tracing and 698 // cause *bufp changes. Following code now works with *bufp, 699 // so there must be no memory allocation or any activities 700 // that causes tracing after this point. 701 702 buf := bufp.ptr() 703 size := 1 + 2*traceBytesPerNumber + len(s) 704 if buf == nil || len(buf.arr)-buf.pos < size { 705 buf = traceFlush(traceBufPtrOf(buf), pid).ptr() 706 bufp.set(buf) 707 } 708 buf.byte(traceEvString) 709 buf.varint(id) 710 711 // double-check the string and the length can fit. 712 // Otherwise, truncate the string. 713 slen := len(s) 714 if room := len(buf.arr) - buf.pos; room < slen+traceBytesPerNumber { 715 slen = room 716 } 717 718 buf.varint(uint64(slen)) 719 buf.pos += copy(buf.arr[buf.pos:], s[:slen]) 720 721 bufp.set(buf) 722 return id, bufp 723 } 724 725 // traceAppend appends v to buf in little-endian-base-128 encoding. 726 func traceAppend(buf []byte, v uint64) []byte { 727 for ; v >= 0x80; v >>= 7 { 728 buf = append(buf, 0x80|byte(v)) 729 } 730 buf = append(buf, byte(v)) 731 return buf 732 } 733 734 // varint appends v to buf in little-endian-base-128 encoding. 735 func (buf *traceBuf) varint(v uint64) { 736 pos := buf.pos 737 for ; v >= 0x80; v >>= 7 { 738 buf.arr[pos] = 0x80 | byte(v) 739 pos++ 740 } 741 buf.arr[pos] = byte(v) 742 pos++ 743 buf.pos = pos 744 } 745 746 // byte appends v to buf. 747 func (buf *traceBuf) byte(v byte) { 748 buf.arr[buf.pos] = v 749 buf.pos++ 750 } 751 752 // traceStackTable maps stack traces (arrays of PC's) to unique uint32 ids. 753 // It is lock-free for reading. 754 type traceStackTable struct { 755 lock mutex 756 seq uint32 757 mem traceAlloc 758 tab [1 << 13]traceStackPtr 759 } 760 761 // traceStack is a single stack in traceStackTable. 762 type traceStack struct { 763 link traceStackPtr 764 hash uintptr 765 id uint32 766 n int 767 stk [0]uintptr // real type [n]uintptr 768 } 769 770 type traceStackPtr uintptr 771 772 func (tp traceStackPtr) ptr() *traceStack { return (*traceStack)(unsafe.Pointer(tp)) } 773 774 // stack returns slice of PCs. 775 func (ts *traceStack) stack() []uintptr { 776 return (*[traceStackSize]uintptr)(unsafe.Pointer(&ts.stk))[:ts.n] 777 } 778 779 // put returns a unique id for the stack trace pcs and caches it in the table, 780 // if it sees the trace for the first time. 781 func (tab *traceStackTable) put(pcs []uintptr) uint32 { 782 if len(pcs) == 0 { 783 return 0 784 } 785 hash := memhash(unsafe.Pointer(&pcs[0]), 0, uintptr(len(pcs))*unsafe.Sizeof(pcs[0])) 786 // First, search the hashtable w/o the mutex. 787 if id := tab.find(pcs, hash); id != 0 { 788 return id 789 } 790 // Now, double check under the mutex. 791 lock(&tab.lock) 792 if id := tab.find(pcs, hash); id != 0 { 793 unlock(&tab.lock) 794 return id 795 } 796 // Create new record. 797 tab.seq++ 798 stk := tab.newStack(len(pcs)) 799 stk.hash = hash 800 stk.id = tab.seq 801 stk.n = len(pcs) 802 stkpc := stk.stack() 803 for i, pc := range pcs { 804 stkpc[i] = pc 805 } 806 part := int(hash % uintptr(len(tab.tab))) 807 stk.link = tab.tab[part] 808 atomicstorep(unsafe.Pointer(&tab.tab[part]), unsafe.Pointer(stk)) 809 unlock(&tab.lock) 810 return stk.id 811 } 812 813 // find checks if the stack trace pcs is already present in the table. 814 func (tab *traceStackTable) find(pcs []uintptr, hash uintptr) uint32 { 815 part := int(hash % uintptr(len(tab.tab))) 816 Search: 817 for stk := tab.tab[part].ptr(); stk != nil; stk = stk.link.ptr() { 818 if stk.hash == hash && stk.n == len(pcs) { 819 for i, stkpc := range stk.stack() { 820 if stkpc != pcs[i] { 821 continue Search 822 } 823 } 824 return stk.id 825 } 826 } 827 return 0 828 } 829 830 // newStack allocates a new stack of size n. 831 func (tab *traceStackTable) newStack(n int) *traceStack { 832 return (*traceStack)(tab.mem.alloc(unsafe.Sizeof(traceStack{}) + uintptr(n)*sys.PtrSize)) 833 } 834 835 // allFrames returns all of the Frames corresponding to pcs. 836 func allFrames(pcs []uintptr) []Frame { 837 frames := make([]Frame, 0, len(pcs)) 838 ci := CallersFrames(pcs) 839 for { 840 f, more := ci.Next() 841 frames = append(frames, f) 842 if !more { 843 return frames 844 } 845 } 846 } 847 848 // dump writes all previously cached stacks to trace buffers, 849 // releases all memory and resets state. 850 func (tab *traceStackTable) dump() { 851 var tmp [(2 + 4*traceStackSize) * traceBytesPerNumber]byte 852 bufp := traceFlush(0, 0) 853 for _, stk := range tab.tab { 854 stk := stk.ptr() 855 for ; stk != nil; stk = stk.link.ptr() { 856 tmpbuf := tmp[:0] 857 tmpbuf = traceAppend(tmpbuf, uint64(stk.id)) 858 frames := allFrames(stk.stack()) 859 tmpbuf = traceAppend(tmpbuf, uint64(len(frames))) 860 for _, f := range frames { 861 var frame traceFrame 862 frame, bufp = traceFrameForPC(bufp, 0, f) 863 tmpbuf = traceAppend(tmpbuf, uint64(f.PC)) 864 tmpbuf = traceAppend(tmpbuf, uint64(frame.funcID)) 865 tmpbuf = traceAppend(tmpbuf, uint64(frame.fileID)) 866 tmpbuf = traceAppend(tmpbuf, uint64(frame.line)) 867 } 868 // Now copy to the buffer. 869 size := 1 + traceBytesPerNumber + len(tmpbuf) 870 if buf := bufp.ptr(); len(buf.arr)-buf.pos < size { 871 bufp = traceFlush(bufp, 0) 872 } 873 buf := bufp.ptr() 874 buf.byte(traceEvStack | 3<<traceArgCountShift) 875 buf.varint(uint64(len(tmpbuf))) 876 buf.pos += copy(buf.arr[buf.pos:], tmpbuf) 877 } 878 } 879 880 lock(&trace.lock) 881 traceFullQueue(bufp) 882 unlock(&trace.lock) 883 884 tab.mem.drop() 885 *tab = traceStackTable{} 886 lockInit(&((*tab).lock), lockRankTraceStackTab) 887 } 888 889 type traceFrame struct { 890 funcID uint64 891 fileID uint64 892 line uint64 893 } 894 895 // traceFrameForPC records the frame information. 896 // It may allocate memory. 897 func traceFrameForPC(buf traceBufPtr, pid int32, f Frame) (traceFrame, traceBufPtr) { 898 bufp := &buf 899 var frame traceFrame 900 901 fn := f.Function 902 const maxLen = 1 << 10 903 if len(fn) > maxLen { 904 fn = fn[len(fn)-maxLen:] 905 } 906 frame.funcID, bufp = traceString(bufp, pid, fn) 907 frame.line = uint64(f.Line) 908 file := f.File 909 if len(file) > maxLen { 910 file = file[len(file)-maxLen:] 911 } 912 frame.fileID, bufp = traceString(bufp, pid, file) 913 return frame, (*bufp) 914 } 915 916 // traceAlloc is a non-thread-safe region allocator. 917 // It holds a linked list of traceAllocBlock. 918 type traceAlloc struct { 919 head traceAllocBlockPtr 920 off uintptr 921 } 922 923 // traceAllocBlock is a block in traceAlloc. 924 // 925 // traceAllocBlock is allocated from non-GC'd memory, so it must not 926 // contain heap pointers. Writes to pointers to traceAllocBlocks do 927 // not need write barriers. 928 // 929 //go:notinheap 930 type traceAllocBlock struct { 931 next traceAllocBlockPtr 932 data [64<<10 - sys.PtrSize]byte 933 } 934 935 // TODO: Since traceAllocBlock is now go:notinheap, this isn't necessary. 936 type traceAllocBlockPtr uintptr 937 938 func (p traceAllocBlockPtr) ptr() *traceAllocBlock { return (*traceAllocBlock)(unsafe.Pointer(p)) } 939 func (p *traceAllocBlockPtr) set(x *traceAllocBlock) { *p = traceAllocBlockPtr(unsafe.Pointer(x)) } 940 941 // alloc allocates n-byte block. 942 func (a *traceAlloc) alloc(n uintptr) unsafe.Pointer { 943 n = alignUp(n, sys.PtrSize) 944 if a.head == 0 || a.off+n > uintptr(len(a.head.ptr().data)) { 945 if n > uintptr(len(a.head.ptr().data)) { 946 throw("trace: alloc too large") 947 } 948 block := (*traceAllocBlock)(sysAlloc(unsafe.Sizeof(traceAllocBlock{}), &memstats.other_sys)) 949 if block == nil { 950 throw("trace: out of memory") 951 } 952 block.next.set(a.head.ptr()) 953 a.head.set(block) 954 a.off = 0 955 } 956 p := &a.head.ptr().data[a.off] 957 a.off += n 958 return unsafe.Pointer(p) 959 } 960 961 // drop frees all previously allocated memory and resets the allocator. 962 func (a *traceAlloc) drop() { 963 for a.head != 0 { 964 block := a.head.ptr() 965 a.head.set(block.next.ptr()) 966 sysFree(unsafe.Pointer(block), unsafe.Sizeof(traceAllocBlock{}), &memstats.other_sys) 967 } 968 } 969 970 // The following functions write specific events to trace. 971 972 func traceGomaxprocs(procs int32) { 973 traceEvent(traceEvGomaxprocs, 1, uint64(procs)) 974 } 975 976 func traceProcStart() { 977 traceEvent(traceEvProcStart, -1, uint64(getg().m.id)) 978 } 979 980 func traceProcStop(pp *p) { 981 // Sysmon and stopTheWorld can stop Ps blocked in syscalls, 982 // to handle this we temporary employ the P. 983 mp := acquirem() 984 oldp := mp.p 985 mp.p.set(pp) 986 traceEvent(traceEvProcStop, -1) 987 mp.p = oldp 988 releasem(mp) 989 } 990 991 func traceGCStart() { 992 traceEvent(traceEvGCStart, 3, trace.seqGC) 993 trace.seqGC++ 994 } 995 996 func traceGCDone() { 997 traceEvent(traceEvGCDone, -1) 998 } 999 1000 func traceGCSTWStart(kind int) { 1001 traceEvent(traceEvGCSTWStart, -1, uint64(kind)) 1002 } 1003 1004 func traceGCSTWDone() { 1005 traceEvent(traceEvGCSTWDone, -1) 1006 } 1007 1008 // traceGCSweepStart prepares to trace a sweep loop. This does not 1009 // emit any events until traceGCSweepSpan is called. 1010 // 1011 // traceGCSweepStart must be paired with traceGCSweepDone and there 1012 // must be no preemption points between these two calls. 1013 func traceGCSweepStart() { 1014 // Delay the actual GCSweepStart event until the first span 1015 // sweep. If we don't sweep anything, don't emit any events. 1016 _p_ := getg().m.p.ptr() 1017 if _p_.traceSweep { 1018 throw("double traceGCSweepStart") 1019 } 1020 _p_.traceSweep, _p_.traceSwept, _p_.traceReclaimed = true, 0, 0 1021 } 1022 1023 // traceGCSweepSpan traces the sweep of a single page. 1024 // 1025 // This may be called outside a traceGCSweepStart/traceGCSweepDone 1026 // pair; however, it will not emit any trace events in this case. 1027 func traceGCSweepSpan(bytesSwept uintptr) { 1028 _p_ := getg().m.p.ptr() 1029 if _p_.traceSweep { 1030 if _p_.traceSwept == 0 { 1031 traceEvent(traceEvGCSweepStart, 1) 1032 } 1033 _p_.traceSwept += bytesSwept 1034 } 1035 } 1036 1037 func traceGCSweepDone() { 1038 _p_ := getg().m.p.ptr() 1039 if !_p_.traceSweep { 1040 throw("missing traceGCSweepStart") 1041 } 1042 if _p_.traceSwept != 0 { 1043 traceEvent(traceEvGCSweepDone, -1, uint64(_p_.traceSwept), uint64(_p_.traceReclaimed)) 1044 } 1045 _p_.traceSweep = false 1046 } 1047 1048 func traceGCMarkAssistStart() { 1049 traceEvent(traceEvGCMarkAssistStart, 1) 1050 } 1051 1052 func traceGCMarkAssistDone() { 1053 traceEvent(traceEvGCMarkAssistDone, -1) 1054 } 1055 1056 func traceGoCreate(newg *g, pc uintptr) { 1057 newg.traceseq = 0 1058 newg.tracelastp = getg().m.p 1059 // +PCQuantum because traceFrameForPC expects return PCs and subtracts PCQuantum. 1060 id := trace.stackTab.put([]uintptr{pc + sys.PCQuantum}) 1061 traceEvent(traceEvGoCreate, 2, uint64(newg.goid), uint64(id)) 1062 } 1063 1064 func traceGoStart() { 1065 _g_ := getg().m.curg 1066 _p_ := _g_.m.p 1067 _g_.traceseq++ 1068 if _p_.ptr().gcMarkWorkerMode != gcMarkWorkerNotWorker { 1069 traceEvent(traceEvGoStartLabel, -1, uint64(_g_.goid), _g_.traceseq, trace.markWorkerLabels[_p_.ptr().gcMarkWorkerMode]) 1070 } else if _g_.tracelastp == _p_ { 1071 traceEvent(traceEvGoStartLocal, -1, uint64(_g_.goid)) 1072 } else { 1073 _g_.tracelastp = _p_ 1074 traceEvent(traceEvGoStart, -1, uint64(_g_.goid), _g_.traceseq) 1075 } 1076 } 1077 1078 func traceGoEnd() { 1079 traceEvent(traceEvGoEnd, -1) 1080 } 1081 1082 func traceGoSched() { 1083 _g_ := getg() 1084 _g_.tracelastp = _g_.m.p 1085 traceEvent(traceEvGoSched, 1) 1086 } 1087 1088 func traceGoPreempt() { 1089 _g_ := getg() 1090 _g_.tracelastp = _g_.m.p 1091 traceEvent(traceEvGoPreempt, 1) 1092 } 1093 1094 func traceGoPark(traceEv byte, skip int) { 1095 if traceEv&traceFutileWakeup != 0 { 1096 traceEvent(traceEvFutileWakeup, -1) 1097 } 1098 traceEvent(traceEv & ^traceFutileWakeup, skip) 1099 } 1100 1101 func traceGoUnpark(gp *g, skip int) { 1102 _p_ := getg().m.p 1103 gp.traceseq++ 1104 if gp.tracelastp == _p_ { 1105 traceEvent(traceEvGoUnblockLocal, skip, uint64(gp.goid)) 1106 } else { 1107 gp.tracelastp = _p_ 1108 traceEvent(traceEvGoUnblock, skip, uint64(gp.goid), gp.traceseq) 1109 } 1110 } 1111 1112 func traceGoSysCall() { 1113 traceEvent(traceEvGoSysCall, 1) 1114 } 1115 1116 func traceGoSysExit(ts int64) { 1117 if ts != 0 && ts < trace.ticksStart { 1118 // There is a race between the code that initializes sysexitticks 1119 // (in exitsyscall, which runs without a P, and therefore is not 1120 // stopped with the rest of the world) and the code that initializes 1121 // a new trace. The recorded sysexitticks must therefore be treated 1122 // as "best effort". If they are valid for this trace, then great, 1123 // use them for greater accuracy. But if they're not valid for this 1124 // trace, assume that the trace was started after the actual syscall 1125 // exit (but before we actually managed to start the goroutine, 1126 // aka right now), and assign a fresh time stamp to keep the log consistent. 1127 ts = 0 1128 } 1129 _g_ := getg().m.curg 1130 _g_.traceseq++ 1131 _g_.tracelastp = _g_.m.p 1132 traceEvent(traceEvGoSysExit, -1, uint64(_g_.goid), _g_.traceseq, uint64(ts)/traceTickDiv) 1133 } 1134 1135 func traceGoSysBlock(pp *p) { 1136 // Sysmon and stopTheWorld can declare syscalls running on remote Ps as blocked, 1137 // to handle this we temporary employ the P. 1138 mp := acquirem() 1139 oldp := mp.p 1140 mp.p.set(pp) 1141 traceEvent(traceEvGoSysBlock, -1) 1142 mp.p = oldp 1143 releasem(mp) 1144 } 1145 1146 func traceHeapAlloc() { 1147 traceEvent(traceEvHeapAlloc, -1, gcController.heapLive) 1148 } 1149 1150 func traceHeapGoal() { 1151 if heapGoal := atomic.Load64(&gcController.heapGoal); heapGoal == ^uint64(0) { 1152 // Heap-based triggering is disabled. 1153 traceEvent(traceEvHeapGoal, -1, 0) 1154 } else { 1155 traceEvent(traceEvHeapGoal, -1, heapGoal) 1156 } 1157 } 1158 1159 // To access runtime functions from runtime/trace. 1160 // See runtime/trace/annotation.go 1161 1162 //go:linkname trace_userTaskCreate runtime/trace.userTaskCreate 1163 func trace_userTaskCreate(id, parentID uint64, taskType string) { 1164 if !trace.enabled { 1165 return 1166 } 1167 1168 // Same as in traceEvent. 1169 mp, pid, bufp := traceAcquireBuffer() 1170 if !trace.enabled && !mp.startingtrace { 1171 traceReleaseBuffer(pid) 1172 return 1173 } 1174 1175 typeStringID, bufp := traceString(bufp, pid, taskType) 1176 traceEventLocked(0, mp, pid, bufp, traceEvUserTaskCreate, 3, id, parentID, typeStringID) 1177 traceReleaseBuffer(pid) 1178 } 1179 1180 //go:linkname trace_userTaskEnd runtime/trace.userTaskEnd 1181 func trace_userTaskEnd(id uint64) { 1182 traceEvent(traceEvUserTaskEnd, 2, id) 1183 } 1184 1185 //go:linkname trace_userRegion runtime/trace.userRegion 1186 func trace_userRegion(id, mode uint64, name string) { 1187 if !trace.enabled { 1188 return 1189 } 1190 1191 mp, pid, bufp := traceAcquireBuffer() 1192 if !trace.enabled && !mp.startingtrace { 1193 traceReleaseBuffer(pid) 1194 return 1195 } 1196 1197 nameStringID, bufp := traceString(bufp, pid, name) 1198 traceEventLocked(0, mp, pid, bufp, traceEvUserRegion, 3, id, mode, nameStringID) 1199 traceReleaseBuffer(pid) 1200 } 1201 1202 //go:linkname trace_userLog runtime/trace.userLog 1203 func trace_userLog(id uint64, category, message string) { 1204 if !trace.enabled { 1205 return 1206 } 1207 1208 mp, pid, bufp := traceAcquireBuffer() 1209 if !trace.enabled && !mp.startingtrace { 1210 traceReleaseBuffer(pid) 1211 return 1212 } 1213 1214 categoryID, bufp := traceString(bufp, pid, category) 1215 1216 extraSpace := traceBytesPerNumber + len(message) // extraSpace for the value string 1217 traceEventLocked(extraSpace, mp, pid, bufp, traceEvUserLog, 3, id, categoryID) 1218 // traceEventLocked reserved extra space for val and len(val) 1219 // in buf, so buf now has room for the following. 1220 buf := bufp.ptr() 1221 1222 // double-check the message and its length can fit. 1223 // Otherwise, truncate the message. 1224 slen := len(message) 1225 if room := len(buf.arr) - buf.pos; room < slen+traceBytesPerNumber { 1226 slen = room 1227 } 1228 buf.varint(uint64(slen)) 1229 buf.pos += copy(buf.arr[buf.pos:], message[:slen]) 1230 1231 traceReleaseBuffer(pid) 1232 }