github.com/dannin/go@v0.0.0-20161031215817-d35dfd405eaa/src/cmd/trace/trace.go (about) 1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package main 6 7 import ( 8 "encoding/json" 9 "fmt" 10 "internal/trace" 11 "log" 12 "net/http" 13 "path/filepath" 14 "runtime" 15 "strconv" 16 "strings" 17 "time" 18 ) 19 20 func init() { 21 http.HandleFunc("/trace", httpTrace) 22 http.HandleFunc("/jsontrace", httpJsonTrace) 23 http.HandleFunc("/trace_viewer_html", httpTraceViewerHTML) 24 } 25 26 // httpTrace serves either whole trace (goid==0) or trace for goid goroutine. 27 func httpTrace(w http.ResponseWriter, r *http.Request) { 28 _, err := parseEvents() 29 if err != nil { 30 http.Error(w, err.Error(), http.StatusInternalServerError) 31 return 32 } 33 if err := r.ParseForm(); err != nil { 34 http.Error(w, err.Error(), http.StatusInternalServerError) 35 return 36 } 37 html := strings.Replace(templTrace, "{{PARAMS}}", r.Form.Encode(), -1) 38 w.Write([]byte(html)) 39 40 } 41 42 // See https://github.com/catapult-project/catapult/blob/master/tracing/docs/embedding-trace-viewer.md 43 // This is almost verbatim copy of: 44 // https://github.com/catapult-project/catapult/blob/master/tracing/bin/index.html 45 // on revision 623a005a3ffa9de13c4b92bc72290e7bcd1ca591. 46 var templTrace = ` 47 <html> 48 <head> 49 <link href="/trace_viewer_html" rel="import"> 50 <script> 51 (function() { 52 var viewer; 53 var url; 54 var model; 55 56 function load() { 57 var req = new XMLHttpRequest(); 58 var is_binary = /[.]gz$/.test(url) || /[.]zip$/.test(url); 59 req.overrideMimeType('text/plain; charset=x-user-defined'); 60 req.open('GET', url, true); 61 if (is_binary) 62 req.responseType = 'arraybuffer'; 63 64 req.onreadystatechange = function(event) { 65 if (req.readyState !== 4) 66 return; 67 68 window.setTimeout(function() { 69 if (req.status === 200) 70 onResult(is_binary ? req.response : req.responseText); 71 else 72 onResultFail(req.status); 73 }, 0); 74 }; 75 req.send(null); 76 } 77 78 function onResultFail(err) { 79 var overlay = new tr.ui.b.Overlay(); 80 overlay.textContent = err + ': ' + url + ' could not be loaded'; 81 overlay.title = 'Failed to fetch data'; 82 overlay.visible = true; 83 } 84 85 function onResult(result) { 86 model = new tr.Model(); 87 var i = new tr.importer.Import(model); 88 var p = i.importTracesWithProgressDialog([result]); 89 p.then(onModelLoaded, onImportFail); 90 } 91 92 function onModelLoaded() { 93 viewer.model = model; 94 viewer.viewTitle = "trace"; 95 } 96 97 function onImportFail() { 98 var overlay = new tr.ui.b.Overlay(); 99 overlay.textContent = tr.b.normalizeException(err).message; 100 overlay.title = 'Import error'; 101 overlay.visible = true; 102 } 103 104 document.addEventListener('DOMContentLoaded', function() { 105 var container = document.createElement('track-view-container'); 106 container.id = 'track_view_container'; 107 108 viewer = document.createElement('tr-ui-timeline-view'); 109 viewer.track_view_container = container; 110 viewer.appendChild(container); 111 112 viewer.id = 'trace-viewer'; 113 viewer.globalMode = true; 114 document.body.appendChild(viewer); 115 116 url = '/jsontrace?{{PARAMS}}'; 117 load(); 118 }); 119 }()); 120 </script> 121 </head> 122 <body> 123 </body> 124 </html> 125 ` 126 127 // httpTraceViewerHTML serves static part of trace-viewer. 128 // This URL is queried from templTrace HTML. 129 func httpTraceViewerHTML(w http.ResponseWriter, r *http.Request) { 130 http.ServeFile(w, r, filepath.Join(runtime.GOROOT(), "misc", "trace", "trace_viewer_lean.html")) 131 } 132 133 // httpJsonTrace serves json trace, requested from within templTrace HTML. 134 func httpJsonTrace(w http.ResponseWriter, r *http.Request) { 135 // This is an AJAX handler, so instead of http.Error we use log.Printf to log errors. 136 events, err := parseEvents() 137 if err != nil { 138 log.Printf("failed to parse trace: %v", err) 139 return 140 } 141 142 params := &traceParams{ 143 events: events, 144 endTime: int64(1<<63 - 1), 145 } 146 147 if goids := r.FormValue("goid"); goids != "" { 148 // If goid argument is present, we are rendering a trace for this particular goroutine. 149 goid, err := strconv.ParseUint(goids, 10, 64) 150 if err != nil { 151 log.Printf("failed to parse goid parameter '%v': %v", goids, err) 152 return 153 } 154 analyzeGoroutines(events) 155 g := gs[goid] 156 params.gtrace = true 157 params.startTime = g.StartTime 158 params.endTime = g.EndTime 159 params.maing = goid 160 params.gs = trace.RelatedGoroutines(events, goid) 161 } 162 163 data, err := generateTrace(params) 164 if err != nil { 165 log.Printf("failed to generate trace: %v", err) 166 return 167 } 168 169 if startStr, endStr := r.FormValue("start"), r.FormValue("end"); startStr != "" && endStr != "" { 170 // If start/end arguments are present, we are rendering a range of the trace. 171 start, err := strconv.ParseUint(startStr, 10, 64) 172 if err != nil { 173 log.Printf("failed to parse start parameter '%v': %v", startStr, err) 174 return 175 } 176 end, err := strconv.ParseUint(endStr, 10, 64) 177 if err != nil { 178 log.Printf("failed to parse end parameter '%v': %v", endStr, err) 179 return 180 } 181 if start >= uint64(len(data.Events)) || end <= start || end > uint64(len(data.Events)) { 182 log.Printf("bogus start/end parameters: %v/%v, trace size %v", start, end, len(data.Events)) 183 return 184 } 185 data.Events = append(data.Events[start:end], data.Events[data.footer:]...) 186 } 187 err = json.NewEncoder(w).Encode(data) 188 if err != nil { 189 log.Printf("failed to serialize trace: %v", err) 190 return 191 } 192 } 193 194 type Range struct { 195 Name string 196 Start int 197 End int 198 } 199 200 // splitTrace splits the trace into a number of ranges, 201 // each resulting in approx 100MB of json output (trace viewer can hardly handle more). 202 func splitTrace(data ViewerData) []Range { 203 const rangeSize = 100 << 20 204 var ranges []Range 205 cw := new(countingWriter) 206 enc := json.NewEncoder(cw) 207 // First calculate size of the mandatory part of the trace. 208 // This includes stack traces and thread names. 209 data1 := data 210 data1.Events = data.Events[data.footer:] 211 enc.Encode(data1) 212 auxSize := cw.size 213 cw.size = 0 214 // Then calculate size of each individual event and group them into ranges. 215 for i, start := 0, 0; i < data.footer; i++ { 216 enc.Encode(data.Events[i]) 217 if cw.size+auxSize > rangeSize || i == data.footer-1 { 218 ranges = append(ranges, Range{ 219 Name: fmt.Sprintf("%v-%v", time.Duration(data.Events[start].Time*1000), time.Duration(data.Events[i].Time*1000)), 220 Start: start, 221 End: i + 1, 222 }) 223 start = i + 1 224 cw.size = 0 225 } 226 } 227 if len(ranges) == 1 { 228 ranges = nil 229 } 230 return ranges 231 } 232 233 type countingWriter struct { 234 size int 235 } 236 237 func (cw *countingWriter) Write(data []byte) (int, error) { 238 cw.size += len(data) 239 return len(data), nil 240 } 241 242 type traceParams struct { 243 events []*trace.Event 244 gtrace bool 245 startTime int64 246 endTime int64 247 maing uint64 248 gs map[uint64]bool 249 } 250 251 type traceContext struct { 252 *traceParams 253 data ViewerData 254 frameTree frameNode 255 frameSeq int 256 arrowSeq uint64 257 heapAlloc uint64 258 nextGC uint64 259 gcount uint64 260 gstates [gStateCount]uint64 261 insyscall uint64 262 prunning uint64 263 } 264 265 type frameNode struct { 266 id int 267 children map[uint64]frameNode 268 } 269 270 type gState int 271 272 const ( 273 gDead gState = iota 274 gRunnable 275 gRunning 276 gWaiting 277 gWaitingGC 278 279 gStateCount 280 ) 281 282 type ViewerData struct { 283 Events []*ViewerEvent `json:"traceEvents"` 284 Frames map[string]ViewerFrame `json:"stackFrames"` 285 TimeUnit string `json:"displayTimeUnit"` 286 287 // This is where mandatory part of the trace starts (e.g. thread names) 288 footer int 289 } 290 291 type ViewerEvent struct { 292 Name string `json:"name,omitempty"` 293 Phase string `json:"ph"` 294 Scope string `json:"s,omitempty"` 295 Time float64 `json:"ts"` 296 Dur float64 `json:"dur,omitempty"` 297 Pid uint64 `json:"pid"` 298 Tid uint64 `json:"tid"` 299 ID uint64 `json:"id,omitempty"` 300 Stack int `json:"sf,omitempty"` 301 EndStack int `json:"esf,omitempty"` 302 Arg interface{} `json:"args,omitempty"` 303 } 304 305 type ViewerFrame struct { 306 Name string `json:"name"` 307 Parent int `json:"parent,omitempty"` 308 } 309 310 type NameArg struct { 311 Name string `json:"name"` 312 } 313 314 type SortIndexArg struct { 315 Index int `json:"sort_index"` 316 } 317 318 // generateTrace generates json trace for trace-viewer: 319 // https://github.com/google/trace-viewer 320 // Trace format is described at: 321 // https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/view 322 // If gtrace=true, generate trace for goroutine goid, otherwise whole trace. 323 // startTime, endTime determine part of the trace that we are interested in. 324 // gset restricts goroutines that are included in the resulting trace. 325 func generateTrace(params *traceParams) (ViewerData, error) { 326 ctx := &traceContext{traceParams: params} 327 ctx.frameTree.children = make(map[uint64]frameNode) 328 ctx.data.Frames = make(map[string]ViewerFrame) 329 ctx.data.TimeUnit = "ns" 330 maxProc := 0 331 gnames := make(map[uint64]string) 332 gstates := make(map[uint64]gState) 333 // Since we make many calls to setGState, we record a sticky 334 // error in setGStateErr and check it after every event. 335 var setGStateErr error 336 setGState := func(ev *trace.Event, g uint64, oldState, newState gState) { 337 if oldState == gWaiting && gstates[g] == gWaitingGC { 338 // For checking, gWaiting counts as any gWaiting*. 339 oldState = gstates[g] 340 } 341 if gstates[g] != oldState && setGStateErr == nil { 342 setGStateErr = fmt.Errorf("expected G %d to be in state %d, but got state %d", g, oldState, newState) 343 } 344 ctx.gstates[gstates[g]]-- 345 ctx.gstates[newState]++ 346 gstates[g] = newState 347 ctx.emitGoroutineCounters(ev) 348 } 349 for _, ev := range ctx.events { 350 // Handle trace.EvGoStart separately, because we need the goroutine name 351 // even if ignore the event otherwise. 352 if ev.Type == trace.EvGoStart { 353 if _, ok := gnames[ev.G]; !ok { 354 if len(ev.Stk) > 0 { 355 gnames[ev.G] = fmt.Sprintf("G%v %s", ev.G, ev.Stk[0].Fn) 356 } else { 357 gnames[ev.G] = fmt.Sprintf("G%v", ev.G) 358 } 359 } 360 } 361 362 // Ignore events that are from uninteresting goroutines 363 // or outside of the interesting timeframe. 364 if ctx.gs != nil && ev.P < trace.FakeP && !ctx.gs[ev.G] { 365 continue 366 } 367 if ev.Ts < ctx.startTime || ev.Ts > ctx.endTime { 368 continue 369 } 370 371 if ev.P < trace.FakeP && ev.P > maxProc { 372 maxProc = ev.P 373 } 374 375 switch ev.Type { 376 case trace.EvProcStart: 377 if ctx.gtrace { 378 continue 379 } 380 ctx.prunning++ 381 ctx.emitThreadCounters(ev) 382 ctx.emitInstant(ev, "proc start") 383 case trace.EvProcStop: 384 if ctx.gtrace { 385 continue 386 } 387 ctx.prunning-- 388 ctx.emitThreadCounters(ev) 389 ctx.emitInstant(ev, "proc stop") 390 case trace.EvGCStart: 391 ctx.emitSlice(ev, "GC") 392 case trace.EvGCDone: 393 case trace.EvGCScanStart: 394 if ctx.gtrace { 395 continue 396 } 397 ctx.emitSlice(ev, "MARK TERMINATION") 398 case trace.EvGCScanDone: 399 case trace.EvGCSweepStart: 400 ctx.emitSlice(ev, "SWEEP") 401 case trace.EvGCSweepDone: 402 case trace.EvGoStart, trace.EvGoStartLabel: 403 setGState(ev, ev.G, gRunnable, gRunning) 404 if ev.Type == trace.EvGoStartLabel { 405 ctx.emitSlice(ev, ev.SArgs[0]) 406 } else { 407 ctx.emitSlice(ev, gnames[ev.G]) 408 } 409 case trace.EvGoCreate: 410 ctx.gcount++ 411 setGState(ev, ev.Args[0], gDead, gRunnable) 412 ctx.emitArrow(ev, "go") 413 case trace.EvGoEnd: 414 ctx.gcount-- 415 setGState(ev, ev.G, gRunning, gDead) 416 case trace.EvGoUnblock: 417 setGState(ev, ev.Args[0], gWaiting, gRunnable) 418 ctx.emitArrow(ev, "unblock") 419 case trace.EvGoSysCall: 420 ctx.emitInstant(ev, "syscall") 421 case trace.EvGoSysExit: 422 setGState(ev, ev.G, gWaiting, gRunnable) 423 ctx.insyscall-- 424 ctx.emitThreadCounters(ev) 425 ctx.emitArrow(ev, "sysexit") 426 case trace.EvGoSysBlock: 427 setGState(ev, ev.G, gRunning, gWaiting) 428 ctx.insyscall++ 429 ctx.emitThreadCounters(ev) 430 case trace.EvGoSched, trace.EvGoPreempt: 431 setGState(ev, ev.G, gRunning, gRunnable) 432 case trace.EvGoStop, 433 trace.EvGoSleep, trace.EvGoBlock, trace.EvGoBlockSend, trace.EvGoBlockRecv, 434 trace.EvGoBlockSelect, trace.EvGoBlockSync, trace.EvGoBlockCond, trace.EvGoBlockNet: 435 setGState(ev, ev.G, gRunning, gWaiting) 436 case trace.EvGoBlockGC: 437 setGState(ev, ev.G, gRunning, gWaitingGC) 438 case trace.EvGoWaiting: 439 setGState(ev, ev.G, gRunnable, gWaiting) 440 case trace.EvGoInSyscall: 441 // Cancel out the effect of EvGoCreate at the beginning. 442 setGState(ev, ev.G, gRunnable, gWaiting) 443 ctx.insyscall++ 444 ctx.emitThreadCounters(ev) 445 case trace.EvHeapAlloc: 446 ctx.heapAlloc = ev.Args[0] 447 ctx.emitHeapCounters(ev) 448 case trace.EvNextGC: 449 ctx.nextGC = ev.Args[0] 450 ctx.emitHeapCounters(ev) 451 } 452 if setGStateErr != nil { 453 return ctx.data, setGStateErr 454 } 455 if ctx.gstates[gRunnable] < 0 || ctx.gstates[gRunning] < 0 || ctx.insyscall < 0 { 456 return ctx.data, fmt.Errorf("invalid state after processing %v: runnable=%d running=%d insyscall=%d", ev, ctx.gstates[gRunnable], ctx.gstates[gRunning], ctx.insyscall) 457 } 458 } 459 460 ctx.data.footer = len(ctx.data.Events) 461 ctx.emit(&ViewerEvent{Name: "process_name", Phase: "M", Pid: 0, Arg: &NameArg{"PROCS"}}) 462 ctx.emit(&ViewerEvent{Name: "process_sort_index", Phase: "M", Pid: 0, Arg: &SortIndexArg{1}}) 463 464 ctx.emit(&ViewerEvent{Name: "process_name", Phase: "M", Pid: 1, Arg: &NameArg{"STATS"}}) 465 ctx.emit(&ViewerEvent{Name: "process_sort_index", Phase: "M", Pid: 1, Arg: &SortIndexArg{0}}) 466 467 ctx.emit(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: 0, Tid: trace.GCP, Arg: &NameArg{"GC"}}) 468 ctx.emit(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: 0, Tid: trace.GCP, Arg: &SortIndexArg{-6}}) 469 470 ctx.emit(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: 0, Tid: trace.NetpollP, Arg: &NameArg{"Network"}}) 471 ctx.emit(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: 0, Tid: trace.NetpollP, Arg: &SortIndexArg{-5}}) 472 473 ctx.emit(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: 0, Tid: trace.TimerP, Arg: &NameArg{"Timers"}}) 474 ctx.emit(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: 0, Tid: trace.TimerP, Arg: &SortIndexArg{-4}}) 475 476 ctx.emit(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: 0, Tid: trace.SyscallP, Arg: &NameArg{"Syscalls"}}) 477 ctx.emit(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: 0, Tid: trace.SyscallP, Arg: &SortIndexArg{-3}}) 478 479 if !ctx.gtrace { 480 for i := 0; i <= maxProc; i++ { 481 ctx.emit(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: 0, Tid: uint64(i), Arg: &NameArg{fmt.Sprintf("Proc %v", i)}}) 482 ctx.emit(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: 0, Tid: uint64(i), Arg: &SortIndexArg{i}}) 483 } 484 } 485 486 if ctx.gtrace && ctx.gs != nil { 487 for k, v := range gnames { 488 if !ctx.gs[k] { 489 continue 490 } 491 ctx.emit(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: 0, Tid: k, Arg: &NameArg{v}}) 492 } 493 ctx.emit(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: 0, Tid: ctx.maing, Arg: &SortIndexArg{-2}}) 494 ctx.emit(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: 0, Tid: 0, Arg: &SortIndexArg{-1}}) 495 } 496 497 return ctx.data, nil 498 } 499 500 func (ctx *traceContext) emit(e *ViewerEvent) { 501 ctx.data.Events = append(ctx.data.Events, e) 502 } 503 504 func (ctx *traceContext) time(ev *trace.Event) float64 { 505 // Trace viewer wants timestamps in microseconds. 506 return float64(ev.Ts-ctx.startTime) / 1000 507 } 508 509 func (ctx *traceContext) proc(ev *trace.Event) uint64 { 510 if ctx.gtrace && ev.P < trace.FakeP { 511 return ev.G 512 } else { 513 return uint64(ev.P) 514 } 515 } 516 517 func (ctx *traceContext) emitSlice(ev *trace.Event, name string) { 518 ctx.emit(&ViewerEvent{ 519 Name: name, 520 Phase: "X", 521 Time: ctx.time(ev), 522 Dur: ctx.time(ev.Link) - ctx.time(ev), 523 Tid: ctx.proc(ev), 524 Stack: ctx.stack(ev.Stk), 525 EndStack: ctx.stack(ev.Link.Stk), 526 }) 527 } 528 529 type heapCountersArg struct { 530 Allocated uint64 531 NextGC uint64 532 } 533 534 func (ctx *traceContext) emitHeapCounters(ev *trace.Event) { 535 if ctx.gtrace { 536 return 537 } 538 diff := uint64(0) 539 if ctx.nextGC > ctx.heapAlloc { 540 diff = ctx.nextGC - ctx.heapAlloc 541 } 542 ctx.emit(&ViewerEvent{Name: "Heap", Phase: "C", Time: ctx.time(ev), Pid: 1, Arg: &heapCountersArg{ctx.heapAlloc, diff}}) 543 } 544 545 type goroutineCountersArg struct { 546 Running uint64 547 Runnable uint64 548 GCWaiting uint64 549 } 550 551 func (ctx *traceContext) emitGoroutineCounters(ev *trace.Event) { 552 if ctx.gtrace { 553 return 554 } 555 ctx.emit(&ViewerEvent{Name: "Goroutines", Phase: "C", Time: ctx.time(ev), Pid: 1, Arg: &goroutineCountersArg{ctx.gstates[gRunning], ctx.gstates[gRunnable], ctx.gstates[gWaitingGC]}}) 556 } 557 558 type threadCountersArg struct { 559 Running uint64 560 InSyscall uint64 561 } 562 563 func (ctx *traceContext) emitThreadCounters(ev *trace.Event) { 564 if ctx.gtrace { 565 return 566 } 567 ctx.emit(&ViewerEvent{Name: "Threads", Phase: "C", Time: ctx.time(ev), Pid: 1, Arg: &threadCountersArg{ctx.prunning, ctx.insyscall}}) 568 } 569 570 func (ctx *traceContext) emitInstant(ev *trace.Event, name string) { 571 var arg interface{} 572 if ev.Type == trace.EvProcStart { 573 type Arg struct { 574 ThreadID uint64 575 } 576 arg = &Arg{ev.Args[0]} 577 } 578 ctx.emit(&ViewerEvent{Name: name, Phase: "I", Scope: "t", Time: ctx.time(ev), Tid: ctx.proc(ev), Stack: ctx.stack(ev.Stk), Arg: arg}) 579 } 580 581 func (ctx *traceContext) emitArrow(ev *trace.Event, name string) { 582 if ev.Link == nil { 583 // The other end of the arrow is not captured in the trace. 584 // For example, a goroutine was unblocked but was not scheduled before trace stop. 585 return 586 } 587 if ctx.gtrace && (!ctx.gs[ev.Link.G] || ev.Link.Ts < ctx.startTime || ev.Link.Ts > ctx.endTime) { 588 return 589 } 590 591 if ev.P == trace.NetpollP || ev.P == trace.TimerP || ev.P == trace.SyscallP { 592 // Trace-viewer discards arrows if they don't start/end inside of a slice or instant. 593 // So emit a fake instant at the start of the arrow. 594 ctx.emitInstant(&trace.Event{P: ev.P, Ts: ev.Ts}, "unblock") 595 } 596 597 ctx.arrowSeq++ 598 ctx.emit(&ViewerEvent{Name: name, Phase: "s", Tid: ctx.proc(ev), ID: ctx.arrowSeq, Time: ctx.time(ev), Stack: ctx.stack(ev.Stk)}) 599 ctx.emit(&ViewerEvent{Name: name, Phase: "t", Tid: ctx.proc(ev.Link), ID: ctx.arrowSeq, Time: ctx.time(ev.Link)}) 600 } 601 602 func (ctx *traceContext) stack(stk []*trace.Frame) int { 603 return ctx.buildBranch(ctx.frameTree, stk) 604 } 605 606 // buildBranch builds one branch in the prefix tree rooted at ctx.frameTree. 607 func (ctx *traceContext) buildBranch(parent frameNode, stk []*trace.Frame) int { 608 if len(stk) == 0 { 609 return parent.id 610 } 611 last := len(stk) - 1 612 frame := stk[last] 613 stk = stk[:last] 614 615 node, ok := parent.children[frame.PC] 616 if !ok { 617 ctx.frameSeq++ 618 node.id = ctx.frameSeq 619 node.children = make(map[uint64]frameNode) 620 parent.children[frame.PC] = node 621 ctx.data.Frames[strconv.Itoa(node.id)] = ViewerFrame{fmt.Sprintf("%v:%v", frame.Fn, frame.Line), parent.id} 622 } 623 return ctx.buildBranch(node, stk) 624 }