github.com/mtsmfm/go/src@v0.0.0-20221020090648-44bdcb9f8fde/runtime/os_linux.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/abi" 9 "internal/goarch" 10 "runtime/internal/atomic" 11 "runtime/internal/syscall" 12 "unsafe" 13 ) 14 15 // sigPerThreadSyscall is the same signal (SIGSETXID) used by glibc for 16 // per-thread syscalls on Linux. We use it for the same purpose in non-cgo 17 // binaries. 18 const sigPerThreadSyscall = _SIGRTMIN + 1 19 20 type mOS struct { 21 // profileTimer holds the ID of the POSIX interval timer for profiling CPU 22 // usage on this thread. 23 // 24 // It is valid when the profileTimerValid field is true. A thread 25 // creates and manages its own timer, and these fields are read and written 26 // only by this thread. But because some of the reads on profileTimerValid 27 // are in signal handling code, this field should be atomic type. 28 profileTimer int32 29 profileTimerValid atomic.Bool 30 31 // needPerThreadSyscall indicates that a per-thread syscall is required 32 // for doAllThreadsSyscall. 33 needPerThreadSyscall atomic.Uint8 34 } 35 36 //go:noescape 37 func futex(addr unsafe.Pointer, op int32, val uint32, ts, addr2 unsafe.Pointer, val3 uint32) int32 38 39 // Linux futex. 40 // 41 // futexsleep(uint32 *addr, uint32 val) 42 // futexwakeup(uint32 *addr) 43 // 44 // Futexsleep atomically checks if *addr == val and if so, sleeps on addr. 45 // Futexwakeup wakes up threads sleeping on addr. 46 // Futexsleep is allowed to wake up spuriously. 47 48 const ( 49 _FUTEX_PRIVATE_FLAG = 128 50 _FUTEX_WAIT_PRIVATE = 0 | _FUTEX_PRIVATE_FLAG 51 _FUTEX_WAKE_PRIVATE = 1 | _FUTEX_PRIVATE_FLAG 52 ) 53 54 // Atomically, 55 // 56 // if(*addr == val) sleep 57 // 58 // Might be woken up spuriously; that's allowed. 59 // Don't sleep longer than ns; ns < 0 means forever. 60 // 61 //go:nosplit 62 func futexsleep(addr *uint32, val uint32, ns int64) { 63 // Some Linux kernels have a bug where futex of 64 // FUTEX_WAIT returns an internal error code 65 // as an errno. Libpthread ignores the return value 66 // here, and so can we: as it says a few lines up, 67 // spurious wakeups are allowed. 68 if ns < 0 { 69 futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, nil, nil, 0) 70 return 71 } 72 73 var ts timespec 74 ts.setNsec(ns) 75 futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, unsafe.Pointer(&ts), nil, 0) 76 } 77 78 // If any procs are sleeping on addr, wake up at most cnt. 79 // 80 //go:nosplit 81 func futexwakeup(addr *uint32, cnt uint32) { 82 ret := futex(unsafe.Pointer(addr), _FUTEX_WAKE_PRIVATE, cnt, nil, nil, 0) 83 if ret >= 0 { 84 return 85 } 86 87 // I don't know that futex wakeup can return 88 // EAGAIN or EINTR, but if it does, it would be 89 // safe to loop and call futex again. 90 systemstack(func() { 91 print("futexwakeup addr=", addr, " returned ", ret, "\n") 92 }) 93 94 *(*int32)(unsafe.Pointer(uintptr(0x1006))) = 0x1006 95 } 96 97 func getproccount() int32 { 98 // This buffer is huge (8 kB) but we are on the system stack 99 // and there should be plenty of space (64 kB). 100 // Also this is a leaf, so we're not holding up the memory for long. 101 // See golang.org/issue/11823. 102 // The suggested behavior here is to keep trying with ever-larger 103 // buffers, but we don't have a dynamic memory allocator at the 104 // moment, so that's a bit tricky and seems like overkill. 105 const maxCPUs = 64 * 1024 106 var buf [maxCPUs / 8]byte 107 r := sched_getaffinity(0, unsafe.Sizeof(buf), &buf[0]) 108 if r < 0 { 109 return 1 110 } 111 n := int32(0) 112 for _, v := range buf[:r] { 113 for v != 0 { 114 n += int32(v & 1) 115 v >>= 1 116 } 117 } 118 if n == 0 { 119 n = 1 120 } 121 return n 122 } 123 124 // Clone, the Linux rfork. 125 const ( 126 _CLONE_VM = 0x100 127 _CLONE_FS = 0x200 128 _CLONE_FILES = 0x400 129 _CLONE_SIGHAND = 0x800 130 _CLONE_PTRACE = 0x2000 131 _CLONE_VFORK = 0x4000 132 _CLONE_PARENT = 0x8000 133 _CLONE_THREAD = 0x10000 134 _CLONE_NEWNS = 0x20000 135 _CLONE_SYSVSEM = 0x40000 136 _CLONE_SETTLS = 0x80000 137 _CLONE_PARENT_SETTID = 0x100000 138 _CLONE_CHILD_CLEARTID = 0x200000 139 _CLONE_UNTRACED = 0x800000 140 _CLONE_CHILD_SETTID = 0x1000000 141 _CLONE_STOPPED = 0x2000000 142 _CLONE_NEWUTS = 0x4000000 143 _CLONE_NEWIPC = 0x8000000 144 145 // As of QEMU 2.8.0 (5ea2fc84d), user emulation requires all six of these 146 // flags to be set when creating a thread; attempts to share the other 147 // five but leave SYSVSEM unshared will fail with -EINVAL. 148 // 149 // In non-QEMU environments CLONE_SYSVSEM is inconsequential as we do not 150 // use System V semaphores. 151 152 cloneFlags = _CLONE_VM | /* share memory */ 153 _CLONE_FS | /* share cwd, etc */ 154 _CLONE_FILES | /* share fd table */ 155 _CLONE_SIGHAND | /* share sig handler table */ 156 _CLONE_SYSVSEM | /* share SysV semaphore undo lists (see issue #20763) */ 157 _CLONE_THREAD /* revisit - okay for now */ 158 ) 159 160 //go:noescape 161 func clone(flags int32, stk, mp, gp, fn unsafe.Pointer) int32 162 163 // May run with m.p==nil, so write barriers are not allowed. 164 // 165 //go:nowritebarrier 166 func newosproc(mp *m) { 167 stk := unsafe.Pointer(mp.g0.stack.hi) 168 /* 169 * note: strace gets confused if we use CLONE_PTRACE here. 170 */ 171 if false { 172 print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " clone=", abi.FuncPCABI0(clone), " id=", mp.id, " ostk=", &mp, "\n") 173 } 174 175 // Disable signals during clone, so that the new thread starts 176 // with signals disabled. It will enable them in minit. 177 var oset sigset 178 sigprocmask(_SIG_SETMASK, &sigset_all, &oset) 179 ret := clone(cloneFlags, stk, unsafe.Pointer(mp), unsafe.Pointer(mp.g0), unsafe.Pointer(abi.FuncPCABI0(mstart))) 180 sigprocmask(_SIG_SETMASK, &oset, nil) 181 182 if ret < 0 { 183 print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", -ret, ")\n") 184 if ret == -_EAGAIN { 185 println("runtime: may need to increase max user processes (ulimit -u)") 186 } 187 throw("newosproc") 188 } 189 } 190 191 // Version of newosproc that doesn't require a valid G. 192 // 193 //go:nosplit 194 func newosproc0(stacksize uintptr, fn unsafe.Pointer) { 195 stack := sysAlloc(stacksize, &memstats.stacks_sys) 196 if stack == nil { 197 write(2, unsafe.Pointer(&failallocatestack[0]), int32(len(failallocatestack))) 198 exit(1) 199 } 200 ret := clone(cloneFlags, unsafe.Pointer(uintptr(stack)+stacksize), nil, nil, fn) 201 if ret < 0 { 202 write(2, unsafe.Pointer(&failthreadcreate[0]), int32(len(failthreadcreate))) 203 exit(1) 204 } 205 } 206 207 var failallocatestack = []byte("runtime: failed to allocate stack for the new OS thread\n") 208 var failthreadcreate = []byte("runtime: failed to create new OS thread\n") 209 210 const ( 211 _AT_NULL = 0 // End of vector 212 _AT_PAGESZ = 6 // System physical page size 213 _AT_HWCAP = 16 // hardware capability bit vector 214 _AT_RANDOM = 25 // introduced in 2.6.29 215 _AT_HWCAP2 = 26 // hardware capability bit vector 2 216 ) 217 218 var procAuxv = []byte("/proc/self/auxv\x00") 219 220 var addrspace_vec [1]byte 221 222 func mincore(addr unsafe.Pointer, n uintptr, dst *byte) int32 223 224 func sysargs(argc int32, argv **byte) { 225 n := argc + 1 226 227 // skip over argv, envp to get to auxv 228 for argv_index(argv, n) != nil { 229 n++ 230 } 231 232 // skip NULL separator 233 n++ 234 235 // now argv+n is auxv 236 auxv := (*[1 << 28]uintptr)(add(unsafe.Pointer(argv), uintptr(n)*goarch.PtrSize)) 237 if sysauxv(auxv[:]) != 0 { 238 return 239 } 240 // In some situations we don't get a loader-provided 241 // auxv, such as when loaded as a library on Android. 242 // Fall back to /proc/self/auxv. 243 fd := open(&procAuxv[0], 0 /* O_RDONLY */, 0) 244 if fd < 0 { 245 // On Android, /proc/self/auxv might be unreadable (issue 9229), so we fallback to 246 // try using mincore to detect the physical page size. 247 // mincore should return EINVAL when address is not a multiple of system page size. 248 const size = 256 << 10 // size of memory region to allocate 249 p, err := mmap(nil, size, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0) 250 if err != 0 { 251 return 252 } 253 var n uintptr 254 for n = 4 << 10; n < size; n <<= 1 { 255 err := mincore(unsafe.Pointer(uintptr(p)+n), 1, &addrspace_vec[0]) 256 if err == 0 { 257 physPageSize = n 258 break 259 } 260 } 261 if physPageSize == 0 { 262 physPageSize = size 263 } 264 munmap(p, size) 265 return 266 } 267 var buf [128]uintptr 268 n = read(fd, noescape(unsafe.Pointer(&buf[0])), int32(unsafe.Sizeof(buf))) 269 closefd(fd) 270 if n < 0 { 271 return 272 } 273 // Make sure buf is terminated, even if we didn't read 274 // the whole file. 275 buf[len(buf)-2] = _AT_NULL 276 sysauxv(buf[:]) 277 } 278 279 // startupRandomData holds random bytes initialized at startup. These come from 280 // the ELF AT_RANDOM auxiliary vector. 281 var startupRandomData []byte 282 283 func sysauxv(auxv []uintptr) int { 284 var i int 285 for ; auxv[i] != _AT_NULL; i += 2 { 286 tag, val := auxv[i], auxv[i+1] 287 switch tag { 288 case _AT_RANDOM: 289 // The kernel provides a pointer to 16-bytes 290 // worth of random data. 291 startupRandomData = (*[16]byte)(unsafe.Pointer(val))[:] 292 293 case _AT_PAGESZ: 294 physPageSize = val 295 } 296 297 archauxv(tag, val) 298 vdsoauxv(tag, val) 299 } 300 return i / 2 301 } 302 303 var sysTHPSizePath = []byte("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size\x00") 304 305 func getHugePageSize() uintptr { 306 var numbuf [20]byte 307 fd := open(&sysTHPSizePath[0], 0 /* O_RDONLY */, 0) 308 if fd < 0 { 309 return 0 310 } 311 ptr := noescape(unsafe.Pointer(&numbuf[0])) 312 n := read(fd, ptr, int32(len(numbuf))) 313 closefd(fd) 314 if n <= 0 { 315 return 0 316 } 317 n-- // remove trailing newline 318 v, ok := atoi(slicebytetostringtmp((*byte)(ptr), int(n))) 319 if !ok || v < 0 { 320 v = 0 321 } 322 if v&(v-1) != 0 { 323 // v is not a power of 2 324 return 0 325 } 326 return uintptr(v) 327 } 328 329 func osinit() { 330 ncpu = getproccount() 331 physHugePageSize = getHugePageSize() 332 if iscgo { 333 // #42494 glibc and musl reserve some signals for 334 // internal use and require they not be blocked by 335 // the rest of a normal C runtime. When the go runtime 336 // blocks...unblocks signals, temporarily, the blocked 337 // interval of time is generally very short. As such, 338 // these expectations of *libc code are mostly met by 339 // the combined go+cgo system of threads. However, 340 // when go causes a thread to exit, via a return from 341 // mstart(), the combined runtime can deadlock if 342 // these signals are blocked. Thus, don't block these 343 // signals when exiting threads. 344 // - glibc: SIGCANCEL (32), SIGSETXID (33) 345 // - musl: SIGTIMER (32), SIGCANCEL (33), SIGSYNCCALL (34) 346 sigdelset(&sigsetAllExiting, 32) 347 sigdelset(&sigsetAllExiting, 33) 348 sigdelset(&sigsetAllExiting, 34) 349 } 350 osArchInit() 351 } 352 353 var urandom_dev = []byte("/dev/urandom\x00") 354 355 func getRandomData(r []byte) { 356 if startupRandomData != nil { 357 n := copy(r, startupRandomData) 358 extendRandom(r, n) 359 return 360 } 361 fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) 362 n := read(fd, unsafe.Pointer(&r[0]), int32(len(r))) 363 closefd(fd) 364 extendRandom(r, int(n)) 365 } 366 367 func goenvs() { 368 goenvs_unix() 369 } 370 371 // Called to do synchronous initialization of Go code built with 372 // -buildmode=c-archive or -buildmode=c-shared. 373 // None of the Go runtime is initialized. 374 // 375 //go:nosplit 376 //go:nowritebarrierrec 377 func libpreinit() { 378 initsig(true) 379 } 380 381 // Called to initialize a new m (including the bootstrap m). 382 // Called on the parent thread (main thread in case of bootstrap), can allocate memory. 383 func mpreinit(mp *m) { 384 mp.gsignal = malg(32 * 1024) // Linux wants >= 2K 385 mp.gsignal.m = mp 386 } 387 388 func gettid() uint32 389 390 // Called to initialize a new m (including the bootstrap m). 391 // Called on the new thread, cannot allocate memory. 392 func minit() { 393 minitSignals() 394 395 // Cgo-created threads and the bootstrap m are missing a 396 // procid. We need this for asynchronous preemption and it's 397 // useful in debuggers. 398 getg().m.procid = uint64(gettid()) 399 } 400 401 // Called from dropm to undo the effect of an minit. 402 // 403 //go:nosplit 404 func unminit() { 405 unminitSignals() 406 } 407 408 // Called from exitm, but not from drop, to undo the effect of thread-owned 409 // resources in minit, semacreate, or elsewhere. Do not take locks after calling this. 410 func mdestroy(mp *m) { 411 } 412 413 //#ifdef GOARCH_386 414 //#define sa_handler k_sa_handler 415 //#endif 416 417 func sigreturn() 418 func sigtramp() // Called via C ABI 419 func cgoSigtramp() 420 421 //go:noescape 422 func sigaltstack(new, old *stackt) 423 424 //go:noescape 425 func setitimer(mode int32, new, old *itimerval) 426 427 //go:noescape 428 func timer_create(clockid int32, sevp *sigevent, timerid *int32) int32 429 430 //go:noescape 431 func timer_settime(timerid int32, flags int32, new, old *itimerspec) int32 432 433 //go:noescape 434 func timer_delete(timerid int32) int32 435 436 //go:noescape 437 func rtsigprocmask(how int32, new, old *sigset, size int32) 438 439 //go:nosplit 440 //go:nowritebarrierrec 441 func sigprocmask(how int32, new, old *sigset) { 442 rtsigprocmask(how, new, old, int32(unsafe.Sizeof(*new))) 443 } 444 445 func raise(sig uint32) 446 func raiseproc(sig uint32) 447 448 //go:noescape 449 func sched_getaffinity(pid, len uintptr, buf *byte) int32 450 func osyield() 451 452 //go:nosplit 453 func osyield_no_g() { 454 osyield() 455 } 456 457 func pipe2(flags int32) (r, w int32, errno int32) 458 459 const ( 460 _si_max_size = 128 461 _sigev_max_size = 64 462 ) 463 464 //go:nosplit 465 //go:nowritebarrierrec 466 func setsig(i uint32, fn uintptr) { 467 var sa sigactiont 468 sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK | _SA_RESTORER | _SA_RESTART 469 sigfillset(&sa.sa_mask) 470 // Although Linux manpage says "sa_restorer element is obsolete and 471 // should not be used". x86_64 kernel requires it. Only use it on 472 // x86. 473 if GOARCH == "386" || GOARCH == "amd64" { 474 sa.sa_restorer = abi.FuncPCABI0(sigreturn) 475 } 476 if fn == abi.FuncPCABIInternal(sighandler) { // abi.FuncPCABIInternal(sighandler) matches the callers in signal_unix.go 477 if iscgo { 478 fn = abi.FuncPCABI0(cgoSigtramp) 479 } else { 480 fn = abi.FuncPCABI0(sigtramp) 481 } 482 } 483 sa.sa_handler = fn 484 sigaction(i, &sa, nil) 485 } 486 487 //go:nosplit 488 //go:nowritebarrierrec 489 func setsigstack(i uint32) { 490 var sa sigactiont 491 sigaction(i, nil, &sa) 492 if sa.sa_flags&_SA_ONSTACK != 0 { 493 return 494 } 495 sa.sa_flags |= _SA_ONSTACK 496 sigaction(i, &sa, nil) 497 } 498 499 //go:nosplit 500 //go:nowritebarrierrec 501 func getsig(i uint32) uintptr { 502 var sa sigactiont 503 sigaction(i, nil, &sa) 504 return sa.sa_handler 505 } 506 507 // setSignaltstackSP sets the ss_sp field of a stackt. 508 // 509 //go:nosplit 510 func setSignalstackSP(s *stackt, sp uintptr) { 511 *(*uintptr)(unsafe.Pointer(&s.ss_sp)) = sp 512 } 513 514 //go:nosplit 515 func (c *sigctxt) fixsigcode(sig uint32) { 516 } 517 518 // sysSigaction calls the rt_sigaction system call. 519 // 520 //go:nosplit 521 func sysSigaction(sig uint32, new, old *sigactiont) { 522 if rt_sigaction(uintptr(sig), new, old, unsafe.Sizeof(sigactiont{}.sa_mask)) != 0 { 523 // Workaround for bugs in QEMU user mode emulation. 524 // 525 // QEMU turns calls to the sigaction system call into 526 // calls to the C library sigaction call; the C 527 // library call rejects attempts to call sigaction for 528 // SIGCANCEL (32) or SIGSETXID (33). 529 // 530 // QEMU rejects calling sigaction on SIGRTMAX (64). 531 // 532 // Just ignore the error in these case. There isn't 533 // anything we can do about it anyhow. 534 if sig != 32 && sig != 33 && sig != 64 { 535 // Use system stack to avoid split stack overflow on ppc64/ppc64le. 536 systemstack(func() { 537 throw("sigaction failed") 538 }) 539 } 540 } 541 } 542 543 // rt_sigaction is implemented in assembly. 544 // 545 //go:noescape 546 func rt_sigaction(sig uintptr, new, old *sigactiont, size uintptr) int32 547 548 func getpid() int 549 func tgkill(tgid, tid, sig int) 550 551 // signalM sends a signal to mp. 552 func signalM(mp *m, sig int) { 553 tgkill(getpid(), int(mp.procid), sig) 554 } 555 556 // go118UseTimerCreateProfiler enables the per-thread CPU profiler. 557 const go118UseTimerCreateProfiler = true 558 559 // validSIGPROF compares this signal delivery's code against the signal sources 560 // that the profiler uses, returning whether the delivery should be processed. 561 // To be processed, a signal delivery from a known profiling mechanism should 562 // correspond to the best profiling mechanism available to this thread. Signals 563 // from other sources are always considered valid. 564 // 565 //go:nosplit 566 func validSIGPROF(mp *m, c *sigctxt) bool { 567 code := int32(c.sigcode()) 568 setitimer := code == _SI_KERNEL 569 timer_create := code == _SI_TIMER 570 571 if !(setitimer || timer_create) { 572 // The signal doesn't correspond to a profiling mechanism that the 573 // runtime enables itself. There's no reason to process it, but there's 574 // no reason to ignore it either. 575 return true 576 } 577 578 if mp == nil { 579 // Since we don't have an M, we can't check if there's an active 580 // per-thread timer for this thread. We don't know how long this thread 581 // has been around, and if it happened to interact with the Go scheduler 582 // at a time when profiling was active (causing it to have a per-thread 583 // timer). But it may have never interacted with the Go scheduler, or 584 // never while profiling was active. To avoid double-counting, process 585 // only signals from setitimer. 586 // 587 // When a custom cgo traceback function has been registered (on 588 // platforms that support runtime.SetCgoTraceback), SIGPROF signals 589 // delivered to a thread that cannot find a matching M do this check in 590 // the assembly implementations of runtime.cgoSigtramp. 591 return setitimer 592 } 593 594 // Having an M means the thread interacts with the Go scheduler, and we can 595 // check whether there's an active per-thread timer for this thread. 596 if mp.profileTimerValid.Load() { 597 // If this M has its own per-thread CPU profiling interval timer, we 598 // should track the SIGPROF signals that come from that timer (for 599 // accurate reporting of its CPU usage; see issue 35057) and ignore any 600 // that it gets from the process-wide setitimer (to not over-count its 601 // CPU consumption). 602 return timer_create 603 } 604 605 // No active per-thread timer means the only valid profiler is setitimer. 606 return setitimer 607 } 608 609 func setProcessCPUProfiler(hz int32) { 610 setProcessCPUProfilerTimer(hz) 611 } 612 613 func setThreadCPUProfiler(hz int32) { 614 mp := getg().m 615 mp.profilehz = hz 616 617 if !go118UseTimerCreateProfiler { 618 return 619 } 620 621 // destroy any active timer 622 if mp.profileTimerValid.Load() { 623 timerid := mp.profileTimer 624 mp.profileTimerValid.Store(false) 625 mp.profileTimer = 0 626 627 ret := timer_delete(timerid) 628 if ret != 0 { 629 print("runtime: failed to disable profiling timer; timer_delete(", timerid, ") errno=", -ret, "\n") 630 throw("timer_delete") 631 } 632 } 633 634 if hz == 0 { 635 // If the goal was to disable profiling for this thread, then the job's done. 636 return 637 } 638 639 // The period of the timer should be 1/Hz. For every "1/Hz" of additional 640 // work, the user should expect one additional sample in the profile. 641 // 642 // But to scale down to very small amounts of application work, to observe 643 // even CPU usage of "one tenth" of the requested period, set the initial 644 // timing delay in a different way: So that "one tenth" of a period of CPU 645 // spend shows up as a 10% chance of one sample (for an expected value of 646 // 0.1 samples), and so that "two and six tenths" periods of CPU spend show 647 // up as a 60% chance of 3 samples and a 40% chance of 2 samples (for an 648 // expected value of 2.6). Set the initial delay to a value in the unifom 649 // random distribution between 0 and the desired period. And because "0" 650 // means "disable timer", add 1 so the half-open interval [0,period) turns 651 // into (0,period]. 652 // 653 // Otherwise, this would show up as a bias away from short-lived threads and 654 // from threads that are only occasionally active: for example, when the 655 // garbage collector runs on a mostly-idle system, the additional threads it 656 // activates may do a couple milliseconds of GC-related work and nothing 657 // else in the few seconds that the profiler observes. 658 spec := new(itimerspec) 659 spec.it_value.setNsec(1 + int64(fastrandn(uint32(1e9/hz)))) 660 spec.it_interval.setNsec(1e9 / int64(hz)) 661 662 var timerid int32 663 var sevp sigevent 664 sevp.notify = _SIGEV_THREAD_ID 665 sevp.signo = _SIGPROF 666 sevp.sigev_notify_thread_id = int32(mp.procid) 667 ret := timer_create(_CLOCK_THREAD_CPUTIME_ID, &sevp, &timerid) 668 if ret != 0 { 669 // If we cannot create a timer for this M, leave profileTimerValid false 670 // to fall back to the process-wide setitimer profiler. 671 return 672 } 673 674 ret = timer_settime(timerid, 0, spec, nil) 675 if ret != 0 { 676 print("runtime: failed to configure profiling timer; timer_settime(", timerid, 677 ", 0, {interval: {", 678 spec.it_interval.tv_sec, "s + ", spec.it_interval.tv_nsec, "ns} value: {", 679 spec.it_value.tv_sec, "s + ", spec.it_value.tv_nsec, "ns}}, nil) errno=", -ret, "\n") 680 throw("timer_settime") 681 } 682 683 mp.profileTimer = timerid 684 mp.profileTimerValid.Store(true) 685 } 686 687 // perThreadSyscallArgs contains the system call number, arguments, and 688 // expected return values for a system call to be executed on all threads. 689 type perThreadSyscallArgs struct { 690 trap uintptr 691 a1 uintptr 692 a2 uintptr 693 a3 uintptr 694 a4 uintptr 695 a5 uintptr 696 a6 uintptr 697 r1 uintptr 698 r2 uintptr 699 } 700 701 // perThreadSyscall is the system call to execute for the ongoing 702 // doAllThreadsSyscall. 703 // 704 // perThreadSyscall may only be written while mp.needPerThreadSyscall == 0 on 705 // all Ms. 706 var perThreadSyscall perThreadSyscallArgs 707 708 // syscall_runtime_doAllThreadsSyscall and executes a specified system call on 709 // all Ms. 710 // 711 // The system call is expected to succeed and return the same value on every 712 // thread. If any threads do not match, the runtime throws. 713 // 714 //go:linkname syscall_runtime_doAllThreadsSyscall syscall.runtime_doAllThreadsSyscall 715 //go:uintptrescapes 716 func syscall_runtime_doAllThreadsSyscall(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { 717 if iscgo { 718 // In cgo, we are not aware of threads created in C, so this approach will not work. 719 panic("doAllThreadsSyscall not supported with cgo enabled") 720 } 721 722 // STW to guarantee that user goroutines see an atomic change to thread 723 // state. Without STW, goroutines could migrate Ms while change is in 724 // progress and e.g., see state old -> new -> old -> new. 725 // 726 // N.B. Internally, this function does not depend on STW to 727 // successfully change every thread. It is only needed for user 728 // expectations, per above. 729 stopTheWorld("doAllThreadsSyscall") 730 731 // This function depends on several properties: 732 // 733 // 1. All OS threads that already exist are associated with an M in 734 // allm. i.e., we won't miss any pre-existing threads. 735 // 2. All Ms listed in allm will eventually have an OS thread exist. 736 // i.e., they will set procid and be able to receive signals. 737 // 3. OS threads created after we read allm will clone from a thread 738 // that has executed the system call. i.e., they inherit the 739 // modified state. 740 // 741 // We achieve these through different mechanisms: 742 // 743 // 1. Addition of new Ms to allm in allocm happens before clone of its 744 // OS thread later in newm. 745 // 2. newm does acquirem to avoid being preempted, ensuring that new Ms 746 // created in allocm will eventually reach OS thread clone later in 747 // newm. 748 // 3. We take allocmLock for write here to prevent allocation of new Ms 749 // while this function runs. Per (1), this prevents clone of OS 750 // threads that are not yet in allm. 751 allocmLock.lock() 752 753 // Disable preemption, preventing us from changing Ms, as we handle 754 // this M specially. 755 // 756 // N.B. STW and lock() above do this as well, this is added for extra 757 // clarity. 758 acquirem() 759 760 // N.B. allocmLock also prevents concurrent execution of this function, 761 // serializing use of perThreadSyscall, mp.needPerThreadSyscall, and 762 // ensuring all threads execute system calls from multiple calls in the 763 // same order. 764 765 r1, r2, errno := syscall.Syscall6(trap, a1, a2, a3, a4, a5, a6) 766 if GOARCH == "ppc64" || GOARCH == "ppc64le" { 767 // TODO(https://go.dev/issue/51192 ): ppc64 doesn't use r2. 768 r2 = 0 769 } 770 if errno != 0 { 771 releasem(getg().m) 772 allocmLock.unlock() 773 startTheWorld() 774 return r1, r2, errno 775 } 776 777 perThreadSyscall = perThreadSyscallArgs{ 778 trap: trap, 779 a1: a1, 780 a2: a2, 781 a3: a3, 782 a4: a4, 783 a5: a5, 784 a6: a6, 785 r1: r1, 786 r2: r2, 787 } 788 789 // Wait for all threads to start. 790 // 791 // As described above, some Ms have been added to allm prior to 792 // allocmLock, but not yet completed OS clone and set procid. 793 // 794 // At minimum we must wait for a thread to set procid before we can 795 // send it a signal. 796 // 797 // We take this one step further and wait for all threads to start 798 // before sending any signals. This prevents system calls from getting 799 // applied twice: once in the parent and once in the child, like so: 800 // 801 // A B C 802 // add C to allm 803 // doAllThreadsSyscall 804 // allocmLock.lock() 805 // signal B 806 // <receive signal> 807 // execute syscall 808 // <signal return> 809 // clone C 810 // <thread start> 811 // set procid 812 // signal C 813 // <receive signal> 814 // execute syscall 815 // <signal return> 816 // 817 // In this case, thread C inherited the syscall-modified state from 818 // thread B and did not need to execute the syscall, but did anyway 819 // because doAllThreadsSyscall could not be sure whether it was 820 // required. 821 // 822 // Some system calls may not be idempotent, so we ensure each thread 823 // executes the system call exactly once. 824 for mp := allm; mp != nil; mp = mp.alllink { 825 for atomic.Load64(&mp.procid) == 0 { 826 // Thread is starting. 827 osyield() 828 } 829 } 830 831 // Signal every other thread, where they will execute perThreadSyscall 832 // from the signal handler. 833 gp := getg() 834 tid := gp.m.procid 835 for mp := allm; mp != nil; mp = mp.alllink { 836 if atomic.Load64(&mp.procid) == tid { 837 // Our thread already performed the syscall. 838 continue 839 } 840 mp.needPerThreadSyscall.Store(1) 841 signalM(mp, sigPerThreadSyscall) 842 } 843 844 // Wait for all threads to complete. 845 for mp := allm; mp != nil; mp = mp.alllink { 846 if mp.procid == tid { 847 continue 848 } 849 for mp.needPerThreadSyscall.Load() != 0 { 850 osyield() 851 } 852 } 853 854 perThreadSyscall = perThreadSyscallArgs{} 855 856 releasem(getg().m) 857 allocmLock.unlock() 858 startTheWorld() 859 860 return r1, r2, errno 861 } 862 863 // runPerThreadSyscall runs perThreadSyscall for this M if required. 864 // 865 // This function throws if the system call returns with anything other than the 866 // expected values. 867 // 868 //go:nosplit 869 func runPerThreadSyscall() { 870 gp := getg() 871 if gp.m.needPerThreadSyscall.Load() == 0 { 872 return 873 } 874 875 args := perThreadSyscall 876 r1, r2, errno := syscall.Syscall6(args.trap, args.a1, args.a2, args.a3, args.a4, args.a5, args.a6) 877 if GOARCH == "ppc64" || GOARCH == "ppc64le" { 878 // TODO(https://go.dev/issue/51192 ): ppc64 doesn't use r2. 879 r2 = 0 880 } 881 if errno != 0 || r1 != args.r1 || r2 != args.r2 { 882 print("trap:", args.trap, ", a123456=[", args.a1, ",", args.a2, ",", args.a3, ",", args.a4, ",", args.a5, ",", args.a6, "]\n") 883 print("results: got {r1=", r1, ",r2=", r2, ",errno=", errno, "}, want {r1=", args.r1, ",r2=", args.r2, ",errno=0\n") 884 fatal("AllThreadsSyscall6 results differ between threads; runtime corrupted") 885 } 886 887 gp.m.needPerThreadSyscall.Store(0) 888 } 889 890 const ( 891 _SI_USER = 0 892 _SI_TKILL = -6 893 ) 894 895 // sigFromUser reports whether the signal was sent because of a call 896 // to kill or tgkill. 897 // 898 //go:nosplit 899 func (c *sigctxt) sigFromUser() bool { 900 code := int32(c.sigcode()) 901 return code == _SI_USER || code == _SI_TKILL 902 }