github.com/comwrg/go/src@v0.0.0-20220319063731-c238d0440370/runtime/os_linux.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "runtime/internal/sys" 9 "unsafe" 10 ) 11 12 type mOS struct{} 13 14 //go:noescape 15 func futex(addr unsafe.Pointer, op int32, val uint32, ts, addr2 unsafe.Pointer, val3 uint32) int32 16 17 // Linux futex. 18 // 19 // futexsleep(uint32 *addr, uint32 val) 20 // futexwakeup(uint32 *addr) 21 // 22 // Futexsleep atomically checks if *addr == val and if so, sleeps on addr. 23 // Futexwakeup wakes up threads sleeping on addr. 24 // Futexsleep is allowed to wake up spuriously. 25 26 const ( 27 _FUTEX_PRIVATE_FLAG = 128 28 _FUTEX_WAIT_PRIVATE = 0 | _FUTEX_PRIVATE_FLAG 29 _FUTEX_WAKE_PRIVATE = 1 | _FUTEX_PRIVATE_FLAG 30 ) 31 32 // Atomically, 33 // if(*addr == val) sleep 34 // Might be woken up spuriously; that's allowed. 35 // Don't sleep longer than ns; ns < 0 means forever. 36 //go:nosplit 37 func futexsleep(addr *uint32, val uint32, ns int64) { 38 // Some Linux kernels have a bug where futex of 39 // FUTEX_WAIT returns an internal error code 40 // as an errno. Libpthread ignores the return value 41 // here, and so can we: as it says a few lines up, 42 // spurious wakeups are allowed. 43 if ns < 0 { 44 futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, nil, nil, 0) 45 return 46 } 47 48 var ts timespec 49 ts.setNsec(ns) 50 futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, unsafe.Pointer(&ts), nil, 0) 51 } 52 53 // If any procs are sleeping on addr, wake up at most cnt. 54 //go:nosplit 55 func futexwakeup(addr *uint32, cnt uint32) { 56 ret := futex(unsafe.Pointer(addr), _FUTEX_WAKE_PRIVATE, cnt, nil, nil, 0) 57 if ret >= 0 { 58 return 59 } 60 61 // I don't know that futex wakeup can return 62 // EAGAIN or EINTR, but if it does, it would be 63 // safe to loop and call futex again. 64 systemstack(func() { 65 print("futexwakeup addr=", addr, " returned ", ret, "\n") 66 }) 67 68 *(*int32)(unsafe.Pointer(uintptr(0x1006))) = 0x1006 69 } 70 71 func getproccount() int32 { 72 // This buffer is huge (8 kB) but we are on the system stack 73 // and there should be plenty of space (64 kB). 74 // Also this is a leaf, so we're not holding up the memory for long. 75 // See golang.org/issue/11823. 76 // The suggested behavior here is to keep trying with ever-larger 77 // buffers, but we don't have a dynamic memory allocator at the 78 // moment, so that's a bit tricky and seems like overkill. 79 const maxCPUs = 64 * 1024 80 var buf [maxCPUs / 8]byte 81 r := sched_getaffinity(0, unsafe.Sizeof(buf), &buf[0]) 82 if r < 0 { 83 return 1 84 } 85 n := int32(0) 86 for _, v := range buf[:r] { 87 for v != 0 { 88 n += int32(v & 1) 89 v >>= 1 90 } 91 } 92 if n == 0 { 93 n = 1 94 } 95 return n 96 } 97 98 // Clone, the Linux rfork. 99 const ( 100 _CLONE_VM = 0x100 101 _CLONE_FS = 0x200 102 _CLONE_FILES = 0x400 103 _CLONE_SIGHAND = 0x800 104 _CLONE_PTRACE = 0x2000 105 _CLONE_VFORK = 0x4000 106 _CLONE_PARENT = 0x8000 107 _CLONE_THREAD = 0x10000 108 _CLONE_NEWNS = 0x20000 109 _CLONE_SYSVSEM = 0x40000 110 _CLONE_SETTLS = 0x80000 111 _CLONE_PARENT_SETTID = 0x100000 112 _CLONE_CHILD_CLEARTID = 0x200000 113 _CLONE_UNTRACED = 0x800000 114 _CLONE_CHILD_SETTID = 0x1000000 115 _CLONE_STOPPED = 0x2000000 116 _CLONE_NEWUTS = 0x4000000 117 _CLONE_NEWIPC = 0x8000000 118 119 // As of QEMU 2.8.0 (5ea2fc84d), user emulation requires all six of these 120 // flags to be set when creating a thread; attempts to share the other 121 // five but leave SYSVSEM unshared will fail with -EINVAL. 122 // 123 // In non-QEMU environments CLONE_SYSVSEM is inconsequential as we do not 124 // use System V semaphores. 125 126 cloneFlags = _CLONE_VM | /* share memory */ 127 _CLONE_FS | /* share cwd, etc */ 128 _CLONE_FILES | /* share fd table */ 129 _CLONE_SIGHAND | /* share sig handler table */ 130 _CLONE_SYSVSEM | /* share SysV semaphore undo lists (see issue #20763) */ 131 _CLONE_THREAD /* revisit - okay for now */ 132 ) 133 134 //go:noescape 135 func clone(flags int32, stk, mp, gp, fn unsafe.Pointer) int32 136 137 // May run with m.p==nil, so write barriers are not allowed. 138 //go:nowritebarrier 139 func newosproc(mp *m) { 140 stk := unsafe.Pointer(mp.g0.stack.hi) 141 /* 142 * note: strace gets confused if we use CLONE_PTRACE here. 143 */ 144 if false { 145 print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " clone=", funcPC(clone), " id=", mp.id, " ostk=", &mp, "\n") 146 } 147 148 // Disable signals during clone, so that the new thread starts 149 // with signals disabled. It will enable them in minit. 150 var oset sigset 151 sigprocmask(_SIG_SETMASK, &sigset_all, &oset) 152 ret := clone(cloneFlags, stk, unsafe.Pointer(mp), unsafe.Pointer(mp.g0), unsafe.Pointer(funcPC(mstart))) 153 sigprocmask(_SIG_SETMASK, &oset, nil) 154 155 if ret < 0 { 156 print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", -ret, ")\n") 157 if ret == -_EAGAIN { 158 println("runtime: may need to increase max user processes (ulimit -u)") 159 } 160 throw("newosproc") 161 } 162 } 163 164 // Version of newosproc that doesn't require a valid G. 165 //go:nosplit 166 func newosproc0(stacksize uintptr, fn unsafe.Pointer) { 167 stack := sysAlloc(stacksize, &memstats.stacks_sys) 168 if stack == nil { 169 write(2, unsafe.Pointer(&failallocatestack[0]), int32(len(failallocatestack))) 170 exit(1) 171 } 172 ret := clone(cloneFlags, unsafe.Pointer(uintptr(stack)+stacksize), nil, nil, fn) 173 if ret < 0 { 174 write(2, unsafe.Pointer(&failthreadcreate[0]), int32(len(failthreadcreate))) 175 exit(1) 176 } 177 } 178 179 var failallocatestack = []byte("runtime: failed to allocate stack for the new OS thread\n") 180 var failthreadcreate = []byte("runtime: failed to create new OS thread\n") 181 182 const ( 183 _AT_NULL = 0 // End of vector 184 _AT_PAGESZ = 6 // System physical page size 185 _AT_HWCAP = 16 // hardware capability bit vector 186 _AT_RANDOM = 25 // introduced in 2.6.29 187 _AT_HWCAP2 = 26 // hardware capability bit vector 2 188 ) 189 190 var procAuxv = []byte("/proc/self/auxv\x00") 191 192 var addrspace_vec [1]byte 193 194 func mincore(addr unsafe.Pointer, n uintptr, dst *byte) int32 195 196 func sysargs(argc int32, argv **byte) { 197 n := argc + 1 198 199 // skip over argv, envp to get to auxv 200 for argv_index(argv, n) != nil { 201 n++ 202 } 203 204 // skip NULL separator 205 n++ 206 207 // now argv+n is auxv 208 auxv := (*[1 << 28]uintptr)(add(unsafe.Pointer(argv), uintptr(n)*sys.PtrSize)) 209 if sysauxv(auxv[:]) != 0 { 210 return 211 } 212 // In some situations we don't get a loader-provided 213 // auxv, such as when loaded as a library on Android. 214 // Fall back to /proc/self/auxv. 215 fd := open(&procAuxv[0], 0 /* O_RDONLY */, 0) 216 if fd < 0 { 217 // On Android, /proc/self/auxv might be unreadable (issue 9229), so we fallback to 218 // try using mincore to detect the physical page size. 219 // mincore should return EINVAL when address is not a multiple of system page size. 220 const size = 256 << 10 // size of memory region to allocate 221 p, err := mmap(nil, size, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0) 222 if err != 0 { 223 return 224 } 225 var n uintptr 226 for n = 4 << 10; n < size; n <<= 1 { 227 err := mincore(unsafe.Pointer(uintptr(p)+n), 1, &addrspace_vec[0]) 228 if err == 0 { 229 physPageSize = n 230 break 231 } 232 } 233 if physPageSize == 0 { 234 physPageSize = size 235 } 236 munmap(p, size) 237 return 238 } 239 var buf [128]uintptr 240 n = read(fd, noescape(unsafe.Pointer(&buf[0])), int32(unsafe.Sizeof(buf))) 241 closefd(fd) 242 if n < 0 { 243 return 244 } 245 // Make sure buf is terminated, even if we didn't read 246 // the whole file. 247 buf[len(buf)-2] = _AT_NULL 248 sysauxv(buf[:]) 249 } 250 251 // startupRandomData holds random bytes initialized at startup. These come from 252 // the ELF AT_RANDOM auxiliary vector. 253 var startupRandomData []byte 254 255 func sysauxv(auxv []uintptr) int { 256 var i int 257 for ; auxv[i] != _AT_NULL; i += 2 { 258 tag, val := auxv[i], auxv[i+1] 259 switch tag { 260 case _AT_RANDOM: 261 // The kernel provides a pointer to 16-bytes 262 // worth of random data. 263 startupRandomData = (*[16]byte)(unsafe.Pointer(val))[:] 264 265 case _AT_PAGESZ: 266 physPageSize = val 267 } 268 269 archauxv(tag, val) 270 vdsoauxv(tag, val) 271 } 272 return i / 2 273 } 274 275 var sysTHPSizePath = []byte("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size\x00") 276 277 func getHugePageSize() uintptr { 278 var numbuf [20]byte 279 fd := open(&sysTHPSizePath[0], 0 /* O_RDONLY */, 0) 280 if fd < 0 { 281 return 0 282 } 283 ptr := noescape(unsafe.Pointer(&numbuf[0])) 284 n := read(fd, ptr, int32(len(numbuf))) 285 closefd(fd) 286 if n <= 0 { 287 return 0 288 } 289 n-- // remove trailing newline 290 v, ok := atoi(slicebytetostringtmp((*byte)(ptr), int(n))) 291 if !ok || v < 0 { 292 v = 0 293 } 294 if v&(v-1) != 0 { 295 // v is not a power of 2 296 return 0 297 } 298 return uintptr(v) 299 } 300 301 func osinit() { 302 ncpu = getproccount() 303 physHugePageSize = getHugePageSize() 304 if iscgo { 305 // #42494 glibc and musl reserve some signals for 306 // internal use and require they not be blocked by 307 // the rest of a normal C runtime. When the go runtime 308 // blocks...unblocks signals, temporarily, the blocked 309 // interval of time is generally very short. As such, 310 // these expectations of *libc code are mostly met by 311 // the combined go+cgo system of threads. However, 312 // when go causes a thread to exit, via a return from 313 // mstart(), the combined runtime can deadlock if 314 // these signals are blocked. Thus, don't block these 315 // signals when exiting threads. 316 // - glibc: SIGCANCEL (32), SIGSETXID (33) 317 // - musl: SIGTIMER (32), SIGCANCEL (33), SIGSYNCCALL (34) 318 sigdelset(&sigsetAllExiting, 32) 319 sigdelset(&sigsetAllExiting, 33) 320 sigdelset(&sigsetAllExiting, 34) 321 } 322 osArchInit() 323 } 324 325 var urandom_dev = []byte("/dev/urandom\x00") 326 327 func getRandomData(r []byte) { 328 if startupRandomData != nil { 329 n := copy(r, startupRandomData) 330 extendRandom(r, n) 331 return 332 } 333 fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) 334 n := read(fd, unsafe.Pointer(&r[0]), int32(len(r))) 335 closefd(fd) 336 extendRandom(r, int(n)) 337 } 338 339 func goenvs() { 340 goenvs_unix() 341 } 342 343 // Called to do synchronous initialization of Go code built with 344 // -buildmode=c-archive or -buildmode=c-shared. 345 // None of the Go runtime is initialized. 346 //go:nosplit 347 //go:nowritebarrierrec 348 func libpreinit() { 349 initsig(true) 350 } 351 352 // Called to initialize a new m (including the bootstrap m). 353 // Called on the parent thread (main thread in case of bootstrap), can allocate memory. 354 func mpreinit(mp *m) { 355 mp.gsignal = malg(32 * 1024) // Linux wants >= 2K 356 mp.gsignal.m = mp 357 } 358 359 func gettid() uint32 360 361 // Called to initialize a new m (including the bootstrap m). 362 // Called on the new thread, cannot allocate memory. 363 func minit() { 364 minitSignals() 365 366 // Cgo-created threads and the bootstrap m are missing a 367 // procid. We need this for asynchronous preemption and it's 368 // useful in debuggers. 369 getg().m.procid = uint64(gettid()) 370 } 371 372 // Called from dropm to undo the effect of an minit. 373 //go:nosplit 374 func unminit() { 375 unminitSignals() 376 } 377 378 // Called from exitm, but not from drop, to undo the effect of thread-owned 379 // resources in minit, semacreate, or elsewhere. Do not take locks after calling this. 380 func mdestroy(mp *m) { 381 } 382 383 //#ifdef GOARCH_386 384 //#define sa_handler k_sa_handler 385 //#endif 386 387 func sigreturn() 388 func sigtramp() // Called via C ABI 389 func cgoSigtramp() 390 391 //go:noescape 392 func sigaltstack(new, old *stackt) 393 394 //go:noescape 395 func setitimer(mode int32, new, old *itimerval) 396 397 //go:noescape 398 func rtsigprocmask(how int32, new, old *sigset, size int32) 399 400 //go:nosplit 401 //go:nowritebarrierrec 402 func sigprocmask(how int32, new, old *sigset) { 403 rtsigprocmask(how, new, old, int32(unsafe.Sizeof(*new))) 404 } 405 406 func raise(sig uint32) 407 func raiseproc(sig uint32) 408 409 //go:noescape 410 func sched_getaffinity(pid, len uintptr, buf *byte) int32 411 func osyield() 412 413 //go:nosplit 414 func osyield_no_g() { 415 osyield() 416 } 417 418 func pipe() (r, w int32, errno int32) 419 func pipe2(flags int32) (r, w int32, errno int32) 420 func setNonblock(fd int32) 421 422 //go:nosplit 423 //go:nowritebarrierrec 424 func setsig(i uint32, fn uintptr) { 425 var sa sigactiont 426 sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK | _SA_RESTORER | _SA_RESTART 427 sigfillset(&sa.sa_mask) 428 // Although Linux manpage says "sa_restorer element is obsolete and 429 // should not be used". x86_64 kernel requires it. Only use it on 430 // x86. 431 if GOARCH == "386" || GOARCH == "amd64" { 432 sa.sa_restorer = funcPC(sigreturn) 433 } 434 if fn == funcPC(sighandler) { 435 if iscgo { 436 fn = funcPC(cgoSigtramp) 437 } else { 438 fn = funcPC(sigtramp) 439 } 440 } 441 sa.sa_handler = fn 442 sigaction(i, &sa, nil) 443 } 444 445 //go:nosplit 446 //go:nowritebarrierrec 447 func setsigstack(i uint32) { 448 var sa sigactiont 449 sigaction(i, nil, &sa) 450 if sa.sa_flags&_SA_ONSTACK != 0 { 451 return 452 } 453 sa.sa_flags |= _SA_ONSTACK 454 sigaction(i, &sa, nil) 455 } 456 457 //go:nosplit 458 //go:nowritebarrierrec 459 func getsig(i uint32) uintptr { 460 var sa sigactiont 461 sigaction(i, nil, &sa) 462 return sa.sa_handler 463 } 464 465 // setSignaltstackSP sets the ss_sp field of a stackt. 466 //go:nosplit 467 func setSignalstackSP(s *stackt, sp uintptr) { 468 *(*uintptr)(unsafe.Pointer(&s.ss_sp)) = sp 469 } 470 471 //go:nosplit 472 func (c *sigctxt) fixsigcode(sig uint32) { 473 } 474 475 // sysSigaction calls the rt_sigaction system call. 476 //go:nosplit 477 func sysSigaction(sig uint32, new, old *sigactiont) { 478 if rt_sigaction(uintptr(sig), new, old, unsafe.Sizeof(sigactiont{}.sa_mask)) != 0 { 479 // Workaround for bugs in QEMU user mode emulation. 480 // 481 // QEMU turns calls to the sigaction system call into 482 // calls to the C library sigaction call; the C 483 // library call rejects attempts to call sigaction for 484 // SIGCANCEL (32) or SIGSETXID (33). 485 // 486 // QEMU rejects calling sigaction on SIGRTMAX (64). 487 // 488 // Just ignore the error in these case. There isn't 489 // anything we can do about it anyhow. 490 if sig != 32 && sig != 33 && sig != 64 { 491 // Use system stack to avoid split stack overflow on ppc64/ppc64le. 492 systemstack(func() { 493 throw("sigaction failed") 494 }) 495 } 496 } 497 } 498 499 // rt_sigaction is implemented in assembly. 500 //go:noescape 501 func rt_sigaction(sig uintptr, new, old *sigactiont, size uintptr) int32 502 503 func getpid() int 504 func tgkill(tgid, tid, sig int) 505 506 // signalM sends a signal to mp. 507 func signalM(mp *m, sig int) { 508 tgkill(getpid(), int(mp.procid), sig) 509 }