github.com/lovishpuri/go-40569/src@v0.0.0-20230519171745-f8623e7c56cf/runtime/stubs.go (about) 1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/abi" 9 "internal/goarch" 10 "runtime/internal/math" 11 "unsafe" 12 ) 13 14 // Should be a built-in for unsafe.Pointer? 15 // 16 //go:nosplit 17 func add(p unsafe.Pointer, x uintptr) unsafe.Pointer { 18 return unsafe.Pointer(uintptr(p) + x) 19 } 20 21 // getg returns the pointer to the current g. 22 // The compiler rewrites calls to this function into instructions 23 // that fetch the g directly (from TLS or from the dedicated register). 24 func getg() *g 25 26 // mcall switches from the g to the g0 stack and invokes fn(g), 27 // where g is the goroutine that made the call. 28 // mcall saves g's current PC/SP in g->sched so that it can be restored later. 29 // It is up to fn to arrange for that later execution, typically by recording 30 // g in a data structure, causing something to call ready(g) later. 31 // mcall returns to the original goroutine g later, when g has been rescheduled. 32 // fn must not return at all; typically it ends by calling schedule, to let the m 33 // run other goroutines. 34 // 35 // mcall can only be called from g stacks (not g0, not gsignal). 36 // 37 // This must NOT be go:noescape: if fn is a stack-allocated closure, 38 // fn puts g on a run queue, and g executes before fn returns, the 39 // closure will be invalidated while it is still executing. 40 func mcall(fn func(*g)) 41 42 // systemstack runs fn on a system stack. 43 // If systemstack is called from the per-OS-thread (g0) stack, or 44 // if systemstack is called from the signal handling (gsignal) stack, 45 // systemstack calls fn directly and returns. 46 // Otherwise, systemstack is being called from the limited stack 47 // of an ordinary goroutine. In this case, systemstack switches 48 // to the per-OS-thread stack, calls fn, and switches back. 49 // It is common to use a func literal as the argument, in order 50 // to share inputs and outputs with the code around the call 51 // to system stack: 52 // 53 // ... set up y ... 54 // systemstack(func() { 55 // x = bigcall(y) 56 // }) 57 // ... use x ... 58 // 59 //go:noescape 60 func systemstack(fn func()) 61 62 //go:nosplit 63 //go:nowritebarrierrec 64 func badsystemstack() { 65 writeErrStr("fatal: systemstack called from unexpected goroutine") 66 } 67 68 // memclrNoHeapPointers clears n bytes starting at ptr. 69 // 70 // Usually you should use typedmemclr. memclrNoHeapPointers should be 71 // used only when the caller knows that *ptr contains no heap pointers 72 // because either: 73 // 74 // *ptr is initialized memory and its type is pointer-free, or 75 // 76 // *ptr is uninitialized memory (e.g., memory that's being reused 77 // for a new allocation) and hence contains only "junk". 78 // 79 // memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n 80 // is a multiple of the pointer size, then any pointer-aligned, 81 // pointer-sized portion is cleared atomically. Despite the function 82 // name, this is necessary because this function is the underlying 83 // implementation of typedmemclr and memclrHasPointers. See the doc of 84 // memmove for more details. 85 // 86 // The (CPU-specific) implementations of this function are in memclr_*.s. 87 // 88 //go:noescape 89 func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) 90 91 //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers 92 func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) { 93 memclrNoHeapPointers(ptr, n) 94 } 95 96 // memmove copies n bytes from "from" to "to". 97 // 98 // memmove ensures that any pointer in "from" is written to "to" with 99 // an indivisible write, so that racy reads cannot observe a 100 // half-written pointer. This is necessary to prevent the garbage 101 // collector from observing invalid pointers, and differs from memmove 102 // in unmanaged languages. However, memmove is only required to do 103 // this if "from" and "to" may contain pointers, which can only be the 104 // case if "from", "to", and "n" are all be word-aligned. 105 // 106 // Implementations are in memmove_*.s. 107 // 108 //go:noescape 109 func memmove(to, from unsafe.Pointer, n uintptr) 110 111 // Outside assembly calls memmove. Make sure it has ABI wrappers. 112 // 113 //go:linkname memmove 114 115 //go:linkname reflect_memmove reflect.memmove 116 func reflect_memmove(to, from unsafe.Pointer, n uintptr) { 117 memmove(to, from, n) 118 } 119 120 // exported value for testing 121 const hashLoad = float32(loadFactorNum) / float32(loadFactorDen) 122 123 //go:nosplit 124 func fastrand() uint32 { 125 mp := getg().m 126 // Implement wyrand: https://github.com/wangyi-fudan/wyhash 127 // Only the platform that math.Mul64 can be lowered 128 // by the compiler should be in this list. 129 if goarch.IsAmd64|goarch.IsArm64|goarch.IsPpc64| 130 goarch.IsPpc64le|goarch.IsMips64|goarch.IsMips64le| 131 goarch.IsS390x|goarch.IsRiscv64|goarch.IsLoong64 == 1 { 132 mp.fastrand += 0xa0761d6478bd642f 133 hi, lo := math.Mul64(mp.fastrand, mp.fastrand^0xe7037ed1a0b428db) 134 return uint32(hi ^ lo) 135 } 136 137 // Implement xorshift64+: 2 32-bit xorshift sequences added together. 138 // Shift triplet [17,7,16] was calculated as indicated in Marsaglia's 139 // Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf 140 // This generator passes the SmallCrush suite, part of TestU01 framework: 141 // http://simul.iro.umontreal.ca/testu01/tu01.html 142 t := (*[2]uint32)(unsafe.Pointer(&mp.fastrand)) 143 s1, s0 := t[0], t[1] 144 s1 ^= s1 << 17 145 s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16 146 t[0], t[1] = s0, s1 147 return s0 + s1 148 } 149 150 //go:nosplit 151 func fastrandn(n uint32) uint32 { 152 // This is similar to fastrand() % n, but faster. 153 // See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ 154 return uint32(uint64(fastrand()) * uint64(n) >> 32) 155 } 156 157 func fastrand64() uint64 { 158 mp := getg().m 159 // Implement wyrand: https://github.com/wangyi-fudan/wyhash 160 // Only the platform that math.Mul64 can be lowered 161 // by the compiler should be in this list. 162 if goarch.IsAmd64|goarch.IsArm64|goarch.IsPpc64| 163 goarch.IsPpc64le|goarch.IsMips64|goarch.IsMips64le| 164 goarch.IsS390x|goarch.IsRiscv64 == 1 { 165 mp.fastrand += 0xa0761d6478bd642f 166 hi, lo := math.Mul64(mp.fastrand, mp.fastrand^0xe7037ed1a0b428db) 167 return hi ^ lo 168 } 169 170 // Implement xorshift64+: 2 32-bit xorshift sequences added together. 171 // Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf 172 // This generator passes the SmallCrush suite, part of TestU01 framework: 173 // http://simul.iro.umontreal.ca/testu01/tu01.html 174 t := (*[2]uint32)(unsafe.Pointer(&mp.fastrand)) 175 s1, s0 := t[0], t[1] 176 s1 ^= s1 << 17 177 s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16 178 r := uint64(s0 + s1) 179 180 s0, s1 = s1, s0 181 s1 ^= s1 << 17 182 s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16 183 r += uint64(s0+s1) << 32 184 185 t[0], t[1] = s0, s1 186 return r 187 } 188 189 func fastrandu() uint { 190 if goarch.PtrSize == 4 { 191 return uint(fastrand()) 192 } 193 return uint(fastrand64()) 194 } 195 196 //go:linkname rand_fastrand64 math/rand.fastrand64 197 func rand_fastrand64() uint64 { return fastrand64() } 198 199 //go:linkname sync_fastrandn sync.fastrandn 200 func sync_fastrandn(n uint32) uint32 { return fastrandn(n) } 201 202 //go:linkname net_fastrandu net.fastrandu 203 func net_fastrandu() uint { return fastrandu() } 204 205 //go:linkname os_fastrand os.fastrand 206 func os_fastrand() uint32 { return fastrand() } 207 208 // in internal/bytealg/equal_*.s 209 // 210 //go:noescape 211 func memequal(a, b unsafe.Pointer, size uintptr) bool 212 213 // noescape hides a pointer from escape analysis. noescape is 214 // the identity function but escape analysis doesn't think the 215 // output depends on the input. noescape is inlined and currently 216 // compiles down to zero instructions. 217 // USE CAREFULLY! 218 // 219 //go:nosplit 220 func noescape(p unsafe.Pointer) unsafe.Pointer { 221 x := uintptr(p) 222 return unsafe.Pointer(x ^ 0) 223 } 224 225 // noEscapePtr hides a pointer from escape analysis. See noescape. 226 // USE CAREFULLY! 227 // 228 //go:nosplit 229 func noEscapePtr[T any](p *T) *T { 230 x := uintptr(unsafe.Pointer(p)) 231 return (*T)(unsafe.Pointer(x ^ 0)) 232 } 233 234 // Not all cgocallback frames are actually cgocallback, 235 // so not all have these arguments. Mark them uintptr so that the GC 236 // does not misinterpret memory when the arguments are not present. 237 // cgocallback is not called from Go, only from crosscall2. 238 // This in turn calls cgocallbackg, which is where we'll find 239 // pointer-declared arguments. 240 // 241 // When fn is nil (frame is saved g), call dropm instead, 242 // this is used when the C thread is exiting. 243 func cgocallback(fn, frame, ctxt uintptr) 244 245 func gogo(buf *gobuf) 246 247 func asminit() 248 func setg(gg *g) 249 func breakpoint() 250 251 // reflectcall calls fn with arguments described by stackArgs, stackArgsSize, 252 // frameSize, and regArgs. 253 // 254 // Arguments passed on the stack and space for return values passed on the stack 255 // must be laid out at the space pointed to by stackArgs (with total length 256 // stackArgsSize) according to the ABI. 257 // 258 // stackRetOffset must be some value <= stackArgsSize that indicates the 259 // offset within stackArgs where the return value space begins. 260 // 261 // frameSize is the total size of the argument frame at stackArgs and must 262 // therefore be >= stackArgsSize. It must include additional space for spilling 263 // register arguments for stack growth and preemption. 264 // 265 // TODO(mknyszek): Once we don't need the additional spill space, remove frameSize, 266 // since frameSize will be redundant with stackArgsSize. 267 // 268 // Arguments passed in registers must be laid out in regArgs according to the ABI. 269 // regArgs will hold any return values passed in registers after the call. 270 // 271 // reflectcall copies stack arguments from stackArgs to the goroutine stack, and 272 // then copies back stackArgsSize-stackRetOffset bytes back to the return space 273 // in stackArgs once fn has completed. It also "unspills" argument registers from 274 // regArgs before calling fn, and spills them back into regArgs immediately 275 // following the call to fn. If there are results being returned on the stack, 276 // the caller should pass the argument frame type as stackArgsType so that 277 // reflectcall can execute appropriate write barriers during the copy. 278 // 279 // reflectcall expects regArgs.ReturnIsPtr to be populated indicating which 280 // registers on the return path will contain Go pointers. It will then store 281 // these pointers in regArgs.Ptrs such that they are visible to the GC. 282 // 283 // Package reflect passes a frame type. In package runtime, there is only 284 // one call that copies results back, in callbackWrap in syscall_windows.go, and it 285 // does NOT pass a frame type, meaning there are no write barriers invoked. See that 286 // call site for justification. 287 // 288 // Package reflect accesses this symbol through a linkname. 289 // 290 // Arguments passed through to reflectcall do not escape. The type is used 291 // only in a very limited callee of reflectcall, the stackArgs are copied, and 292 // regArgs is only used in the reflectcall frame. 293 // 294 //go:noescape 295 func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 296 297 func procyield(cycles uint32) 298 299 type neverCallThisFunction struct{} 300 301 // goexit is the return stub at the top of every goroutine call stack. 302 // Each goroutine stack is constructed as if goexit called the 303 // goroutine's entry point function, so that when the entry point 304 // function returns, it will return to goexit, which will call goexit1 305 // to perform the actual exit. 306 // 307 // This function must never be called directly. Call goexit1 instead. 308 // gentraceback assumes that goexit terminates the stack. A direct 309 // call on the stack will cause gentraceback to stop walking the stack 310 // prematurely and if there is leftover state it may panic. 311 func goexit(neverCallThisFunction) 312 313 // publicationBarrier performs a store/store barrier (a "publication" 314 // or "export" barrier). Some form of synchronization is required 315 // between initializing an object and making that object accessible to 316 // another processor. Without synchronization, the initialization 317 // writes and the "publication" write may be reordered, allowing the 318 // other processor to follow the pointer and observe an uninitialized 319 // object. In general, higher-level synchronization should be used, 320 // such as locking or an atomic pointer write. publicationBarrier is 321 // for when those aren't an option, such as in the implementation of 322 // the memory manager. 323 // 324 // There's no corresponding barrier for the read side because the read 325 // side naturally has a data dependency order. All architectures that 326 // Go supports or seems likely to ever support automatically enforce 327 // data dependency ordering. 328 func publicationBarrier() 329 330 // getcallerpc returns the program counter (PC) of its caller's caller. 331 // getcallersp returns the stack pointer (SP) of its caller's caller. 332 // The implementation may be a compiler intrinsic; there is not 333 // necessarily code implementing this on every platform. 334 // 335 // For example: 336 // 337 // func f(arg1, arg2, arg3 int) { 338 // pc := getcallerpc() 339 // sp := getcallersp() 340 // } 341 // 342 // These two lines find the PC and SP immediately following 343 // the call to f (where f will return). 344 // 345 // The call to getcallerpc and getcallersp must be done in the 346 // frame being asked about. 347 // 348 // The result of getcallersp is correct at the time of the return, 349 // but it may be invalidated by any subsequent call to a function 350 // that might relocate the stack in order to grow or shrink it. 351 // A general rule is that the result of getcallersp should be used 352 // immediately and can only be passed to nosplit functions. 353 354 //go:noescape 355 func getcallerpc() uintptr 356 357 //go:noescape 358 func getcallersp() uintptr // implemented as an intrinsic on all platforms 359 360 // getclosureptr returns the pointer to the current closure. 361 // getclosureptr can only be used in an assignment statement 362 // at the entry of a function. Moreover, go:nosplit directive 363 // must be specified at the declaration of caller function, 364 // so that the function prolog does not clobber the closure register. 365 // for example: 366 // 367 // //go:nosplit 368 // func f(arg1, arg2, arg3 int) { 369 // dx := getclosureptr() 370 // } 371 // 372 // The compiler rewrites calls to this function into instructions that fetch the 373 // pointer from a well-known register (DX on x86 architecture, etc.) directly. 374 func getclosureptr() uintptr 375 376 //go:noescape 377 func asmcgocall(fn, arg unsafe.Pointer) int32 378 379 func morestack() 380 func morestack_noctxt() 381 func rt0_go() 382 383 // return0 is a stub used to return 0 from deferproc. 384 // It is called at the very end of deferproc to signal 385 // the calling Go function that it should not jump 386 // to deferreturn. 387 // in asm_*.s 388 func return0() 389 390 // in asm_*.s 391 // not called directly; definitions here supply type information for traceback. 392 // These must have the same signature (arg pointer map) as reflectcall. 393 func call16(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 394 func call32(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 395 func call64(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 396 func call128(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 397 func call256(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 398 func call512(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 399 func call1024(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 400 func call2048(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 401 func call4096(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 402 func call8192(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 403 func call16384(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 404 func call32768(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 405 func call65536(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 406 func call131072(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 407 func call262144(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 408 func call524288(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 409 func call1048576(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 410 func call2097152(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 411 func call4194304(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 412 func call8388608(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 413 func call16777216(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 414 func call33554432(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 415 func call67108864(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 416 func call134217728(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 417 func call268435456(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 418 func call536870912(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 419 func call1073741824(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 420 421 func systemstack_switch() 422 423 // alignUp rounds n up to a multiple of a. a must be a power of 2. 424 func alignUp(n, a uintptr) uintptr { 425 return (n + a - 1) &^ (a - 1) 426 } 427 428 // alignDown rounds n down to a multiple of a. a must be a power of 2. 429 func alignDown(n, a uintptr) uintptr { 430 return n &^ (a - 1) 431 } 432 433 // divRoundUp returns ceil(n / a). 434 func divRoundUp(n, a uintptr) uintptr { 435 // a is generally a power of two. This will get inlined and 436 // the compiler will optimize the division. 437 return (n + a - 1) / a 438 } 439 440 // checkASM reports whether assembly runtime checks have passed. 441 func checkASM() bool 442 443 func memequal_varlen(a, b unsafe.Pointer) bool 444 445 // bool2int returns 0 if x is false or 1 if x is true. 446 func bool2int(x bool) int { 447 // Avoid branches. In the SSA compiler, this compiles to 448 // exactly what you would want it to. 449 return int(uint8(*(*uint8)(unsafe.Pointer(&x)))) 450 } 451 452 // abort crashes the runtime in situations where even throw might not 453 // work. In general it should do something a debugger will recognize 454 // (e.g., an INT3 on x86). A crash in abort is recognized by the 455 // signal handler, which will attempt to tear down the runtime 456 // immediately. 457 func abort() 458 459 // Called from compiled code; declared for vet; do NOT call from Go. 460 func gcWriteBarrier1() 461 func gcWriteBarrier2() 462 func gcWriteBarrier3() 463 func gcWriteBarrier4() 464 func gcWriteBarrier5() 465 func gcWriteBarrier6() 466 func gcWriteBarrier7() 467 func gcWriteBarrier8() 468 func duffzero() 469 func duffcopy() 470 471 // Called from linker-generated .initarray; declared for go vet; do NOT call from Go. 472 func addmoduledata() 473 474 // Injected by the signal handler for panicking signals. 475 // Initializes any registers that have fixed meaning at calls but 476 // are scratch in bodies and calls sigpanic. 477 // On many platforms it just jumps to sigpanic. 478 func sigpanic0() 479 480 // intArgRegs is used by the various register assignment 481 // algorithm implementations in the runtime. These include:. 482 // - Finalizers (mfinal.go) 483 // - Windows callbacks (syscall_windows.go) 484 // 485 // Both are stripped-down versions of the algorithm since they 486 // only have to deal with a subset of cases (finalizers only 487 // take a pointer or interface argument, Go Windows callbacks 488 // don't support floating point). 489 // 490 // It should be modified with care and are generally only 491 // modified when testing this package. 492 // 493 // It should never be set higher than its internal/abi 494 // constant counterparts, because the system relies on a 495 // structure that is at least large enough to hold the 496 // registers the system supports. 497 // 498 // Protected by finlock. 499 var intArgRegs = abi.IntArgRegs