github.com/ice-blockchain/go/src@v0.0.0-20240403114104-1564d284e521/runtime/os_windows.go (about) 1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/abi" 9 "internal/goarch" 10 "runtime/internal/atomic" 11 "unsafe" 12 ) 13 14 // TODO(brainman): should not need those 15 const ( 16 _NSIG = 65 17 ) 18 19 //go:cgo_import_dynamic runtime._AddVectoredContinueHandler AddVectoredContinueHandler%2 "kernel32.dll" 20 //go:cgo_import_dynamic runtime._AddVectoredExceptionHandler AddVectoredExceptionHandler%2 "kernel32.dll" 21 //go:cgo_import_dynamic runtime._CloseHandle CloseHandle%1 "kernel32.dll" 22 //go:cgo_import_dynamic runtime._CreateEventA CreateEventA%4 "kernel32.dll" 23 //go:cgo_import_dynamic runtime._CreateIoCompletionPort CreateIoCompletionPort%4 "kernel32.dll" 24 //go:cgo_import_dynamic runtime._CreateThread CreateThread%6 "kernel32.dll" 25 //go:cgo_import_dynamic runtime._CreateWaitableTimerA CreateWaitableTimerA%3 "kernel32.dll" 26 //go:cgo_import_dynamic runtime._CreateWaitableTimerExW CreateWaitableTimerExW%4 "kernel32.dll" 27 //go:cgo_import_dynamic runtime._DuplicateHandle DuplicateHandle%7 "kernel32.dll" 28 //go:cgo_import_dynamic runtime._ExitProcess ExitProcess%1 "kernel32.dll" 29 //go:cgo_import_dynamic runtime._FreeEnvironmentStringsW FreeEnvironmentStringsW%1 "kernel32.dll" 30 //go:cgo_import_dynamic runtime._GetConsoleMode GetConsoleMode%2 "kernel32.dll" 31 //go:cgo_import_dynamic runtime._GetCurrentThreadId GetCurrentThreadId%0 "kernel32.dll" 32 //go:cgo_import_dynamic runtime._GetEnvironmentStringsW GetEnvironmentStringsW%0 "kernel32.dll" 33 //go:cgo_import_dynamic runtime._GetErrorMode GetErrorMode%0 "kernel32.dll" 34 //go:cgo_import_dynamic runtime._GetProcAddress GetProcAddress%2 "kernel32.dll" 35 //go:cgo_import_dynamic runtime._GetProcessAffinityMask GetProcessAffinityMask%3 "kernel32.dll" 36 //go:cgo_import_dynamic runtime._GetQueuedCompletionStatusEx GetQueuedCompletionStatusEx%6 "kernel32.dll" 37 //go:cgo_import_dynamic runtime._GetStdHandle GetStdHandle%1 "kernel32.dll" 38 //go:cgo_import_dynamic runtime._GetSystemDirectoryA GetSystemDirectoryA%2 "kernel32.dll" 39 //go:cgo_import_dynamic runtime._GetSystemInfo GetSystemInfo%1 "kernel32.dll" 40 //go:cgo_import_dynamic runtime._GetThreadContext GetThreadContext%2 "kernel32.dll" 41 //go:cgo_import_dynamic runtime._SetThreadContext SetThreadContext%2 "kernel32.dll" 42 //go:cgo_import_dynamic runtime._LoadLibraryExW LoadLibraryExW%3 "kernel32.dll" 43 //go:cgo_import_dynamic runtime._LoadLibraryW LoadLibraryW%1 "kernel32.dll" 44 //go:cgo_import_dynamic runtime._PostQueuedCompletionStatus PostQueuedCompletionStatus%4 "kernel32.dll" 45 //go:cgo_import_dynamic runtime._QueryPerformanceCounter QueryPerformanceCounter%1 "kernel32.dll" 46 //go:cgo_import_dynamic runtime._RaiseFailFastException RaiseFailFastException%3 "kernel32.dll" 47 //go:cgo_import_dynamic runtime._ResumeThread ResumeThread%1 "kernel32.dll" 48 //go:cgo_import_dynamic runtime._RtlLookupFunctionEntry RtlLookupFunctionEntry%3 "kernel32.dll" 49 //go:cgo_import_dynamic runtime._RtlVirtualUnwind RtlVirtualUnwind%8 "kernel32.dll" 50 //go:cgo_import_dynamic runtime._SetConsoleCtrlHandler SetConsoleCtrlHandler%2 "kernel32.dll" 51 //go:cgo_import_dynamic runtime._SetErrorMode SetErrorMode%1 "kernel32.dll" 52 //go:cgo_import_dynamic runtime._SetEvent SetEvent%1 "kernel32.dll" 53 //go:cgo_import_dynamic runtime._SetProcessPriorityBoost SetProcessPriorityBoost%2 "kernel32.dll" 54 //go:cgo_import_dynamic runtime._SetThreadPriority SetThreadPriority%2 "kernel32.dll" 55 //go:cgo_import_dynamic runtime._SetUnhandledExceptionFilter SetUnhandledExceptionFilter%1 "kernel32.dll" 56 //go:cgo_import_dynamic runtime._SetWaitableTimer SetWaitableTimer%6 "kernel32.dll" 57 //go:cgo_import_dynamic runtime._SuspendThread SuspendThread%1 "kernel32.dll" 58 //go:cgo_import_dynamic runtime._SwitchToThread SwitchToThread%0 "kernel32.dll" 59 //go:cgo_import_dynamic runtime._TlsAlloc TlsAlloc%0 "kernel32.dll" 60 //go:cgo_import_dynamic runtime._VirtualAlloc VirtualAlloc%4 "kernel32.dll" 61 //go:cgo_import_dynamic runtime._VirtualFree VirtualFree%3 "kernel32.dll" 62 //go:cgo_import_dynamic runtime._VirtualQuery VirtualQuery%3 "kernel32.dll" 63 //go:cgo_import_dynamic runtime._WaitForSingleObject WaitForSingleObject%2 "kernel32.dll" 64 //go:cgo_import_dynamic runtime._WaitForMultipleObjects WaitForMultipleObjects%4 "kernel32.dll" 65 //go:cgo_import_dynamic runtime._WerGetFlags WerGetFlags%2 "kernel32.dll" 66 //go:cgo_import_dynamic runtime._WerSetFlags WerSetFlags%1 "kernel32.dll" 67 //go:cgo_import_dynamic runtime._WriteConsoleW WriteConsoleW%5 "kernel32.dll" 68 //go:cgo_import_dynamic runtime._WriteFile WriteFile%5 "kernel32.dll" 69 70 type stdFunction unsafe.Pointer 71 72 var ( 73 // Following syscalls are available on every Windows PC. 74 // All these variables are set by the Windows executable 75 // loader before the Go program starts. 76 _AddVectoredContinueHandler, 77 _AddVectoredExceptionHandler, 78 _CloseHandle, 79 _CreateEventA, 80 _CreateIoCompletionPort, 81 _CreateThread, 82 _CreateWaitableTimerA, 83 _CreateWaitableTimerExW, 84 _DuplicateHandle, 85 _ExitProcess, 86 _FreeEnvironmentStringsW, 87 _GetConsoleMode, 88 _GetCurrentThreadId, 89 _GetEnvironmentStringsW, 90 _GetErrorMode, 91 _GetProcAddress, 92 _GetProcessAffinityMask, 93 _GetQueuedCompletionStatusEx, 94 _GetStdHandle, 95 _GetSystemDirectoryA, 96 _GetSystemInfo, 97 _GetThreadContext, 98 _SetThreadContext, 99 _LoadLibraryExW, 100 _LoadLibraryW, 101 _PostQueuedCompletionStatus, 102 _QueryPerformanceCounter, 103 _RaiseFailFastException, 104 _ResumeThread, 105 _RtlLookupFunctionEntry, 106 _RtlVirtualUnwind, 107 _SetConsoleCtrlHandler, 108 _SetErrorMode, 109 _SetEvent, 110 _SetProcessPriorityBoost, 111 _SetThreadPriority, 112 _SetUnhandledExceptionFilter, 113 _SetWaitableTimer, 114 _SuspendThread, 115 _SwitchToThread, 116 _TlsAlloc, 117 _VirtualAlloc, 118 _VirtualFree, 119 _VirtualQuery, 120 _WaitForSingleObject, 121 _WaitForMultipleObjects, 122 _WerGetFlags, 123 _WerSetFlags, 124 _WriteConsoleW, 125 _WriteFile, 126 _ stdFunction 127 128 // Use ProcessPrng to generate cryptographically random data. 129 _ProcessPrng stdFunction 130 131 // Load ntdll.dll manually during startup, otherwise Mingw 132 // links wrong printf function to cgo executable (see issue 133 // 12030 for details). 134 _NtCreateWaitCompletionPacket stdFunction 135 _NtAssociateWaitCompletionPacket stdFunction 136 _NtCancelWaitCompletionPacket stdFunction 137 _RtlGetCurrentPeb stdFunction 138 _RtlGetVersion stdFunction 139 140 // These are from non-kernel32.dll, so we prefer to LoadLibraryEx them. 141 _timeBeginPeriod, 142 _timeEndPeriod, 143 _ stdFunction 144 ) 145 146 var ( 147 bcryptprimitivesdll = [...]uint16{'b', 'c', 'r', 'y', 'p', 't', 'p', 'r', 'i', 'm', 'i', 't', 'i', 'v', 'e', 's', '.', 'd', 'l', 'l', 0} 148 ntdlldll = [...]uint16{'n', 't', 'd', 'l', 'l', '.', 'd', 'l', 'l', 0} 149 powrprofdll = [...]uint16{'p', 'o', 'w', 'r', 'p', 'r', 'o', 'f', '.', 'd', 'l', 'l', 0} 150 winmmdll = [...]uint16{'w', 'i', 'n', 'm', 'm', '.', 'd', 'l', 'l', 0} 151 ) 152 153 // Function to be called by windows CreateThread 154 // to start new os thread. 155 func tstart_stdcall(newm *m) 156 157 // Init-time helper 158 func wintls() 159 160 type mOS struct { 161 threadLock mutex // protects "thread" and prevents closing 162 thread uintptr // thread handle 163 164 waitsema uintptr // semaphore for parking on locks 165 resumesema uintptr // semaphore to indicate suspend/resume 166 167 highResTimer uintptr // high resolution timer handle used in usleep 168 waitIocpTimer uintptr // high resolution timer handle used in netpoll 169 waitIocpHandle uintptr // wait completion handle used in netpoll 170 171 // preemptExtLock synchronizes preemptM with entry/exit from 172 // external C code. 173 // 174 // This protects against races between preemptM calling 175 // SuspendThread and external code on this thread calling 176 // ExitProcess. If these happen concurrently, it's possible to 177 // exit the suspending thread and suspend the exiting thread, 178 // leading to deadlock. 179 // 180 // 0 indicates this M is not being preempted or in external 181 // code. Entering external code CASes this from 0 to 1. If 182 // this fails, a preemption is in progress, so the thread must 183 // wait for the preemption. preemptM also CASes this from 0 to 184 // 1. If this fails, the preemption fails (as it would if the 185 // PC weren't in Go code). The value is reset to 0 when 186 // returning from external code or after a preemption is 187 // complete. 188 // 189 // TODO(austin): We may not need this if preemption were more 190 // tightly synchronized on the G/P status and preemption 191 // blocked transition into _Gsyscall/_Psyscall. 192 preemptExtLock uint32 193 } 194 195 // Stubs so tests can link correctly. These should never be called. 196 func open(name *byte, mode, perm int32) int32 { 197 throw("unimplemented") 198 return -1 199 } 200 func closefd(fd int32) int32 { 201 throw("unimplemented") 202 return -1 203 } 204 func read(fd int32, p unsafe.Pointer, n int32) int32 { 205 throw("unimplemented") 206 return -1 207 } 208 209 type sigset struct{} 210 211 // Call a Windows function with stdcall conventions, 212 // and switch to os stack during the call. 213 func asmstdcall(fn unsafe.Pointer) 214 215 var asmstdcallAddr unsafe.Pointer 216 217 func windowsFindfunc(lib uintptr, name []byte) stdFunction { 218 if name[len(name)-1] != 0 { 219 throw("usage") 220 } 221 f := stdcall2(_GetProcAddress, lib, uintptr(unsafe.Pointer(&name[0]))) 222 return stdFunction(unsafe.Pointer(f)) 223 } 224 225 const _MAX_PATH = 260 // https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation 226 var sysDirectory [_MAX_PATH + 1]byte 227 var sysDirectoryLen uintptr 228 229 func initSysDirectory() { 230 l := stdcall2(_GetSystemDirectoryA, uintptr(unsafe.Pointer(&sysDirectory[0])), uintptr(len(sysDirectory)-1)) 231 if l == 0 || l > uintptr(len(sysDirectory)-1) { 232 throw("Unable to determine system directory") 233 } 234 sysDirectory[l] = '\\' 235 sysDirectoryLen = l + 1 236 } 237 238 //go:linkname windows_GetSystemDirectory internal/syscall/windows.GetSystemDirectory 239 func windows_GetSystemDirectory() string { 240 return unsafe.String(&sysDirectory[0], sysDirectoryLen) 241 } 242 243 func windowsLoadSystemLib(name []uint16) uintptr { 244 return stdcall3(_LoadLibraryExW, uintptr(unsafe.Pointer(&name[0])), 0, _LOAD_LIBRARY_SEARCH_SYSTEM32) 245 } 246 247 func loadOptionalSyscalls() { 248 bcryptPrimitives := windowsLoadSystemLib(bcryptprimitivesdll[:]) 249 if bcryptPrimitives == 0 { 250 throw("bcryptprimitives.dll not found") 251 } 252 _ProcessPrng = windowsFindfunc(bcryptPrimitives, []byte("ProcessPrng\000")) 253 254 n32 := windowsLoadSystemLib(ntdlldll[:]) 255 if n32 == 0 { 256 throw("ntdll.dll not found") 257 } 258 _NtCreateWaitCompletionPacket = windowsFindfunc(n32, []byte("NtCreateWaitCompletionPacket\000")) 259 if _NtCreateWaitCompletionPacket != nil { 260 // These functions should exists if NtCreateWaitCompletionPacket exists. 261 _NtAssociateWaitCompletionPacket = windowsFindfunc(n32, []byte("NtAssociateWaitCompletionPacket\000")) 262 if _NtAssociateWaitCompletionPacket == nil { 263 throw("NtCreateWaitCompletionPacket exists but NtAssociateWaitCompletionPacket does not") 264 } 265 _NtCancelWaitCompletionPacket = windowsFindfunc(n32, []byte("NtCancelWaitCompletionPacket\000")) 266 if _NtCancelWaitCompletionPacket == nil { 267 throw("NtCreateWaitCompletionPacket exists but NtCancelWaitCompletionPacket does not") 268 } 269 } 270 _RtlGetCurrentPeb = windowsFindfunc(n32, []byte("RtlGetCurrentPeb\000")) 271 _RtlGetVersion = windowsFindfunc(n32, []byte("RtlGetVersion\000")) 272 } 273 274 func monitorSuspendResume() { 275 const ( 276 _DEVICE_NOTIFY_CALLBACK = 2 277 ) 278 type _DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS struct { 279 callback uintptr 280 context uintptr 281 } 282 283 powrprof := windowsLoadSystemLib(powrprofdll[:]) 284 if powrprof == 0 { 285 return // Running on Windows 7, where we don't need it anyway. 286 } 287 powerRegisterSuspendResumeNotification := windowsFindfunc(powrprof, []byte("PowerRegisterSuspendResumeNotification\000")) 288 if powerRegisterSuspendResumeNotification == nil { 289 return // Running on Windows 7, where we don't need it anyway. 290 } 291 var fn any = func(context uintptr, changeType uint32, setting uintptr) uintptr { 292 for mp := (*m)(atomic.Loadp(unsafe.Pointer(&allm))); mp != nil; mp = mp.alllink { 293 if mp.resumesema != 0 { 294 stdcall1(_SetEvent, mp.resumesema) 295 } 296 } 297 return 0 298 } 299 params := _DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS{ 300 callback: compileCallback(*efaceOf(&fn), true), 301 } 302 handle := uintptr(0) 303 stdcall3(powerRegisterSuspendResumeNotification, _DEVICE_NOTIFY_CALLBACK, 304 uintptr(unsafe.Pointer(¶ms)), uintptr(unsafe.Pointer(&handle))) 305 } 306 307 func getproccount() int32 { 308 var mask, sysmask uintptr 309 ret := stdcall3(_GetProcessAffinityMask, currentProcess, uintptr(unsafe.Pointer(&mask)), uintptr(unsafe.Pointer(&sysmask))) 310 if ret != 0 { 311 n := 0 312 maskbits := int(unsafe.Sizeof(mask) * 8) 313 for i := 0; i < maskbits; i++ { 314 if mask&(1<<uint(i)) != 0 { 315 n++ 316 } 317 } 318 if n != 0 { 319 return int32(n) 320 } 321 } 322 // use GetSystemInfo if GetProcessAffinityMask fails 323 var info systeminfo 324 stdcall1(_GetSystemInfo, uintptr(unsafe.Pointer(&info))) 325 return int32(info.dwnumberofprocessors) 326 } 327 328 func getPageSize() uintptr { 329 var info systeminfo 330 stdcall1(_GetSystemInfo, uintptr(unsafe.Pointer(&info))) 331 return uintptr(info.dwpagesize) 332 } 333 334 const ( 335 currentProcess = ^uintptr(0) // -1 = current process 336 currentThread = ^uintptr(1) // -2 = current thread 337 ) 338 339 // in sys_windows_386.s and sys_windows_amd64.s: 340 func getlasterror() uint32 341 342 var timeBeginPeriodRetValue uint32 343 344 // osRelaxMinNS indicates that sysmon shouldn't osRelax if the next 345 // timer is less than 60 ms from now. Since osRelaxing may reduce 346 // timer resolution to 15.6 ms, this keeps timer error under roughly 1 347 // part in 4. 348 const osRelaxMinNS = 60 * 1e6 349 350 // osRelax is called by the scheduler when transitioning to and from 351 // all Ps being idle. 352 // 353 // Some versions of Windows have high resolution timer. For those 354 // versions osRelax is noop. 355 // For Windows versions without high resolution timer, osRelax 356 // adjusts the system-wide timer resolution. Go needs a 357 // high resolution timer while running and there's little extra cost 358 // if we're already using the CPU, but if all Ps are idle there's no 359 // need to consume extra power to drive the high-res timer. 360 func osRelax(relax bool) uint32 { 361 if haveHighResTimer { 362 // If the high resolution timer is available, the runtime uses the timer 363 // to sleep for short durations. This means there's no need to adjust 364 // the global clock frequency. 365 return 0 366 } 367 368 if relax { 369 return uint32(stdcall1(_timeEndPeriod, 1)) 370 } else { 371 return uint32(stdcall1(_timeBeginPeriod, 1)) 372 } 373 } 374 375 // haveHighResTimer indicates that the CreateWaitableTimerEx 376 // CREATE_WAITABLE_TIMER_HIGH_RESOLUTION flag is available. 377 var haveHighResTimer = false 378 379 // haveHighResSleep indicates that NtCreateWaitCompletionPacket 380 // exists and haveHighResTimer is true. 381 // NtCreateWaitCompletionPacket has been available since Windows 10, 382 // but has just been publicly documented, so some platforms, like Wine, 383 // doesn't support it yet. 384 var haveHighResSleep = false 385 386 // createHighResTimer calls CreateWaitableTimerEx with 387 // CREATE_WAITABLE_TIMER_HIGH_RESOLUTION flag to create high 388 // resolution timer. createHighResTimer returns new timer 389 // handle or 0, if CreateWaitableTimerEx failed. 390 func createHighResTimer() uintptr { 391 const ( 392 // As per @jstarks, see 393 // https://github.com/golang/go/issues/8687#issuecomment-656259353 394 _CREATE_WAITABLE_TIMER_HIGH_RESOLUTION = 0x00000002 395 396 _SYNCHRONIZE = 0x00100000 397 _TIMER_QUERY_STATE = 0x0001 398 _TIMER_MODIFY_STATE = 0x0002 399 ) 400 return stdcall4(_CreateWaitableTimerExW, 0, 0, 401 _CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, 402 _SYNCHRONIZE|_TIMER_QUERY_STATE|_TIMER_MODIFY_STATE) 403 } 404 405 func initHighResTimer() { 406 h := createHighResTimer() 407 if h != 0 { 408 haveHighResTimer = true 409 haveHighResSleep = _NtCreateWaitCompletionPacket != nil 410 stdcall1(_CloseHandle, h) 411 } else { 412 // Only load winmm.dll if we need it. 413 // This avoids a dependency on winmm.dll for Go programs 414 // that run on new Windows versions. 415 m32 := windowsLoadSystemLib(winmmdll[:]) 416 if m32 == 0 { 417 print("runtime: LoadLibraryExW failed; errno=", getlasterror(), "\n") 418 throw("winmm.dll not found") 419 } 420 _timeBeginPeriod = windowsFindfunc(m32, []byte("timeBeginPeriod\000")) 421 _timeEndPeriod = windowsFindfunc(m32, []byte("timeEndPeriod\000")) 422 if _timeBeginPeriod == nil || _timeEndPeriod == nil { 423 print("runtime: GetProcAddress failed; errno=", getlasterror(), "\n") 424 throw("timeBegin/EndPeriod not found") 425 } 426 } 427 } 428 429 //go:linkname canUseLongPaths internal/syscall/windows.CanUseLongPaths 430 var canUseLongPaths bool 431 432 // initLongPathSupport enables long path support. 433 func initLongPathSupport() { 434 const ( 435 IsLongPathAwareProcess = 0x80 436 PebBitFieldOffset = 3 437 ) 438 439 // Check that we're ≥ 10.0.15063. 440 info := _OSVERSIONINFOW{} 441 info.osVersionInfoSize = uint32(unsafe.Sizeof(info)) 442 stdcall1(_RtlGetVersion, uintptr(unsafe.Pointer(&info))) 443 if info.majorVersion < 10 || (info.majorVersion == 10 && info.minorVersion == 0 && info.buildNumber < 15063) { 444 return 445 } 446 447 // Set the IsLongPathAwareProcess flag of the PEB's bit field. 448 // This flag is not documented, but it's known to be used 449 // by Windows to enable long path support. 450 bitField := (*byte)(unsafe.Pointer(stdcall0(_RtlGetCurrentPeb) + PebBitFieldOffset)) 451 *bitField |= IsLongPathAwareProcess 452 453 canUseLongPaths = true 454 } 455 456 func osinit() { 457 asmstdcallAddr = unsafe.Pointer(abi.FuncPCABI0(asmstdcall)) 458 459 loadOptionalSyscalls() 460 461 preventErrorDialogs() 462 463 initExceptionHandler() 464 465 initHighResTimer() 466 timeBeginPeriodRetValue = osRelax(false) 467 468 initSysDirectory() 469 initLongPathSupport() 470 471 ncpu = getproccount() 472 473 physPageSize = getPageSize() 474 475 // Windows dynamic priority boosting assumes that a process has different types 476 // of dedicated threads -- GUI, IO, computational, etc. Go processes use 477 // equivalent threads that all do a mix of GUI, IO, computations, etc. 478 // In such context dynamic priority boosting does nothing but harm, so we turn it off. 479 stdcall2(_SetProcessPriorityBoost, currentProcess, 1) 480 } 481 482 //go:nosplit 483 func readRandom(r []byte) int { 484 n := 0 485 if stdcall2(_ProcessPrng, uintptr(unsafe.Pointer(&r[0])), uintptr(len(r)))&0xff != 0 { 486 n = len(r) 487 } 488 return n 489 } 490 491 func goenvs() { 492 // strings is a pointer to environment variable pairs in the form: 493 // "envA=valA\x00envB=valB\x00\x00" (in UTF-16) 494 // Two consecutive zero bytes end the list. 495 strings := unsafe.Pointer(stdcall0(_GetEnvironmentStringsW)) 496 p := (*[1 << 24]uint16)(strings)[:] 497 498 n := 0 499 for from, i := 0, 0; true; i++ { 500 if p[i] == 0 { 501 // empty string marks the end 502 if i == from { 503 break 504 } 505 from = i + 1 506 n++ 507 } 508 } 509 envs = make([]string, n) 510 511 for i := range envs { 512 envs[i] = gostringw(&p[0]) 513 for p[0] != 0 { 514 p = p[1:] 515 } 516 p = p[1:] // skip nil byte 517 } 518 519 stdcall1(_FreeEnvironmentStringsW, uintptr(strings)) 520 521 // We call these all the way here, late in init, so that malloc works 522 // for the callback functions these generate. 523 var fn any = ctrlHandler 524 ctrlHandlerPC := compileCallback(*efaceOf(&fn), true) 525 stdcall2(_SetConsoleCtrlHandler, ctrlHandlerPC, 1) 526 527 monitorSuspendResume() 528 } 529 530 // exiting is set to non-zero when the process is exiting. 531 var exiting uint32 532 533 //go:nosplit 534 func exit(code int32) { 535 // Disallow thread suspension for preemption. Otherwise, 536 // ExitProcess and SuspendThread can race: SuspendThread 537 // queues a suspension request for this thread, ExitProcess 538 // kills the suspending thread, and then this thread suspends. 539 lock(&suspendLock) 540 atomic.Store(&exiting, 1) 541 stdcall1(_ExitProcess, uintptr(code)) 542 } 543 544 // write1 must be nosplit because it's used as a last resort in 545 // functions like badmorestackg0. In such cases, we'll always take the 546 // ASCII path. 547 // 548 //go:nosplit 549 func write1(fd uintptr, buf unsafe.Pointer, n int32) int32 { 550 const ( 551 _STD_OUTPUT_HANDLE = ^uintptr(10) // -11 552 _STD_ERROR_HANDLE = ^uintptr(11) // -12 553 ) 554 var handle uintptr 555 switch fd { 556 case 1: 557 handle = stdcall1(_GetStdHandle, _STD_OUTPUT_HANDLE) 558 case 2: 559 handle = stdcall1(_GetStdHandle, _STD_ERROR_HANDLE) 560 default: 561 // assume fd is real windows handle. 562 handle = fd 563 } 564 isASCII := true 565 b := (*[1 << 30]byte)(buf)[:n] 566 for _, x := range b { 567 if x >= 0x80 { 568 isASCII = false 569 break 570 } 571 } 572 573 if !isASCII { 574 var m uint32 575 isConsole := stdcall2(_GetConsoleMode, handle, uintptr(unsafe.Pointer(&m))) != 0 576 // If this is a console output, various non-unicode code pages can be in use. 577 // Use the dedicated WriteConsole call to ensure unicode is printed correctly. 578 if isConsole { 579 return int32(writeConsole(handle, buf, n)) 580 } 581 } 582 var written uint32 583 stdcall5(_WriteFile, handle, uintptr(buf), uintptr(n), uintptr(unsafe.Pointer(&written)), 0) 584 return int32(written) 585 } 586 587 var ( 588 utf16ConsoleBack [1000]uint16 589 utf16ConsoleBackLock mutex 590 ) 591 592 // writeConsole writes bufLen bytes from buf to the console File. 593 // It returns the number of bytes written. 594 func writeConsole(handle uintptr, buf unsafe.Pointer, bufLen int32) int { 595 const surr2 = (surrogateMin + surrogateMax + 1) / 2 596 597 // Do not use defer for unlock. May cause issues when printing a panic. 598 lock(&utf16ConsoleBackLock) 599 600 b := (*[1 << 30]byte)(buf)[:bufLen] 601 s := *(*string)(unsafe.Pointer(&b)) 602 603 utf16tmp := utf16ConsoleBack[:] 604 605 total := len(s) 606 w := 0 607 for _, r := range s { 608 if w >= len(utf16tmp)-2 { 609 writeConsoleUTF16(handle, utf16tmp[:w]) 610 w = 0 611 } 612 if r < 0x10000 { 613 utf16tmp[w] = uint16(r) 614 w++ 615 } else { 616 r -= 0x10000 617 utf16tmp[w] = surrogateMin + uint16(r>>10)&0x3ff 618 utf16tmp[w+1] = surr2 + uint16(r)&0x3ff 619 w += 2 620 } 621 } 622 writeConsoleUTF16(handle, utf16tmp[:w]) 623 unlock(&utf16ConsoleBackLock) 624 return total 625 } 626 627 // writeConsoleUTF16 is the dedicated windows calls that correctly prints 628 // to the console regardless of the current code page. Input is utf-16 code points. 629 // The handle must be a console handle. 630 func writeConsoleUTF16(handle uintptr, b []uint16) { 631 l := uint32(len(b)) 632 if l == 0 { 633 return 634 } 635 var written uint32 636 stdcall5(_WriteConsoleW, 637 handle, 638 uintptr(unsafe.Pointer(&b[0])), 639 uintptr(l), 640 uintptr(unsafe.Pointer(&written)), 641 0, 642 ) 643 return 644 } 645 646 //go:nosplit 647 func semasleep(ns int64) int32 { 648 const ( 649 _WAIT_ABANDONED = 0x00000080 650 _WAIT_OBJECT_0 = 0x00000000 651 _WAIT_TIMEOUT = 0x00000102 652 _WAIT_FAILED = 0xFFFFFFFF 653 ) 654 655 var result uintptr 656 if ns < 0 { 657 result = stdcall2(_WaitForSingleObject, getg().m.waitsema, uintptr(_INFINITE)) 658 } else { 659 start := nanotime() 660 elapsed := int64(0) 661 for { 662 ms := int64(timediv(ns-elapsed, 1000000, nil)) 663 if ms == 0 { 664 ms = 1 665 } 666 result = stdcall4(_WaitForMultipleObjects, 2, 667 uintptr(unsafe.Pointer(&[2]uintptr{getg().m.waitsema, getg().m.resumesema})), 668 0, uintptr(ms)) 669 if result != _WAIT_OBJECT_0+1 { 670 // Not a suspend/resume event 671 break 672 } 673 elapsed = nanotime() - start 674 if elapsed >= ns { 675 return -1 676 } 677 } 678 } 679 switch result { 680 case _WAIT_OBJECT_0: // Signaled 681 return 0 682 683 case _WAIT_TIMEOUT: 684 return -1 685 686 case _WAIT_ABANDONED: 687 systemstack(func() { 688 throw("runtime.semasleep wait_abandoned") 689 }) 690 691 case _WAIT_FAILED: 692 systemstack(func() { 693 print("runtime: waitforsingleobject wait_failed; errno=", getlasterror(), "\n") 694 throw("runtime.semasleep wait_failed") 695 }) 696 697 default: 698 systemstack(func() { 699 print("runtime: waitforsingleobject unexpected; result=", result, "\n") 700 throw("runtime.semasleep unexpected") 701 }) 702 } 703 704 return -1 // unreachable 705 } 706 707 //go:nosplit 708 func semawakeup(mp *m) { 709 if stdcall1(_SetEvent, mp.waitsema) == 0 { 710 systemstack(func() { 711 print("runtime: setevent failed; errno=", getlasterror(), "\n") 712 throw("runtime.semawakeup") 713 }) 714 } 715 } 716 717 //go:nosplit 718 func semacreate(mp *m) { 719 if mp.waitsema != 0 { 720 return 721 } 722 mp.waitsema = stdcall4(_CreateEventA, 0, 0, 0, 0) 723 if mp.waitsema == 0 { 724 systemstack(func() { 725 print("runtime: createevent failed; errno=", getlasterror(), "\n") 726 throw("runtime.semacreate") 727 }) 728 } 729 mp.resumesema = stdcall4(_CreateEventA, 0, 0, 0, 0) 730 if mp.resumesema == 0 { 731 systemstack(func() { 732 print("runtime: createevent failed; errno=", getlasterror(), "\n") 733 throw("runtime.semacreate") 734 }) 735 stdcall1(_CloseHandle, mp.waitsema) 736 mp.waitsema = 0 737 } 738 } 739 740 // May run with m.p==nil, so write barriers are not allowed. This 741 // function is called by newosproc0, so it is also required to 742 // operate without stack guards. 743 // 744 //go:nowritebarrierrec 745 //go:nosplit 746 func newosproc(mp *m) { 747 // We pass 0 for the stack size to use the default for this binary. 748 thandle := stdcall6(_CreateThread, 0, 0, 749 abi.FuncPCABI0(tstart_stdcall), uintptr(unsafe.Pointer(mp)), 750 0, 0) 751 752 if thandle == 0 { 753 if atomic.Load(&exiting) != 0 { 754 // CreateThread may fail if called 755 // concurrently with ExitProcess. If this 756 // happens, just freeze this thread and let 757 // the process exit. See issue #18253. 758 lock(&deadlock) 759 lock(&deadlock) 760 } 761 print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", getlasterror(), ")\n") 762 throw("runtime.newosproc") 763 } 764 765 // Close thandle to avoid leaking the thread object if it exits. 766 stdcall1(_CloseHandle, thandle) 767 } 768 769 // Used by the C library build mode. On Linux this function would allocate a 770 // stack, but that's not necessary for Windows. No stack guards are present 771 // and the GC has not been initialized, so write barriers will fail. 772 // 773 //go:nowritebarrierrec 774 //go:nosplit 775 func newosproc0(mp *m, stk unsafe.Pointer) { 776 // TODO: this is completely broken. The args passed to newosproc0 (in asm_amd64.s) 777 // are stacksize and function, not *m and stack. 778 // Check os_linux.go for an implementation that might actually work. 779 throw("bad newosproc0") 780 } 781 782 func exitThread(wait *atomic.Uint32) { 783 // We should never reach exitThread on Windows because we let 784 // the OS clean up threads. 785 throw("exitThread") 786 } 787 788 // Called to initialize a new m (including the bootstrap m). 789 // Called on the parent thread (main thread in case of bootstrap), can allocate memory. 790 func mpreinit(mp *m) { 791 } 792 793 //go:nosplit 794 func sigsave(p *sigset) { 795 } 796 797 //go:nosplit 798 func msigrestore(sigmask sigset) { 799 } 800 801 //go:nosplit 802 //go:nowritebarrierrec 803 func clearSignalHandlers() { 804 } 805 806 //go:nosplit 807 func sigblock(exiting bool) { 808 } 809 810 // Called to initialize a new m (including the bootstrap m). 811 // Called on the new thread, cannot allocate Go memory. 812 func minit() { 813 var thandle uintptr 814 if stdcall7(_DuplicateHandle, currentProcess, currentThread, currentProcess, uintptr(unsafe.Pointer(&thandle)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 { 815 print("runtime.minit: duplicatehandle failed; errno=", getlasterror(), "\n") 816 throw("runtime.minit: duplicatehandle failed") 817 } 818 819 mp := getg().m 820 lock(&mp.threadLock) 821 mp.thread = thandle 822 mp.procid = uint64(stdcall0(_GetCurrentThreadId)) 823 824 // Configure usleep timer, if possible. 825 if mp.highResTimer == 0 && haveHighResTimer { 826 mp.highResTimer = createHighResTimer() 827 if mp.highResTimer == 0 { 828 print("runtime: CreateWaitableTimerEx failed; errno=", getlasterror(), "\n") 829 throw("CreateWaitableTimerEx when creating timer failed") 830 } 831 } 832 if mp.waitIocpHandle == 0 && haveHighResSleep { 833 mp.waitIocpTimer = createHighResTimer() 834 if mp.waitIocpTimer == 0 { 835 print("runtime: CreateWaitableTimerEx failed; errno=", getlasterror(), "\n") 836 throw("CreateWaitableTimerEx when creating timer failed") 837 } 838 const GENERIC_ALL = 0x10000000 839 errno := stdcall3(_NtCreateWaitCompletionPacket, uintptr(unsafe.Pointer(&mp.waitIocpHandle)), GENERIC_ALL, 0) 840 if mp.waitIocpHandle == 0 { 841 print("runtime: NtCreateWaitCompletionPacket failed; errno=", errno, "\n") 842 throw("NtCreateWaitCompletionPacket failed") 843 } 844 } 845 unlock(&mp.threadLock) 846 847 // Query the true stack base from the OS. Currently we're 848 // running on a small assumed stack. 849 var mbi memoryBasicInformation 850 res := stdcall3(_VirtualQuery, uintptr(unsafe.Pointer(&mbi)), uintptr(unsafe.Pointer(&mbi)), unsafe.Sizeof(mbi)) 851 if res == 0 { 852 print("runtime: VirtualQuery failed; errno=", getlasterror(), "\n") 853 throw("VirtualQuery for stack base failed") 854 } 855 // The system leaves an 8K PAGE_GUARD region at the bottom of 856 // the stack (in theory VirtualQuery isn't supposed to include 857 // that, but it does). Add an additional 8K of slop for 858 // calling C functions that don't have stack checks and for 859 // lastcontinuehandler. We shouldn't be anywhere near this 860 // bound anyway. 861 base := mbi.allocationBase + 16<<10 862 // Sanity check the stack bounds. 863 g0 := getg() 864 if base > g0.stack.hi || g0.stack.hi-base > 64<<20 { 865 print("runtime: g0 stack [", hex(base), ",", hex(g0.stack.hi), ")\n") 866 throw("bad g0 stack") 867 } 868 g0.stack.lo = base 869 g0.stackguard0 = g0.stack.lo + stackGuard 870 g0.stackguard1 = g0.stackguard0 871 // Sanity check the SP. 872 stackcheck() 873 } 874 875 // Called from dropm to undo the effect of an minit. 876 // 877 //go:nosplit 878 func unminit() { 879 mp := getg().m 880 lock(&mp.threadLock) 881 if mp.thread != 0 { 882 stdcall1(_CloseHandle, mp.thread) 883 mp.thread = 0 884 } 885 unlock(&mp.threadLock) 886 887 mp.procid = 0 888 } 889 890 // Called from exitm, but not from drop, to undo the effect of thread-owned 891 // resources in minit, semacreate, or elsewhere. Do not take locks after calling this. 892 // 893 //go:nosplit 894 func mdestroy(mp *m) { 895 if mp.highResTimer != 0 { 896 stdcall1(_CloseHandle, mp.highResTimer) 897 mp.highResTimer = 0 898 } 899 if mp.waitIocpTimer != 0 { 900 stdcall1(_CloseHandle, mp.waitIocpTimer) 901 mp.waitIocpTimer = 0 902 } 903 if mp.waitIocpHandle != 0 { 904 stdcall1(_CloseHandle, mp.waitIocpHandle) 905 mp.waitIocpHandle = 0 906 } 907 if mp.waitsema != 0 { 908 stdcall1(_CloseHandle, mp.waitsema) 909 mp.waitsema = 0 910 } 911 if mp.resumesema != 0 { 912 stdcall1(_CloseHandle, mp.resumesema) 913 mp.resumesema = 0 914 } 915 } 916 917 // asmstdcall_trampoline calls asmstdcall converting from Go to C calling convention. 918 func asmstdcall_trampoline(args unsafe.Pointer) 919 920 // stdcall_no_g calls asmstdcall on os stack without using g. 921 // 922 //go:nosplit 923 func stdcall_no_g(fn stdFunction, n int, args uintptr) uintptr { 924 libcall := libcall{ 925 fn: uintptr(unsafe.Pointer(fn)), 926 n: uintptr(n), 927 args: args, 928 } 929 asmstdcall_trampoline(noescape(unsafe.Pointer(&libcall))) 930 return libcall.r1 931 } 932 933 // Calling stdcall on os stack. 934 // May run during STW, so write barriers are not allowed. 935 // 936 //go:nowritebarrier 937 //go:nosplit 938 func stdcall(fn stdFunction) uintptr { 939 gp := getg() 940 mp := gp.m 941 mp.libcall.fn = uintptr(unsafe.Pointer(fn)) 942 resetLibcall := false 943 if mp.profilehz != 0 && mp.libcallsp == 0 { 944 // leave pc/sp for cpu profiler 945 mp.libcallg.set(gp) 946 mp.libcallpc = getcallerpc() 947 // sp must be the last, because once async cpu profiler finds 948 // all three values to be non-zero, it will use them 949 mp.libcallsp = getcallersp() 950 resetLibcall = true // See comment in sys_darwin.go:libcCall 951 } 952 asmcgocall(asmstdcallAddr, unsafe.Pointer(&mp.libcall)) 953 if resetLibcall { 954 mp.libcallsp = 0 955 } 956 return mp.libcall.r1 957 } 958 959 //go:nosplit 960 func stdcall0(fn stdFunction) uintptr { 961 mp := getg().m 962 mp.libcall.n = 0 963 mp.libcall.args = 0 964 return stdcall(fn) 965 } 966 967 //go:nosplit 968 //go:cgo_unsafe_args 969 func stdcall1(fn stdFunction, a0 uintptr) uintptr { 970 mp := getg().m 971 mp.libcall.n = 1 972 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) 973 return stdcall(fn) 974 } 975 976 //go:nosplit 977 //go:cgo_unsafe_args 978 func stdcall2(fn stdFunction, a0, a1 uintptr) uintptr { 979 mp := getg().m 980 mp.libcall.n = 2 981 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) 982 return stdcall(fn) 983 } 984 985 //go:nosplit 986 //go:cgo_unsafe_args 987 func stdcall3(fn stdFunction, a0, a1, a2 uintptr) uintptr { 988 mp := getg().m 989 mp.libcall.n = 3 990 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) 991 return stdcall(fn) 992 } 993 994 //go:nosplit 995 //go:cgo_unsafe_args 996 func stdcall4(fn stdFunction, a0, a1, a2, a3 uintptr) uintptr { 997 mp := getg().m 998 mp.libcall.n = 4 999 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) 1000 return stdcall(fn) 1001 } 1002 1003 //go:nosplit 1004 //go:cgo_unsafe_args 1005 func stdcall5(fn stdFunction, a0, a1, a2, a3, a4 uintptr) uintptr { 1006 mp := getg().m 1007 mp.libcall.n = 5 1008 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) 1009 return stdcall(fn) 1010 } 1011 1012 //go:nosplit 1013 //go:cgo_unsafe_args 1014 func stdcall6(fn stdFunction, a0, a1, a2, a3, a4, a5 uintptr) uintptr { 1015 mp := getg().m 1016 mp.libcall.n = 6 1017 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) 1018 return stdcall(fn) 1019 } 1020 1021 //go:nosplit 1022 //go:cgo_unsafe_args 1023 func stdcall7(fn stdFunction, a0, a1, a2, a3, a4, a5, a6 uintptr) uintptr { 1024 mp := getg().m 1025 mp.libcall.n = 7 1026 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) 1027 return stdcall(fn) 1028 } 1029 1030 //go:nosplit 1031 //go:cgo_unsafe_args 1032 func stdcall8(fn stdFunction, a0, a1, a2, a3, a4, a5, a6, a7 uintptr) uintptr { 1033 mp := getg().m 1034 mp.libcall.n = 8 1035 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) 1036 return stdcall(fn) 1037 } 1038 1039 // These must run on the system stack only. 1040 1041 //go:nosplit 1042 func osyield_no_g() { 1043 stdcall_no_g(_SwitchToThread, 0, 0) 1044 } 1045 1046 //go:nosplit 1047 func osyield() { 1048 systemstack(func() { 1049 stdcall0(_SwitchToThread) 1050 }) 1051 } 1052 1053 //go:nosplit 1054 func usleep_no_g(us uint32) { 1055 timeout := uintptr(us) / 1000 // ms units 1056 args := [...]uintptr{_INVALID_HANDLE_VALUE, timeout} 1057 stdcall_no_g(_WaitForSingleObject, len(args), uintptr(noescape(unsafe.Pointer(&args[0])))) 1058 } 1059 1060 //go:nosplit 1061 func usleep(us uint32) { 1062 systemstack(func() { 1063 var h, timeout uintptr 1064 // If the high-res timer is available and its handle has been allocated for this m, use it. 1065 // Otherwise fall back to the low-res one, which doesn't need a handle. 1066 if haveHighResTimer && getg().m.highResTimer != 0 { 1067 h = getg().m.highResTimer 1068 dt := -10 * int64(us) // relative sleep (negative), 100ns units 1069 stdcall6(_SetWaitableTimer, h, uintptr(unsafe.Pointer(&dt)), 0, 0, 0, 0) 1070 timeout = _INFINITE 1071 } else { 1072 h = _INVALID_HANDLE_VALUE 1073 timeout = uintptr(us) / 1000 // ms units 1074 } 1075 stdcall2(_WaitForSingleObject, h, timeout) 1076 }) 1077 } 1078 1079 func ctrlHandler(_type uint32) uintptr { 1080 var s uint32 1081 1082 switch _type { 1083 case _CTRL_C_EVENT, _CTRL_BREAK_EVENT: 1084 s = _SIGINT 1085 case _CTRL_CLOSE_EVENT, _CTRL_LOGOFF_EVENT, _CTRL_SHUTDOWN_EVENT: 1086 s = _SIGTERM 1087 default: 1088 return 0 1089 } 1090 1091 if sigsend(s) { 1092 if s == _SIGTERM { 1093 // Windows terminates the process after this handler returns. 1094 // Block indefinitely to give signal handlers a chance to clean up, 1095 // but make sure to be properly parked first, so the rest of the 1096 // program can continue executing. 1097 block() 1098 } 1099 return 1 1100 } 1101 return 0 1102 } 1103 1104 // called from zcallback_windows_*.s to sys_windows_*.s 1105 func callbackasm1() 1106 1107 var profiletimer uintptr 1108 1109 func profilem(mp *m, thread uintptr) { 1110 // Align Context to 16 bytes. 1111 var c *context 1112 var cbuf [unsafe.Sizeof(*c) + 15]byte 1113 c = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15)) 1114 1115 c.contextflags = _CONTEXT_CONTROL 1116 stdcall2(_GetThreadContext, thread, uintptr(unsafe.Pointer(c))) 1117 1118 gp := gFromSP(mp, c.sp()) 1119 1120 sigprof(c.ip(), c.sp(), c.lr(), gp, mp) 1121 } 1122 1123 func gFromSP(mp *m, sp uintptr) *g { 1124 if gp := mp.g0; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi { 1125 return gp 1126 } 1127 if gp := mp.gsignal; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi { 1128 return gp 1129 } 1130 if gp := mp.curg; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi { 1131 return gp 1132 } 1133 return nil 1134 } 1135 1136 func profileLoop() { 1137 stdcall2(_SetThreadPriority, currentThread, _THREAD_PRIORITY_HIGHEST) 1138 1139 for { 1140 stdcall2(_WaitForSingleObject, profiletimer, _INFINITE) 1141 first := (*m)(atomic.Loadp(unsafe.Pointer(&allm))) 1142 for mp := first; mp != nil; mp = mp.alllink { 1143 if mp == getg().m { 1144 // Don't profile ourselves. 1145 continue 1146 } 1147 1148 lock(&mp.threadLock) 1149 // Do not profile threads blocked on Notes, 1150 // this includes idle worker threads, 1151 // idle timer thread, idle heap scavenger, etc. 1152 if mp.thread == 0 || mp.profilehz == 0 || mp.blocked { 1153 unlock(&mp.threadLock) 1154 continue 1155 } 1156 // Acquire our own handle to the thread. 1157 var thread uintptr 1158 if stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 { 1159 print("runtime: duplicatehandle failed; errno=", getlasterror(), "\n") 1160 throw("duplicatehandle failed") 1161 } 1162 unlock(&mp.threadLock) 1163 1164 // mp may exit between the DuplicateHandle 1165 // above and the SuspendThread. The handle 1166 // will remain valid, but SuspendThread may 1167 // fail. 1168 if int32(stdcall1(_SuspendThread, thread)) == -1 { 1169 // The thread no longer exists. 1170 stdcall1(_CloseHandle, thread) 1171 continue 1172 } 1173 if mp.profilehz != 0 && !mp.blocked { 1174 // Pass the thread handle in case mp 1175 // was in the process of shutting down. 1176 profilem(mp, thread) 1177 } 1178 stdcall1(_ResumeThread, thread) 1179 stdcall1(_CloseHandle, thread) 1180 } 1181 } 1182 } 1183 1184 func setProcessCPUProfiler(hz int32) { 1185 if profiletimer == 0 { 1186 var timer uintptr 1187 if haveHighResTimer { 1188 timer = createHighResTimer() 1189 } else { 1190 timer = stdcall3(_CreateWaitableTimerA, 0, 0, 0) 1191 } 1192 atomic.Storeuintptr(&profiletimer, timer) 1193 newm(profileLoop, nil, -1) 1194 } 1195 } 1196 1197 func setThreadCPUProfiler(hz int32) { 1198 ms := int32(0) 1199 due := ^int64(^uint64(1 << 63)) 1200 if hz > 0 { 1201 ms = 1000 / hz 1202 if ms == 0 { 1203 ms = 1 1204 } 1205 due = int64(ms) * -10000 1206 } 1207 stdcall6(_SetWaitableTimer, profiletimer, uintptr(unsafe.Pointer(&due)), uintptr(ms), 0, 0, 0) 1208 atomic.Store((*uint32)(unsafe.Pointer(&getg().m.profilehz)), uint32(hz)) 1209 } 1210 1211 const preemptMSupported = true 1212 1213 // suspendLock protects simultaneous SuspendThread operations from 1214 // suspending each other. 1215 var suspendLock mutex 1216 1217 func preemptM(mp *m) { 1218 if mp == getg().m { 1219 throw("self-preempt") 1220 } 1221 1222 // Synchronize with external code that may try to ExitProcess. 1223 if !atomic.Cas(&mp.preemptExtLock, 0, 1) { 1224 // External code is running. Fail the preemption 1225 // attempt. 1226 mp.preemptGen.Add(1) 1227 return 1228 } 1229 1230 // Acquire our own handle to mp's thread. 1231 lock(&mp.threadLock) 1232 if mp.thread == 0 { 1233 // The M hasn't been minit'd yet (or was just unminit'd). 1234 unlock(&mp.threadLock) 1235 atomic.Store(&mp.preemptExtLock, 0) 1236 mp.preemptGen.Add(1) 1237 return 1238 } 1239 var thread uintptr 1240 if stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 { 1241 print("runtime.preemptM: duplicatehandle failed; errno=", getlasterror(), "\n") 1242 throw("runtime.preemptM: duplicatehandle failed") 1243 } 1244 unlock(&mp.threadLock) 1245 1246 // Prepare thread context buffer. This must be aligned to 16 bytes. 1247 var c *context 1248 var cbuf [unsafe.Sizeof(*c) + 15]byte 1249 c = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15)) 1250 c.contextflags = _CONTEXT_CONTROL 1251 1252 // Serialize thread suspension. SuspendThread is asynchronous, 1253 // so it's otherwise possible for two threads to suspend each 1254 // other and deadlock. We must hold this lock until after 1255 // GetThreadContext, since that blocks until the thread is 1256 // actually suspended. 1257 lock(&suspendLock) 1258 1259 // Suspend the thread. 1260 if int32(stdcall1(_SuspendThread, thread)) == -1 { 1261 unlock(&suspendLock) 1262 stdcall1(_CloseHandle, thread) 1263 atomic.Store(&mp.preemptExtLock, 0) 1264 // The thread no longer exists. This shouldn't be 1265 // possible, but just acknowledge the request. 1266 mp.preemptGen.Add(1) 1267 return 1268 } 1269 1270 // We have to be very careful between this point and once 1271 // we've shown mp is at an async safe-point. This is like a 1272 // signal handler in the sense that mp could have been doing 1273 // anything when we stopped it, including holding arbitrary 1274 // locks. 1275 1276 // We have to get the thread context before inspecting the M 1277 // because SuspendThread only requests a suspend. 1278 // GetThreadContext actually blocks until it's suspended. 1279 stdcall2(_GetThreadContext, thread, uintptr(unsafe.Pointer(c))) 1280 1281 unlock(&suspendLock) 1282 1283 // Does it want a preemption and is it safe to preempt? 1284 gp := gFromSP(mp, c.sp()) 1285 if gp != nil && wantAsyncPreempt(gp) { 1286 if ok, newpc := isAsyncSafePoint(gp, c.ip(), c.sp(), c.lr()); ok { 1287 // Inject call to asyncPreempt 1288 targetPC := abi.FuncPCABI0(asyncPreempt) 1289 switch GOARCH { 1290 default: 1291 throw("unsupported architecture") 1292 case "386", "amd64": 1293 // Make it look like the thread called targetPC. 1294 sp := c.sp() 1295 sp -= goarch.PtrSize 1296 *(*uintptr)(unsafe.Pointer(sp)) = newpc 1297 c.set_sp(sp) 1298 c.set_ip(targetPC) 1299 1300 case "arm": 1301 // Push LR. The injected call is responsible 1302 // for restoring LR. gentraceback is aware of 1303 // this extra slot. See sigctxt.pushCall in 1304 // signal_arm.go, which is similar except we 1305 // subtract 1 from IP here. 1306 sp := c.sp() 1307 sp -= goarch.PtrSize 1308 c.set_sp(sp) 1309 *(*uint32)(unsafe.Pointer(sp)) = uint32(c.lr()) 1310 c.set_lr(newpc - 1) 1311 c.set_ip(targetPC) 1312 1313 case "arm64": 1314 // Push LR. The injected call is responsible 1315 // for restoring LR. gentraceback is aware of 1316 // this extra slot. See sigctxt.pushCall in 1317 // signal_arm64.go. 1318 sp := c.sp() - 16 // SP needs 16-byte alignment 1319 c.set_sp(sp) 1320 *(*uint64)(unsafe.Pointer(sp)) = uint64(c.lr()) 1321 c.set_lr(newpc) 1322 c.set_ip(targetPC) 1323 } 1324 stdcall2(_SetThreadContext, thread, uintptr(unsafe.Pointer(c))) 1325 } 1326 } 1327 1328 atomic.Store(&mp.preemptExtLock, 0) 1329 1330 // Acknowledge the preemption. 1331 mp.preemptGen.Add(1) 1332 1333 stdcall1(_ResumeThread, thread) 1334 stdcall1(_CloseHandle, thread) 1335 } 1336 1337 // osPreemptExtEnter is called before entering external code that may 1338 // call ExitProcess. 1339 // 1340 // This must be nosplit because it may be called from a syscall with 1341 // untyped stack slots, so the stack must not be grown or scanned. 1342 // 1343 //go:nosplit 1344 func osPreemptExtEnter(mp *m) { 1345 for !atomic.Cas(&mp.preemptExtLock, 0, 1) { 1346 // An asynchronous preemption is in progress. It's not 1347 // safe to enter external code because it may call 1348 // ExitProcess and deadlock with SuspendThread. 1349 // Ideally we would do the preemption ourselves, but 1350 // can't since there may be untyped syscall arguments 1351 // on the stack. Instead, just wait and encourage the 1352 // SuspendThread APC to run. The preemption should be 1353 // done shortly. 1354 osyield() 1355 } 1356 // Asynchronous preemption is now blocked. 1357 } 1358 1359 // osPreemptExtExit is called after returning from external code that 1360 // may call ExitProcess. 1361 // 1362 // See osPreemptExtEnter for why this is nosplit. 1363 // 1364 //go:nosplit 1365 func osPreemptExtExit(mp *m) { 1366 atomic.Store(&mp.preemptExtLock, 0) 1367 }