// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package runtime import ( "internal/abi" "internal/goarch" "internal/runtime/atomic" "internal/runtime/sys" "unsafe" ) // TODO(brainman): should not need those const ( _NSIG = 65 ) //go:cgo_import_dynamic runtime._AddVectoredContinueHandler AddVectoredContinueHandler%2 "kernel32.dll" //go:cgo_import_dynamic runtime._AddVectoredExceptionHandler AddVectoredExceptionHandler%2 "kernel32.dll" //go:cgo_import_dynamic runtime._CloseHandle CloseHandle%1 "kernel32.dll" //go:cgo_import_dynamic runtime._CreateEventA CreateEventA%4 "kernel32.dll" //go:cgo_import_dynamic runtime._CreateIoCompletionPort CreateIoCompletionPort%4 "kernel32.dll" //go:cgo_import_dynamic runtime._CreateThread CreateThread%6 "kernel32.dll" //go:cgo_import_dynamic runtime._CreateWaitableTimerA CreateWaitableTimerA%3 "kernel32.dll" //go:cgo_import_dynamic runtime._CreateWaitableTimerExW CreateWaitableTimerExW%4 "kernel32.dll" //go:cgo_import_dynamic runtime._DuplicateHandle DuplicateHandle%7 "kernel32.dll" //go:cgo_import_dynamic runtime._ExitProcess ExitProcess%1 "kernel32.dll" //go:cgo_import_dynamic runtime._FreeEnvironmentStringsW FreeEnvironmentStringsW%1 "kernel32.dll" //go:cgo_import_dynamic runtime._GetConsoleMode GetConsoleMode%2 "kernel32.dll" //go:cgo_import_dynamic runtime._GetCurrentThreadId GetCurrentThreadId%0 "kernel32.dll" //go:cgo_import_dynamic runtime._GetEnvironmentStringsW GetEnvironmentStringsW%0 "kernel32.dll" //go:cgo_import_dynamic runtime._GetErrorMode GetErrorMode%0 "kernel32.dll" //go:cgo_import_dynamic runtime._GetProcAddress GetProcAddress%2 "kernel32.dll" //go:cgo_import_dynamic runtime._GetProcessAffinityMask GetProcessAffinityMask%3 "kernel32.dll" //go:cgo_import_dynamic runtime._GetQueuedCompletionStatusEx GetQueuedCompletionStatusEx%6 "kernel32.dll" //go:cgo_import_dynamic runtime._GetStdHandle GetStdHandle%1 "kernel32.dll" //go:cgo_import_dynamic runtime._GetSystemDirectoryA GetSystemDirectoryA%2 "kernel32.dll" //go:cgo_import_dynamic runtime._GetSystemInfo GetSystemInfo%1 "kernel32.dll" //go:cgo_import_dynamic runtime._GetThreadContext GetThreadContext%2 "kernel32.dll" //go:cgo_import_dynamic runtime._SetThreadContext SetThreadContext%2 "kernel32.dll" //go:cgo_import_dynamic runtime._LoadLibraryExW LoadLibraryExW%3 "kernel32.dll" //go:cgo_import_dynamic runtime._LoadLibraryW LoadLibraryW%1 "kernel32.dll" //go:cgo_import_dynamic runtime._PostQueuedCompletionStatus PostQueuedCompletionStatus%4 "kernel32.dll" //go:cgo_import_dynamic runtime._QueryPerformanceCounter QueryPerformanceCounter%1 "kernel32.dll" //go:cgo_import_dynamic runtime._QueryPerformanceFrequency QueryPerformanceFrequency%1 "kernel32.dll" //go:cgo_import_dynamic runtime._RaiseFailFastException RaiseFailFastException%3 "kernel32.dll" //go:cgo_import_dynamic runtime._ResumeThread ResumeThread%1 "kernel32.dll" //go:cgo_import_dynamic runtime._RtlLookupFunctionEntry RtlLookupFunctionEntry%3 "kernel32.dll" //go:cgo_import_dynamic runtime._RtlVirtualUnwind RtlVirtualUnwind%8 "kernel32.dll" //go:cgo_import_dynamic runtime._SetConsoleCtrlHandler SetConsoleCtrlHandler%2 "kernel32.dll" //go:cgo_import_dynamic runtime._SetErrorMode SetErrorMode%1 "kernel32.dll" //go:cgo_import_dynamic runtime._SetEvent SetEvent%1 "kernel32.dll" //go:cgo_import_dynamic runtime._SetProcessPriorityBoost SetProcessPriorityBoost%2 "kernel32.dll" //go:cgo_import_dynamic runtime._SetThreadPriority SetThreadPriority%2 "kernel32.dll" //go:cgo_import_dynamic runtime._SetUnhandledExceptionFilter SetUnhandledExceptionFilter%1 "kernel32.dll" //go:cgo_import_dynamic runtime._SetWaitableTimer SetWaitableTimer%6 "kernel32.dll" //go:cgo_import_dynamic runtime._SuspendThread SuspendThread%1 "kernel32.dll" //go:cgo_import_dynamic runtime._SwitchToThread SwitchToThread%0 "kernel32.dll" //go:cgo_import_dynamic runtime._TlsAlloc TlsAlloc%0 "kernel32.dll" //go:cgo_import_dynamic runtime._VirtualAlloc VirtualAlloc%4 "kernel32.dll" //go:cgo_import_dynamic runtime._VirtualFree VirtualFree%3 "kernel32.dll" //go:cgo_import_dynamic runtime._VirtualQuery VirtualQuery%3 "kernel32.dll" //go:cgo_import_dynamic runtime._WaitForSingleObject WaitForSingleObject%2 "kernel32.dll" //go:cgo_import_dynamic runtime._WaitForMultipleObjects WaitForMultipleObjects%4 "kernel32.dll" //go:cgo_import_dynamic runtime._WerGetFlags WerGetFlags%2 "kernel32.dll" //go:cgo_import_dynamic runtime._WerSetFlags WerSetFlags%1 "kernel32.dll" //go:cgo_import_dynamic runtime._WriteConsoleW WriteConsoleW%5 "kernel32.dll" //go:cgo_import_dynamic runtime._WriteFile WriteFile%5 "kernel32.dll" type stdFunction unsafe.Pointer var ( // Following syscalls are available on every Windows PC. // All these variables are set by the Windows executable // loader before the Go program starts. _AddVectoredContinueHandler, _AddVectoredExceptionHandler, _CloseHandle, _CreateEventA, _CreateIoCompletionPort, _CreateThread, _CreateWaitableTimerA, _CreateWaitableTimerExW, _DuplicateHandle, _ExitProcess, _FreeEnvironmentStringsW, _GetConsoleMode, _GetCurrentThreadId, _GetEnvironmentStringsW, _GetErrorMode, _GetProcAddress, _GetProcessAffinityMask, _GetQueuedCompletionStatusEx, _GetStdHandle, _GetSystemDirectoryA, _GetSystemInfo, _GetThreadContext, _SetThreadContext, _LoadLibraryExW, _LoadLibraryW, _PostQueuedCompletionStatus, _QueryPerformanceCounter, _QueryPerformanceFrequency, _RaiseFailFastException, _ResumeThread, _RtlLookupFunctionEntry, _RtlVirtualUnwind, _SetConsoleCtrlHandler, _SetErrorMode, _SetEvent, _SetProcessPriorityBoost, _SetThreadPriority, _SetUnhandledExceptionFilter, _SetWaitableTimer, _SuspendThread, _SwitchToThread, _TlsAlloc, _VirtualAlloc, _VirtualFree, _VirtualQuery, _WaitForSingleObject, _WaitForMultipleObjects, _WerGetFlags, _WerSetFlags, _WriteConsoleW, _WriteFile, _ stdFunction // Use ProcessPrng to generate cryptographically random data. _ProcessPrng stdFunction // Load ntdll.dll manually during startup, otherwise Mingw // links wrong printf function to cgo executable (see issue // 12030 for details). _NtCreateWaitCompletionPacket stdFunction _NtAssociateWaitCompletionPacket stdFunction _NtCancelWaitCompletionPacket stdFunction _RtlGetCurrentPeb stdFunction _RtlGetVersion stdFunction // These are from non-kernel32.dll, so we prefer to LoadLibraryEx them. _timeBeginPeriod, _timeEndPeriod, _ stdFunction ) var ( bcryptprimitivesdll = [...]uint16{'b', 'c', 'r', 'y', 'p', 't', 'p', 'r', 'i', 'm', 'i', 't', 'i', 'v', 'e', 's', '.', 'd', 'l', 'l', 0} ntdlldll = [...]uint16{'n', 't', 'd', 'l', 'l', '.', 'd', 'l', 'l', 0} powrprofdll = [...]uint16{'p', 'o', 'w', 'r', 'p', 'r', 'o', 'f', '.', 'd', 'l', 'l', 0} winmmdll = [...]uint16{'w', 'i', 'n', 'm', 'm', '.', 'd', 'l', 'l', 0} ) // Function to be called by windows CreateThread // to start new os thread. func tstart_stdcall(newm *m) // Init-time helper func wintls() type mOS struct { threadLock mutex // protects "thread" and prevents closing thread uintptr // thread handle waitsema uintptr // semaphore for parking on locks resumesema uintptr // semaphore to indicate suspend/resume highResTimer uintptr // high resolution timer handle used in usleep waitIocpTimer uintptr // high resolution timer handle used in netpoll waitIocpHandle uintptr // wait completion handle used in netpoll // preemptExtLock synchronizes preemptM with entry/exit from // external C code. // // This protects against races between preemptM calling // SuspendThread and external code on this thread calling // ExitProcess. If these happen concurrently, it's possible to // exit the suspending thread and suspend the exiting thread, // leading to deadlock. // // 0 indicates this M is not being preempted or in external // code. Entering external code CASes this from 0 to 1. If // this fails, a preemption is in progress, so the thread must // wait for the preemption. preemptM also CASes this from 0 to // 1. If this fails, the preemption fails (as it would if the // PC weren't in Go code). The value is reset to 0 when // returning from external code or after a preemption is // complete. // // TODO(austin): We may not need this if preemption were more // tightly synchronized on the G/P status and preemption // blocked transition into _Gsyscall/_Psyscall. preemptExtLock uint32 } // Stubs so tests can link correctly. These should never be called. func open(name *byte, mode, perm int32) int32 { throw("unimplemented") return -1 } func closefd(fd int32) int32 { throw("unimplemented") return -1 } func read(fd int32, p unsafe.Pointer, n int32) int32 { throw("unimplemented") return -1 } type sigset struct{} // Call a Windows function with stdcall conventions, // and switch to os stack during the call. func asmstdcall(fn unsafe.Pointer) var asmstdcallAddr unsafe.Pointer type winlibcall libcall func windowsFindfunc(lib uintptr, name []byte) stdFunction { if name[len(name)-1] != 0 { throw("usage") } f := stdcall2(_GetProcAddress, lib, uintptr(unsafe.Pointer(&name[0]))) return stdFunction(unsafe.Pointer(f)) } const _MAX_PATH = 260 // https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation var sysDirectory [_MAX_PATH + 1]byte var sysDirectoryLen uintptr func initSysDirectory() { l := stdcall2(_GetSystemDirectoryA, uintptr(unsafe.Pointer(&sysDirectory[0])), uintptr(len(sysDirectory)-1)) if l == 0 || l > uintptr(len(sysDirectory)-1) { throw("Unable to determine system directory") } sysDirectory[l] = '\\' sysDirectoryLen = l + 1 } //go:linkname windows_GetSystemDirectory internal/syscall/windows.GetSystemDirectory func windows_GetSystemDirectory() string { return unsafe.String(&sysDirectory[0], sysDirectoryLen) } func windowsLoadSystemLib(name []uint16) uintptr { return stdcall3(_LoadLibraryExW, uintptr(unsafe.Pointer(&name[0])), 0, _LOAD_LIBRARY_SEARCH_SYSTEM32) } //go:linkname windows_QueryPerformanceCounter internal/syscall/windows.QueryPerformanceCounter func windows_QueryPerformanceCounter() int64 { var counter int64 stdcall1(_QueryPerformanceCounter, uintptr(unsafe.Pointer(&counter))) return counter } //go:linkname windows_QueryPerformanceFrequency internal/syscall/windows.QueryPerformanceFrequency func windows_QueryPerformanceFrequency() int64 { var frequency int64 stdcall1(_QueryPerformanceFrequency, uintptr(unsafe.Pointer(&frequency))) return frequency } func loadOptionalSyscalls() { bcryptPrimitives := windowsLoadSystemLib(bcryptprimitivesdll[:]) if bcryptPrimitives == 0 { throw("bcryptprimitives.dll not found") } _ProcessPrng = windowsFindfunc(bcryptPrimitives, []byte("ProcessPrng\000")) n32 := windowsLoadSystemLib(ntdlldll[:]) if n32 == 0 { throw("ntdll.dll not found") } _NtCreateWaitCompletionPacket = windowsFindfunc(n32, []byte("NtCreateWaitCompletionPacket\000")) if _NtCreateWaitCompletionPacket != nil { // These functions should exists if NtCreateWaitCompletionPacket exists. _NtAssociateWaitCompletionPacket = windowsFindfunc(n32, []byte("NtAssociateWaitCompletionPacket\000")) if _NtAssociateWaitCompletionPacket == nil { throw("NtCreateWaitCompletionPacket exists but NtAssociateWaitCompletionPacket does not") } _NtCancelWaitCompletionPacket = windowsFindfunc(n32, []byte("NtCancelWaitCompletionPacket\000")) if _NtCancelWaitCompletionPacket == nil { throw("NtCreateWaitCompletionPacket exists but NtCancelWaitCompletionPacket does not") } } _RtlGetCurrentPeb = windowsFindfunc(n32, []byte("RtlGetCurrentPeb\000")) _RtlGetVersion = windowsFindfunc(n32, []byte("RtlGetVersion\000")) } func monitorSuspendResume() { const ( _DEVICE_NOTIFY_CALLBACK = 2 ) type _DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS struct { callback uintptr context uintptr } powrprof := windowsLoadSystemLib(powrprofdll[:]) if powrprof == 0 { return // Running on Windows 7, where we don't need it anyway. } powerRegisterSuspendResumeNotification := windowsFindfunc(powrprof, []byte("PowerRegisterSuspendResumeNotification\000")) if powerRegisterSuspendResumeNotification == nil { return // Running on Windows 7, where we don't need it anyway. } var fn any = func(context uintptr, changeType uint32, setting uintptr) uintptr { for mp := (*m)(atomic.Loadp(unsafe.Pointer(&allm))); mp != nil; mp = mp.alllink { if mp.resumesema != 0 { stdcall1(_SetEvent, mp.resumesema) } } return 0 } params := _DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS{ callback: compileCallback(*efaceOf(&fn), true), } handle := uintptr(0) stdcall3(powerRegisterSuspendResumeNotification, _DEVICE_NOTIFY_CALLBACK, uintptr(unsafe.Pointer(¶ms)), uintptr(unsafe.Pointer(&handle))) } func getproccount() int32 { var mask, sysmask uintptr ret := stdcall3(_GetProcessAffinityMask, currentProcess, uintptr(unsafe.Pointer(&mask)), uintptr(unsafe.Pointer(&sysmask))) if ret != 0 { n := 0 maskbits := int(unsafe.Sizeof(mask) * 8) for i := 0; i < maskbits; i++ { if mask&(1<= 0x80 { isASCII = false break } } if !isASCII { var m uint32 isConsole := stdcall2(_GetConsoleMode, handle, uintptr(unsafe.Pointer(&m))) != 0 // If this is a console output, various non-unicode code pages can be in use. // Use the dedicated WriteConsole call to ensure unicode is printed correctly. if isConsole { return int32(writeConsole(handle, buf, n)) } } var written uint32 stdcall5(_WriteFile, handle, uintptr(buf), uintptr(n), uintptr(unsafe.Pointer(&written)), 0) return int32(written) } var ( utf16ConsoleBack [1000]uint16 utf16ConsoleBackLock mutex ) // writeConsole writes bufLen bytes from buf to the console File. // It returns the number of bytes written. func writeConsole(handle uintptr, buf unsafe.Pointer, bufLen int32) int { const surr2 = (surrogateMin + surrogateMax + 1) / 2 // Do not use defer for unlock. May cause issues when printing a panic. lock(&utf16ConsoleBackLock) b := (*[1 << 30]byte)(buf)[:bufLen] s := *(*string)(unsafe.Pointer(&b)) utf16tmp := utf16ConsoleBack[:] total := len(s) w := 0 for _, r := range s { if w >= len(utf16tmp)-2 { writeConsoleUTF16(handle, utf16tmp[:w]) w = 0 } if r < 0x10000 { utf16tmp[w] = uint16(r) w++ } else { r -= 0x10000 utf16tmp[w] = surrogateMin + uint16(r>>10)&0x3ff utf16tmp[w+1] = surr2 + uint16(r)&0x3ff w += 2 } } writeConsoleUTF16(handle, utf16tmp[:w]) unlock(&utf16ConsoleBackLock) return total } // writeConsoleUTF16 is the dedicated windows calls that correctly prints // to the console regardless of the current code page. Input is utf-16 code points. // The handle must be a console handle. func writeConsoleUTF16(handle uintptr, b []uint16) { l := uint32(len(b)) if l == 0 { return } var written uint32 stdcall5(_WriteConsoleW, handle, uintptr(unsafe.Pointer(&b[0])), uintptr(l), uintptr(unsafe.Pointer(&written)), 0, ) return } //go:nosplit func semasleep(ns int64) int32 { const ( _WAIT_ABANDONED = 0x00000080 _WAIT_OBJECT_0 = 0x00000000 _WAIT_TIMEOUT = 0x00000102 _WAIT_FAILED = 0xFFFFFFFF ) var result uintptr if ns < 0 { result = stdcall2(_WaitForSingleObject, getg().m.waitsema, uintptr(_INFINITE)) } else { start := nanotime() elapsed := int64(0) for { ms := int64(timediv(ns-elapsed, 1000000, nil)) if ms == 0 { ms = 1 } result = stdcall4(_WaitForMultipleObjects, 2, uintptr(unsafe.Pointer(&[2]uintptr{getg().m.waitsema, getg().m.resumesema})), 0, uintptr(ms)) if result != _WAIT_OBJECT_0+1 { // Not a suspend/resume event break } elapsed = nanotime() - start if elapsed >= ns { return -1 } } } switch result { case _WAIT_OBJECT_0: // Signaled return 0 case _WAIT_TIMEOUT: return -1 case _WAIT_ABANDONED: systemstack(func() { throw("runtime.semasleep wait_abandoned") }) case _WAIT_FAILED: systemstack(func() { print("runtime: waitforsingleobject wait_failed; errno=", getlasterror(), "\n") throw("runtime.semasleep wait_failed") }) default: systemstack(func() { print("runtime: waitforsingleobject unexpected; result=", result, "\n") throw("runtime.semasleep unexpected") }) } return -1 // unreachable } //go:nosplit func semawakeup(mp *m) { if stdcall1(_SetEvent, mp.waitsema) == 0 { systemstack(func() { print("runtime: setevent failed; errno=", getlasterror(), "\n") throw("runtime.semawakeup") }) } } //go:nosplit func semacreate(mp *m) { if mp.waitsema != 0 { return } mp.waitsema = stdcall4(_CreateEventA, 0, 0, 0, 0) if mp.waitsema == 0 { systemstack(func() { print("runtime: createevent failed; errno=", getlasterror(), "\n") throw("runtime.semacreate") }) } mp.resumesema = stdcall4(_CreateEventA, 0, 0, 0, 0) if mp.resumesema == 0 { systemstack(func() { print("runtime: createevent failed; errno=", getlasterror(), "\n") throw("runtime.semacreate") }) stdcall1(_CloseHandle, mp.waitsema) mp.waitsema = 0 } } // May run with m.p==nil, so write barriers are not allowed. This // function is called by newosproc0, so it is also required to // operate without stack guards. // //go:nowritebarrierrec //go:nosplit func newosproc(mp *m) { // We pass 0 for the stack size to use the default for this binary. thandle := stdcall6(_CreateThread, 0, 0, abi.FuncPCABI0(tstart_stdcall), uintptr(unsafe.Pointer(mp)), 0, 0) if thandle == 0 { if atomic.Load(&exiting) != 0 { // CreateThread may fail if called // concurrently with ExitProcess. If this // happens, just freeze this thread and let // the process exit. See issue #18253. lock(&deadlock) lock(&deadlock) } print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", getlasterror(), ")\n") throw("runtime.newosproc") } // Close thandle to avoid leaking the thread object if it exits. stdcall1(_CloseHandle, thandle) } // Used by the C library build mode. On Linux this function would allocate a // stack, but that's not necessary for Windows. No stack guards are present // and the GC has not been initialized, so write barriers will fail. // //go:nowritebarrierrec //go:nosplit func newosproc0(mp *m, stk unsafe.Pointer) { // TODO: this is completely broken. The args passed to newosproc0 (in asm_amd64.s) // are stacksize and function, not *m and stack. // Check os_linux.go for an implementation that might actually work. throw("bad newosproc0") } func exitThread(wait *atomic.Uint32) { // We should never reach exitThread on Windows because we let // the OS clean up threads. throw("exitThread") } // Called to initialize a new m (including the bootstrap m). // Called on the parent thread (main thread in case of bootstrap), can allocate memory. func mpreinit(mp *m) { } //go:nosplit func sigsave(p *sigset) { } //go:nosplit func msigrestore(sigmask sigset) { } //go:nosplit //go:nowritebarrierrec func clearSignalHandlers() { } //go:nosplit func sigblock(exiting bool) { } // Called to initialize a new m (including the bootstrap m). // Called on the new thread, cannot allocate Go memory. func minit() { var thandle uintptr if stdcall7(_DuplicateHandle, currentProcess, currentThread, currentProcess, uintptr(unsafe.Pointer(&thandle)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 { print("runtime.minit: duplicatehandle failed; errno=", getlasterror(), "\n") throw("runtime.minit: duplicatehandle failed") } mp := getg().m lock(&mp.threadLock) mp.thread = thandle mp.procid = uint64(stdcall0(_GetCurrentThreadId)) // Configure usleep timer, if possible. if mp.highResTimer == 0 && haveHighResTimer { mp.highResTimer = createHighResTimer() if mp.highResTimer == 0 { print("runtime: CreateWaitableTimerEx failed; errno=", getlasterror(), "\n") throw("CreateWaitableTimerEx when creating timer failed") } } if mp.waitIocpHandle == 0 && haveHighResSleep { mp.waitIocpTimer = createHighResTimer() if mp.waitIocpTimer == 0 { print("runtime: CreateWaitableTimerEx failed; errno=", getlasterror(), "\n") throw("CreateWaitableTimerEx when creating timer failed") } const GENERIC_ALL = 0x10000000 errno := stdcall3(_NtCreateWaitCompletionPacket, uintptr(unsafe.Pointer(&mp.waitIocpHandle)), GENERIC_ALL, 0) if mp.waitIocpHandle == 0 { print("runtime: NtCreateWaitCompletionPacket failed; errno=", errno, "\n") throw("NtCreateWaitCompletionPacket failed") } } unlock(&mp.threadLock) // Query the true stack base from the OS. Currently we're // running on a small assumed stack. var mbi memoryBasicInformation res := stdcall3(_VirtualQuery, uintptr(unsafe.Pointer(&mbi)), uintptr(unsafe.Pointer(&mbi)), unsafe.Sizeof(mbi)) if res == 0 { print("runtime: VirtualQuery failed; errno=", getlasterror(), "\n") throw("VirtualQuery for stack base failed") } // The system leaves an 8K PAGE_GUARD region at the bottom of // the stack (in theory VirtualQuery isn't supposed to include // that, but it does). Add an additional 8K of slop for // calling C functions that don't have stack checks and for // lastcontinuehandler. We shouldn't be anywhere near this // bound anyway. base := mbi.allocationBase + 16<<10 // Sanity check the stack bounds. g0 := getg() if base > g0.stack.hi || g0.stack.hi-base > 64<<20 { print("runtime: g0 stack [", hex(base), ",", hex(g0.stack.hi), ")\n") throw("bad g0 stack") } g0.stack.lo = base g0.stackguard0 = g0.stack.lo + stackGuard g0.stackguard1 = g0.stackguard0 // Sanity check the SP. stackcheck() } // Called from dropm to undo the effect of an minit. // //go:nosplit func unminit() { mp := getg().m lock(&mp.threadLock) if mp.thread != 0 { stdcall1(_CloseHandle, mp.thread) mp.thread = 0 } unlock(&mp.threadLock) mp.procid = 0 } // Called from exitm, but not from drop, to undo the effect of thread-owned // resources in minit, semacreate, or elsewhere. Do not take locks after calling this. // //go:nosplit func mdestroy(mp *m) { if mp.highResTimer != 0 { stdcall1(_CloseHandle, mp.highResTimer) mp.highResTimer = 0 } if mp.waitIocpTimer != 0 { stdcall1(_CloseHandle, mp.waitIocpTimer) mp.waitIocpTimer = 0 } if mp.waitIocpHandle != 0 { stdcall1(_CloseHandle, mp.waitIocpHandle) mp.waitIocpHandle = 0 } if mp.waitsema != 0 { stdcall1(_CloseHandle, mp.waitsema) mp.waitsema = 0 } if mp.resumesema != 0 { stdcall1(_CloseHandle, mp.resumesema) mp.resumesema = 0 } } // asmstdcall_trampoline calls asmstdcall converting from Go to C calling convention. func asmstdcall_trampoline(args unsafe.Pointer) // stdcall_no_g calls asmstdcall on os stack without using g. // //go:nosplit func stdcall_no_g(fn stdFunction, n int, args uintptr) uintptr { libcall := libcall{ fn: uintptr(unsafe.Pointer(fn)), n: uintptr(n), args: args, } asmstdcall_trampoline(noescape(unsafe.Pointer(&libcall))) return libcall.r1 } // Calling stdcall on os stack. // May run during STW, so write barriers are not allowed. // //go:nowritebarrier //go:nosplit func stdcall(fn stdFunction) uintptr { gp := getg() mp := gp.m mp.libcall.fn = uintptr(unsafe.Pointer(fn)) resetLibcall := false if mp.profilehz != 0 && mp.libcallsp == 0 { // leave pc/sp for cpu profiler mp.libcallg.set(gp) mp.libcallpc = sys.GetCallerPC() // sp must be the last, because once async cpu profiler finds // all three values to be non-zero, it will use them mp.libcallsp = sys.GetCallerSP() resetLibcall = true // See comment in sys_darwin.go:libcCall } asmcgocall(asmstdcallAddr, unsafe.Pointer(&mp.libcall)) if resetLibcall { mp.libcallsp = 0 } return mp.libcall.r1 } //go:nosplit func stdcall0(fn stdFunction) uintptr { mp := getg().m mp.libcall.n = 0 mp.libcall.args = 0 return stdcall(fn) } //go:nosplit //go:cgo_unsafe_args func stdcall1(fn stdFunction, a0 uintptr) uintptr { mp := getg().m mp.libcall.n = 1 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) return stdcall(fn) } //go:nosplit //go:cgo_unsafe_args func stdcall2(fn stdFunction, a0, a1 uintptr) uintptr { mp := getg().m mp.libcall.n = 2 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) return stdcall(fn) } //go:nosplit //go:cgo_unsafe_args func stdcall3(fn stdFunction, a0, a1, a2 uintptr) uintptr { mp := getg().m mp.libcall.n = 3 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) return stdcall(fn) } //go:nosplit //go:cgo_unsafe_args func stdcall4(fn stdFunction, a0, a1, a2, a3 uintptr) uintptr { mp := getg().m mp.libcall.n = 4 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) return stdcall(fn) } //go:nosplit //go:cgo_unsafe_args func stdcall5(fn stdFunction, a0, a1, a2, a3, a4 uintptr) uintptr { mp := getg().m mp.libcall.n = 5 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) return stdcall(fn) } //go:nosplit //go:cgo_unsafe_args func stdcall6(fn stdFunction, a0, a1, a2, a3, a4, a5 uintptr) uintptr { mp := getg().m mp.libcall.n = 6 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) return stdcall(fn) } //go:nosplit //go:cgo_unsafe_args func stdcall7(fn stdFunction, a0, a1, a2, a3, a4, a5, a6 uintptr) uintptr { mp := getg().m mp.libcall.n = 7 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) return stdcall(fn) } //go:nosplit //go:cgo_unsafe_args func stdcall8(fn stdFunction, a0, a1, a2, a3, a4, a5, a6, a7 uintptr) uintptr { mp := getg().m mp.libcall.n = 8 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) return stdcall(fn) } // These must run on the system stack only. //go:nosplit func osyield_no_g() { stdcall_no_g(_SwitchToThread, 0, 0) } //go:nosplit func osyield() { systemstack(func() { stdcall0(_SwitchToThread) }) } //go:nosplit func usleep_no_g(us uint32) { timeout := uintptr(us) / 1000 // ms units args := [...]uintptr{_INVALID_HANDLE_VALUE, timeout} stdcall_no_g(_WaitForSingleObject, len(args), uintptr(noescape(unsafe.Pointer(&args[0])))) } //go:nosplit func usleep(us uint32) { systemstack(func() { var h, timeout uintptr // If the high-res timer is available and its handle has been allocated for this m, use it. // Otherwise fall back to the low-res one, which doesn't need a handle. if haveHighResTimer && getg().m.highResTimer != 0 { h = getg().m.highResTimer dt := -10 * int64(us) // relative sleep (negative), 100ns units stdcall6(_SetWaitableTimer, h, uintptr(unsafe.Pointer(&dt)), 0, 0, 0, 0) timeout = _INFINITE } else { h = _INVALID_HANDLE_VALUE timeout = uintptr(us) / 1000 // ms units } stdcall2(_WaitForSingleObject, h, timeout) }) } func ctrlHandler(_type uint32) uintptr { var s uint32 switch _type { case _CTRL_C_EVENT, _CTRL_BREAK_EVENT: s = _SIGINT case _CTRL_CLOSE_EVENT, _CTRL_LOGOFF_EVENT, _CTRL_SHUTDOWN_EVENT: s = _SIGTERM default: return 0 } if sigsend(s) { if s == _SIGTERM { // Windows terminates the process after this handler returns. // Block indefinitely to give signal handlers a chance to clean up, // but make sure to be properly parked first, so the rest of the // program can continue executing. block() } return 1 } return 0 } // called from zcallback_windows_*.s to sys_windows_*.s func callbackasm1() var profiletimer uintptr func profilem(mp *m, thread uintptr) { // Align Context to 16 bytes. var c *context var cbuf [unsafe.Sizeof(*c) + 15]byte c = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15)) c.contextflags = _CONTEXT_CONTROL stdcall2(_GetThreadContext, thread, uintptr(unsafe.Pointer(c))) gp := gFromSP(mp, c.sp()) sigprof(c.ip(), c.sp(), c.lr(), gp, mp) } func gFromSP(mp *m, sp uintptr) *g { if gp := mp.g0; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi { return gp } if gp := mp.gsignal; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi { return gp } if gp := mp.curg; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi { return gp } return nil } func profileLoop() { stdcall2(_SetThreadPriority, currentThread, _THREAD_PRIORITY_HIGHEST) for { stdcall2(_WaitForSingleObject, profiletimer, _INFINITE) first := (*m)(atomic.Loadp(unsafe.Pointer(&allm))) for mp := first; mp != nil; mp = mp.alllink { if mp == getg().m { // Don't profile ourselves. continue } lock(&mp.threadLock) // Do not profile threads blocked on Notes, // this includes idle worker threads, // idle timer thread, idle heap scavenger, etc. if mp.thread == 0 || mp.profilehz == 0 || mp.blocked { unlock(&mp.threadLock) continue } // Acquire our own handle to the thread. var thread uintptr if stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 { print("runtime: duplicatehandle failed; errno=", getlasterror(), "\n") throw("duplicatehandle failed") } unlock(&mp.threadLock) // mp may exit between the DuplicateHandle // above and the SuspendThread. The handle // will remain valid, but SuspendThread may // fail. if int32(stdcall1(_SuspendThread, thread)) == -1 { // The thread no longer exists. stdcall1(_CloseHandle, thread) continue } if mp.profilehz != 0 && !mp.blocked { // Pass the thread handle in case mp // was in the process of shutting down. profilem(mp, thread) } stdcall1(_ResumeThread, thread) stdcall1(_CloseHandle, thread) } } } func setProcessCPUProfiler(hz int32) { if profiletimer == 0 { var timer uintptr if haveHighResTimer { timer = createHighResTimer() } else { timer = stdcall3(_CreateWaitableTimerA, 0, 0, 0) } atomic.Storeuintptr(&profiletimer, timer) newm(profileLoop, nil, -1) } } func setThreadCPUProfiler(hz int32) { ms := int32(0) due := ^int64(^uint64(1 << 63)) if hz > 0 { ms = 1000 / hz if ms == 0 { ms = 1 } due = int64(ms) * -10000 } stdcall6(_SetWaitableTimer, profiletimer, uintptr(unsafe.Pointer(&due)), uintptr(ms), 0, 0, 0) atomic.Store((*uint32)(unsafe.Pointer(&getg().m.profilehz)), uint32(hz)) } const preemptMSupported = true // suspendLock protects simultaneous SuspendThread operations from // suspending each other. var suspendLock mutex func preemptM(mp *m) { if mp == getg().m { throw("self-preempt") } // Synchronize with external code that may try to ExitProcess. if !atomic.Cas(&mp.preemptExtLock, 0, 1) { // External code is running. Fail the preemption // attempt. mp.preemptGen.Add(1) return } // Acquire our own handle to mp's thread. lock(&mp.threadLock) if mp.thread == 0 { // The M hasn't been minit'd yet (or was just unminit'd). unlock(&mp.threadLock) atomic.Store(&mp.preemptExtLock, 0) mp.preemptGen.Add(1) return } var thread uintptr if stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 { print("runtime.preemptM: duplicatehandle failed; errno=", getlasterror(), "\n") throw("runtime.preemptM: duplicatehandle failed") } unlock(&mp.threadLock) // Prepare thread context buffer. This must be aligned to 16 bytes. var c *context var cbuf [unsafe.Sizeof(*c) + 15]byte c = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15)) c.contextflags = _CONTEXT_CONTROL // Serialize thread suspension. SuspendThread is asynchronous, // so it's otherwise possible for two threads to suspend each // other and deadlock. We must hold this lock until after // GetThreadContext, since that blocks until the thread is // actually suspended. lock(&suspendLock) // Suspend the thread. if int32(stdcall1(_SuspendThread, thread)) == -1 { unlock(&suspendLock) stdcall1(_CloseHandle, thread) atomic.Store(&mp.preemptExtLock, 0) // The thread no longer exists. This shouldn't be // possible, but just acknowledge the request. mp.preemptGen.Add(1) return } // We have to be very careful between this point and once // we've shown mp is at an async safe-point. This is like a // signal handler in the sense that mp could have been doing // anything when we stopped it, including holding arbitrary // locks. // We have to get the thread context before inspecting the M // because SuspendThread only requests a suspend. // GetThreadContext actually blocks until it's suspended. stdcall2(_GetThreadContext, thread, uintptr(unsafe.Pointer(c))) unlock(&suspendLock) // Does it want a preemption and is it safe to preempt? gp := gFromSP(mp, c.sp()) if gp != nil && wantAsyncPreempt(gp) { if ok, newpc := isAsyncSafePoint(gp, c.ip(), c.sp(), c.lr()); ok { // Inject call to asyncPreempt targetPC := abi.FuncPCABI0(asyncPreempt) switch GOARCH { default: throw("unsupported architecture") case "386", "amd64": // Make it look like the thread called targetPC. sp := c.sp() sp -= goarch.PtrSize *(*uintptr)(unsafe.Pointer(sp)) = newpc c.set_sp(sp) c.set_ip(targetPC) case "arm": // Push LR. The injected call is responsible // for restoring LR. gentraceback is aware of // this extra slot. See sigctxt.pushCall in // signal_arm.go, which is similar except we // subtract 1 from IP here. sp := c.sp() sp -= goarch.PtrSize c.set_sp(sp) *(*uint32)(unsafe.Pointer(sp)) = uint32(c.lr()) c.set_lr(newpc - 1) c.set_ip(targetPC) case "arm64": // Push LR. The injected call is responsible // for restoring LR. gentraceback is aware of // this extra slot. See sigctxt.pushCall in // signal_arm64.go. sp := c.sp() - 16 // SP needs 16-byte alignment c.set_sp(sp) *(*uint64)(unsafe.Pointer(sp)) = uint64(c.lr()) c.set_lr(newpc) c.set_ip(targetPC) } stdcall2(_SetThreadContext, thread, uintptr(unsafe.Pointer(c))) } } atomic.Store(&mp.preemptExtLock, 0) // Acknowledge the preemption. mp.preemptGen.Add(1) stdcall1(_ResumeThread, thread) stdcall1(_CloseHandle, thread) } // osPreemptExtEnter is called before entering external code that may // call ExitProcess. // // This must be nosplit because it may be called from a syscall with // untyped stack slots, so the stack must not be grown or scanned. // //go:nosplit func osPreemptExtEnter(mp *m) { for !atomic.Cas(&mp.preemptExtLock, 0, 1) { // An asynchronous preemption is in progress. It's not // safe to enter external code because it may call // ExitProcess and deadlock with SuspendThread. // Ideally we would do the preemption ourselves, but // can't since there may be untyped syscall arguments // on the stack. Instead, just wait and encourage the // SuspendThread APC to run. The preemption should be // done shortly. osyield() } // Asynchronous preemption is now blocked. } // osPreemptExtExit is called after returning from external code that // may call ExitProcess. // // See osPreemptExtEnter for why this is nosplit. // //go:nosplit func osPreemptExtExit(mp *m) { atomic.Store(&mp.preemptExtLock, 0) }