Source file src/runtime/os_windows.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/abi"
     9  	"internal/runtime/atomic"
    10  	"internal/runtime/sys"
    11  	"internal/runtime/syscall/windows"
    12  	"unsafe"
    13  )
    14  
    15  // TODO(brainman): should not need those
    16  const (
    17  	_NSIG = 65
    18  )
    19  
    20  //go:cgo_import_dynamic runtime._AddVectoredContinueHandler AddVectoredContinueHandler%2 "kernel32.dll"
    21  //go:cgo_import_dynamic runtime._AddVectoredExceptionHandler AddVectoredExceptionHandler%2 "kernel32.dll"
    22  //go:cgo_import_dynamic runtime._CloseHandle CloseHandle%1 "kernel32.dll"
    23  //go:cgo_import_dynamic runtime._CreateEventA CreateEventA%4 "kernel32.dll"
    24  //go:cgo_import_dynamic runtime._CreateIoCompletionPort CreateIoCompletionPort%4 "kernel32.dll"
    25  //go:cgo_import_dynamic runtime._CreateThread CreateThread%6 "kernel32.dll"
    26  //go:cgo_import_dynamic runtime._CreateWaitableTimerA CreateWaitableTimerA%3 "kernel32.dll"
    27  //go:cgo_import_dynamic runtime._CreateWaitableTimerExW CreateWaitableTimerExW%4 "kernel32.dll"
    28  //go:cgo_import_dynamic runtime._DuplicateHandle DuplicateHandle%7 "kernel32.dll"
    29  //go:cgo_import_dynamic runtime._ExitProcess ExitProcess%1 "kernel32.dll"
    30  //go:cgo_import_dynamic runtime._FreeEnvironmentStringsW FreeEnvironmentStringsW%1 "kernel32.dll"
    31  //go:cgo_import_dynamic runtime._GetConsoleMode GetConsoleMode%2 "kernel32.dll"
    32  //go:cgo_import_dynamic runtime._GetCurrentThreadId GetCurrentThreadId%0 "kernel32.dll"
    33  //go:cgo_import_dynamic runtime._GetEnvironmentStringsW GetEnvironmentStringsW%0 "kernel32.dll"
    34  //go:cgo_import_dynamic runtime._GetErrorMode GetErrorMode%0 "kernel32.dll"
    35  //go:cgo_import_dynamic runtime._GetProcAddress GetProcAddress%2 "kernel32.dll"
    36  //go:cgo_import_dynamic runtime._GetProcessAffinityMask GetProcessAffinityMask%3 "kernel32.dll"
    37  //go:cgo_import_dynamic runtime._GetQueuedCompletionStatusEx GetQueuedCompletionStatusEx%6 "kernel32.dll"
    38  //go:cgo_import_dynamic runtime._GetStdHandle GetStdHandle%1 "kernel32.dll"
    39  //go:cgo_import_dynamic runtime._GetSystemDirectoryA GetSystemDirectoryA%2 "kernel32.dll"
    40  //go:cgo_import_dynamic runtime._GetSystemInfo GetSystemInfo%1 "kernel32.dll"
    41  //go:cgo_import_dynamic runtime._GetThreadContext GetThreadContext%2 "kernel32.dll"
    42  //go:cgo_import_dynamic runtime._IsProcessorFeaturePresent IsProcessorFeaturePresent%1 "kernel32.dll"
    43  //go:cgo_import_dynamic runtime._SetThreadContext SetThreadContext%2 "kernel32.dll"
    44  //go:cgo_import_dynamic runtime._LoadLibraryExW LoadLibraryExW%3 "kernel32.dll"
    45  //go:cgo_import_dynamic runtime._PostQueuedCompletionStatus PostQueuedCompletionStatus%4 "kernel32.dll"
    46  //go:cgo_import_dynamic runtime._QueryPerformanceCounter QueryPerformanceCounter%1 "kernel32.dll"
    47  //go:cgo_import_dynamic runtime._QueryPerformanceFrequency QueryPerformanceFrequency%1 "kernel32.dll"
    48  //go:cgo_import_dynamic runtime._RaiseFailFastException RaiseFailFastException%3 "kernel32.dll"
    49  //go:cgo_import_dynamic runtime._ResumeThread ResumeThread%1 "kernel32.dll"
    50  //go:cgo_import_dynamic runtime._RtlLookupFunctionEntry RtlLookupFunctionEntry%3 "kernel32.dll"
    51  //go:cgo_import_dynamic runtime._RtlVirtualUnwind  RtlVirtualUnwind%8 "kernel32.dll"
    52  //go:cgo_import_dynamic runtime._SetConsoleCtrlHandler SetConsoleCtrlHandler%2 "kernel32.dll"
    53  //go:cgo_import_dynamic runtime._SetErrorMode SetErrorMode%1 "kernel32.dll"
    54  //go:cgo_import_dynamic runtime._SetEvent SetEvent%1 "kernel32.dll"
    55  //go:cgo_import_dynamic runtime._SetProcessPriorityBoost SetProcessPriorityBoost%2 "kernel32.dll"
    56  //go:cgo_import_dynamic runtime._SetThreadPriority SetThreadPriority%2 "kernel32.dll"
    57  //go:cgo_import_dynamic runtime._SetUnhandledExceptionFilter SetUnhandledExceptionFilter%1 "kernel32.dll"
    58  //go:cgo_import_dynamic runtime._SetWaitableTimer SetWaitableTimer%6 "kernel32.dll"
    59  //go:cgo_import_dynamic runtime._SuspendThread SuspendThread%1 "kernel32.dll"
    60  //go:cgo_import_dynamic runtime._SwitchToThread SwitchToThread%0 "kernel32.dll"
    61  //go:cgo_import_dynamic runtime._VirtualAlloc VirtualAlloc%4 "kernel32.dll"
    62  //go:cgo_import_dynamic runtime._VirtualFree VirtualFree%3 "kernel32.dll"
    63  //go:cgo_import_dynamic runtime._VirtualQuery VirtualQuery%3 "kernel32.dll"
    64  //go:cgo_import_dynamic runtime._WaitForSingleObject WaitForSingleObject%2 "kernel32.dll"
    65  //go:cgo_import_dynamic runtime._WaitForMultipleObjects WaitForMultipleObjects%4 "kernel32.dll"
    66  //go:cgo_import_dynamic runtime._WerGetFlags WerGetFlags%2 "kernel32.dll"
    67  //go:cgo_import_dynamic runtime._WerSetFlags WerSetFlags%1 "kernel32.dll"
    68  //go:cgo_import_dynamic runtime._WriteConsoleW WriteConsoleW%5 "kernel32.dll"
    69  //go:cgo_import_dynamic runtime._WriteFile WriteFile%5 "kernel32.dll"
    70  
    71  type stdFunction unsafe.Pointer
    72  
    73  var (
    74  	// Following syscalls are available on every Windows PC.
    75  	// All these variables are set by the Windows executable
    76  	// loader before the Go program starts.
    77  	_AddVectoredContinueHandler,
    78  	_AddVectoredExceptionHandler,
    79  	_CloseHandle,
    80  	_CreateEventA,
    81  	_CreateIoCompletionPort,
    82  	_CreateThread,
    83  	_CreateWaitableTimerA,
    84  	_CreateWaitableTimerExW,
    85  	_DuplicateHandle,
    86  	_ExitProcess,
    87  	_FreeEnvironmentStringsW,
    88  	_GetConsoleMode,
    89  	_GetCurrentThreadId,
    90  	_GetEnvironmentStringsW,
    91  	_GetErrorMode,
    92  	_GetProcAddress,
    93  	_GetProcessAffinityMask,
    94  	_GetQueuedCompletionStatusEx,
    95  	_GetStdHandle,
    96  	_GetSystemDirectoryA,
    97  	_GetSystemInfo,
    98  	_GetThreadContext,
    99  	_IsProcessorFeaturePresent,
   100  	_SetThreadContext,
   101  	_LoadLibraryExW,
   102  	_PostQueuedCompletionStatus,
   103  	_QueryPerformanceCounter,
   104  	_QueryPerformanceFrequency,
   105  	_RaiseFailFastException,
   106  	_ResumeThread,
   107  	_RtlLookupFunctionEntry,
   108  	_RtlVirtualUnwind,
   109  	_SetConsoleCtrlHandler,
   110  	_SetErrorMode,
   111  	_SetEvent,
   112  	_SetProcessPriorityBoost,
   113  	_SetThreadPriority,
   114  	_SetUnhandledExceptionFilter,
   115  	_SetWaitableTimer,
   116  	_SuspendThread,
   117  	_SwitchToThread,
   118  	_VirtualAlloc,
   119  	_VirtualFree,
   120  	_VirtualQuery,
   121  	_WaitForSingleObject,
   122  	_WaitForMultipleObjects,
   123  	_WerGetFlags,
   124  	_WerSetFlags,
   125  	_WriteConsoleW,
   126  	_WriteFile,
   127  	_ stdFunction
   128  
   129  	// Use ProcessPrng to generate cryptographically random data.
   130  	_ProcessPrng stdFunction
   131  
   132  	// Initialized by osinit before schedinit calls nanotime. Used without
   133  	// the per-M state or LastError changes made by stdcall.
   134  	_QueryInterruptTimePrecise stdFunction
   135  
   136  	// Load ntdll.dll manually during startup, otherwise Mingw
   137  	// links wrong printf function to cgo executable (see issue
   138  	// 12030 for details).
   139  	_NtCreateWaitCompletionPacket    stdFunction
   140  	_NtAssociateWaitCompletionPacket stdFunction
   141  	_NtCancelWaitCompletionPacket    stdFunction
   142  	_RtlGetCurrentPeb                stdFunction
   143  	_RtlGetVersion                   stdFunction
   144  
   145  	// These are from non-kernel32.dll, so we prefer to LoadLibraryEx them.
   146  	_timeBeginPeriod,
   147  	_timeEndPeriod,
   148  	_ stdFunction
   149  )
   150  
   151  var (
   152  	bcryptprimitivesdll = [...]uint16{'b', 'c', 'r', 'y', 'p', 't', 'p', 'r', 'i', 'm', 'i', 't', 'i', 'v', 'e', 's', '.', 'd', 'l', 'l', 0}
   153  	ntdlldll            = [...]uint16{'n', 't', 'd', 'l', 'l', '.', 'd', 'l', 'l', 0}
   154  	powrprofdll         = [...]uint16{'p', 'o', 'w', 'r', 'p', 'r', 'o', 'f', '.', 'd', 'l', 'l', 0}
   155  	winmmdll            = [...]uint16{'w', 'i', 'n', 'm', 'm', '.', 'd', 'l', 'l', 0}
   156  	realtimeapisetdll   = [...]uint16{'a', 'p', 'i', '-', 'm', 's', '-', 'w', 'i', 'n', '-', 'c', 'o', 'r', 'e', '-', 'r', 'e', 'a', 'l', 't', 'i', 'm', 'e', '-', 'l', '1', '-', '1', '-', '1', '.', 'd', 'l', 'l', 0}
   157  )
   158  
   159  // Function to be called by windows CreateThread
   160  // to start new os thread.
   161  func tstart_stdcall(newm *m)
   162  
   163  type mOS struct {
   164  	// This is here to avoid using the G stack so the stack can move during the call.
   165  	stdCallInfo windows.StdCallInfo
   166  
   167  	threadLock mutex   // protects "thread" and prevents closing
   168  	thread     uintptr // thread handle
   169  
   170  	waitsema   uintptr // semaphore for parking on locks
   171  	resumesema uintptr // semaphore to indicate suspend/resume
   172  
   173  	highResTimer   uintptr // high resolution timer handle used in usleep
   174  	waitIocpTimer  uintptr // high resolution timer handle used in netpoll
   175  	waitIocpHandle uintptr // wait completion handle used in netpoll
   176  }
   177  
   178  // Stubs so tests can link correctly. These should never be called.
   179  func open(name *byte, mode, perm int32) int32 {
   180  	throw("unimplemented")
   181  	return -1
   182  }
   183  func closefd(fd int32) int32 {
   184  	throw("unimplemented")
   185  	return -1
   186  }
   187  func read(fd int32, p unsafe.Pointer, n int32) int32 {
   188  	throw("unimplemented")
   189  	return -1
   190  }
   191  
   192  type sigset struct{}
   193  
   194  var asmstdcallAddr unsafe.Pointer
   195  
   196  type winlibcall windows.StdCallInfo
   197  
   198  func windowsFindfunc(lib uintptr, name []byte) stdFunction {
   199  	if name[len(name)-1] != 0 {
   200  		throw("usage")
   201  	}
   202  	f := stdcall(_GetProcAddress, lib, uintptr(unsafe.Pointer(&name[0])))
   203  	return stdFunction(unsafe.Pointer(f))
   204  }
   205  
   206  const _MAX_PATH = 260 // https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation
   207  var sysDirectory [_MAX_PATH + 1]byte
   208  var sysDirectoryLen uintptr
   209  
   210  func initSysDirectory() {
   211  	l := stdcall(_GetSystemDirectoryA, uintptr(unsafe.Pointer(&sysDirectory[0])), uintptr(len(sysDirectory)-1))
   212  	if l == 0 || l > uintptr(len(sysDirectory)-1) {
   213  		throw("Unable to determine system directory")
   214  	}
   215  	sysDirectory[l] = '\\'
   216  	sysDirectoryLen = l + 1
   217  }
   218  
   219  //go:linkname windows_GetSystemDirectory internal/syscall/windows.GetSystemDirectory
   220  func windows_GetSystemDirectory() string {
   221  	return unsafe.String(&sysDirectory[0], sysDirectoryLen)
   222  }
   223  
   224  func windowsLoadSystemLib(name []uint16) uintptr {
   225  	const _LOAD_LIBRARY_SEARCH_SYSTEM32 = 0x00000800
   226  	return stdcall(_LoadLibraryExW, uintptr(unsafe.Pointer(&name[0])), 0, _LOAD_LIBRARY_SEARCH_SYSTEM32)
   227  }
   228  
   229  //go:linkname windows_QueryPerformanceCounter internal/syscall/windows.QueryPerformanceCounter
   230  func windows_QueryPerformanceCounter() int64 {
   231  	var counter int64
   232  	stdcall(_QueryPerformanceCounter, uintptr(unsafe.Pointer(&counter)))
   233  	return counter
   234  }
   235  
   236  //go:linkname windows_QueryPerformanceFrequency internal/syscall/windows.QueryPerformanceFrequency
   237  func windows_QueryPerformanceFrequency() int64 {
   238  	var frequency int64
   239  	stdcall(_QueryPerformanceFrequency, uintptr(unsafe.Pointer(&frequency)))
   240  	return frequency
   241  }
   242  
   243  // nanotime1 uses a dedicated system-stack call that leaves the per-M stdcall
   244  // state and LastError untouched. Record the interrupted Go stack so profiling
   245  // samples inside the Windows clock implementation can be attributed to its caller.
   246  //
   247  //go:nosplit
   248  func nanotime1() int64 {
   249  	gp := getg()
   250  	if gp == nil || gp.m == nil || gp.m.profilehz == 0 || gp.m.libcallsp != 0 {
   251  		return nanotime2()
   252  	}
   253  	mp := gp.m
   254  	mp.libcallg.set(gp)
   255  	mp.libcallpc = sys.GetCallerPC()
   256  	// Publish the stack pointer last, as in stdcall.
   257  	mp.libcallsp = sys.GetCallerSP()
   258  	now := nanotime2()
   259  	mp.libcallsp = 0
   260  	return now
   261  }
   262  
   263  func nanotime2() int64
   264  
   265  //go:linknamestd cpu_isProcessorFeaturePresent internal/cpu.isProcessorFeaturePresent
   266  func cpu_isProcessorFeaturePresent(processorFeature uint32) bool {
   267  	ret := stdcall(_IsProcessorFeaturePresent, uintptr(processorFeature))
   268  	return ret != 0
   269  }
   270  
   271  func loadOptionalSyscalls() {
   272  	// QueryInterruptTimePrecise is available on all supported Windows versions,
   273  	// but it is not exported from kernel32.dll on every architecture.
   274  	// Resolve it before schedinit calls nanotime to initialize the clocks.
   275  	realtime := windowsLoadSystemLib(realtimeapisetdll[:])
   276  	if realtime == 0 {
   277  		throw("could not load realtime API set")
   278  	}
   279  	_QueryInterruptTimePrecise = windowsFindfunc(realtime, []byte("QueryInterruptTimePrecise\000"))
   280  	if _QueryInterruptTimePrecise == nil {
   281  		throw("QueryInterruptTimePrecise not found")
   282  	}
   283  
   284  	bcryptPrimitives := windowsLoadSystemLib(bcryptprimitivesdll[:])
   285  	if bcryptPrimitives == 0 {
   286  		throw("bcryptprimitives.dll not found")
   287  	}
   288  	_ProcessPrng = windowsFindfunc(bcryptPrimitives, []byte("ProcessPrng\000"))
   289  
   290  	n32 := windowsLoadSystemLib(ntdlldll[:])
   291  	if n32 == 0 {
   292  		throw("ntdll.dll not found")
   293  	}
   294  	_NtCreateWaitCompletionPacket = windowsFindfunc(n32, []byte("NtCreateWaitCompletionPacket\000"))
   295  	if _NtCreateWaitCompletionPacket != nil {
   296  		// These functions should exists if NtCreateWaitCompletionPacket exists.
   297  		_NtAssociateWaitCompletionPacket = windowsFindfunc(n32, []byte("NtAssociateWaitCompletionPacket\000"))
   298  		if _NtAssociateWaitCompletionPacket == nil {
   299  			throw("NtCreateWaitCompletionPacket exists but NtAssociateWaitCompletionPacket does not")
   300  		}
   301  		_NtCancelWaitCompletionPacket = windowsFindfunc(n32, []byte("NtCancelWaitCompletionPacket\000"))
   302  		if _NtCancelWaitCompletionPacket == nil {
   303  			throw("NtCreateWaitCompletionPacket exists but NtCancelWaitCompletionPacket does not")
   304  		}
   305  	}
   306  	_RtlGetCurrentPeb = windowsFindfunc(n32, []byte("RtlGetCurrentPeb\000"))
   307  	_RtlGetVersion = windowsFindfunc(n32, []byte("RtlGetVersion\000"))
   308  }
   309  
   310  func monitorSuspendResume() {
   311  	const (
   312  		_DEVICE_NOTIFY_CALLBACK = 2
   313  	)
   314  	type _DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS struct {
   315  		callback uintptr
   316  		context  uintptr
   317  	}
   318  
   319  	powrprof := windowsLoadSystemLib(powrprofdll[:])
   320  	if powrprof == 0 {
   321  		return // Running on Windows 7, where we don't need it anyway.
   322  	}
   323  	powerRegisterSuspendResumeNotification := windowsFindfunc(powrprof, []byte("PowerRegisterSuspendResumeNotification\000"))
   324  	if powerRegisterSuspendResumeNotification == nil {
   325  		return // Running on Windows 7, where we don't need it anyway.
   326  	}
   327  	var fn any = func(context uintptr, changeType uint32, setting uintptr) uintptr {
   328  		for mp := (*m)(atomic.Loadp(unsafe.Pointer(&allm))); mp != nil; mp = mp.alllink {
   329  			if mp.resumesema != 0 {
   330  				stdcall(_SetEvent, mp.resumesema)
   331  			}
   332  		}
   333  		return 0
   334  	}
   335  	params := _DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS{
   336  		callback: compileCallback(*efaceOf(&fn), true),
   337  	}
   338  	handle := uintptr(0)
   339  	stdcall(powerRegisterSuspendResumeNotification, _DEVICE_NOTIFY_CALLBACK,
   340  		uintptr(unsafe.Pointer(&params)), uintptr(unsafe.Pointer(&handle)))
   341  }
   342  
   343  func getCPUCount() int32 {
   344  	var mask, sysmask uintptr
   345  	ret := stdcall(_GetProcessAffinityMask, windows.CurrentProcess, uintptr(unsafe.Pointer(&mask)), uintptr(unsafe.Pointer(&sysmask)))
   346  	if ret != 0 {
   347  		n := 0
   348  		maskbits := int(unsafe.Sizeof(mask) * 8)
   349  		for i := 0; i < maskbits; i++ {
   350  			if mask&(1<<uint(i)) != 0 {
   351  				n++
   352  			}
   353  		}
   354  		if n != 0 {
   355  			return int32(n)
   356  		}
   357  	}
   358  	// use GetSystemInfo if GetProcessAffinityMask fails
   359  	var info windows.SystemInfo
   360  	stdcall(_GetSystemInfo, uintptr(unsafe.Pointer(&info)))
   361  	return int32(info.NumberOfProcessors)
   362  }
   363  
   364  func getPageSize() uintptr {
   365  	var info windows.SystemInfo
   366  	stdcall(_GetSystemInfo, uintptr(unsafe.Pointer(&info)))
   367  	return uintptr(info.PageSize)
   368  }
   369  
   370  // in sys_windows_386.s and sys_windows_amd64.s:
   371  func getlasterror() uint32
   372  
   373  var timeBeginPeriodRetValue uint32
   374  
   375  // osRelaxMinNS indicates that sysmon shouldn't osRelax if the next
   376  // timer is less than 60 ms from now. Since osRelaxing may reduce
   377  // timer resolution to 15.6 ms, this keeps timer error under roughly 1
   378  // part in 4.
   379  const osRelaxMinNS = 60 * 1e6
   380  
   381  // osRelax is called by the scheduler when transitioning to and from
   382  // all Ps being idle.
   383  //
   384  // Some versions of Windows have high resolution timer. For those
   385  // versions osRelax is noop.
   386  // For Windows versions without high resolution timer, osRelax
   387  // adjusts the system-wide timer resolution. Go needs a
   388  // high resolution timer while running and there's little extra cost
   389  // if we're already using the CPU, but if all Ps are idle there's no
   390  // need to consume extra power to drive the high-res timer.
   391  func osRelax(relax bool) uint32 {
   392  	if haveHighResTimer {
   393  		// If the high resolution timer is available, the runtime uses the timer
   394  		// to sleep for short durations. This means there's no need to adjust
   395  		// the global clock frequency.
   396  		return 0
   397  	}
   398  
   399  	if relax {
   400  		return uint32(stdcall(_timeEndPeriod, 1))
   401  	} else {
   402  		return uint32(stdcall(_timeBeginPeriod, 1))
   403  	}
   404  }
   405  
   406  // haveHighResTimer indicates that the CreateWaitableTimerEx
   407  // CREATE_WAITABLE_TIMER_HIGH_RESOLUTION flag is available.
   408  var haveHighResTimer = false
   409  
   410  // haveHighResSleep indicates that NtCreateWaitCompletionPacket
   411  // exists and haveHighResTimer is true.
   412  // NtCreateWaitCompletionPacket has been available since Windows 10,
   413  // but has just been publicly documented, so some platforms, like Wine,
   414  // doesn't support it yet.
   415  var haveHighResSleep = false
   416  
   417  // createHighResTimer calls CreateWaitableTimerEx with
   418  // CREATE_WAITABLE_TIMER_HIGH_RESOLUTION flag to create high
   419  // resolution timer. createHighResTimer returns new timer
   420  // handle or 0, if CreateWaitableTimerEx failed.
   421  func createHighResTimer() uintptr {
   422  	// As per @jstarks, see
   423  	// https://github.com/golang/go/issues/8687#issuecomment-656259353
   424  	return stdcall(_CreateWaitableTimerExW, 0, 0,
   425  		windows.CREATE_WAITABLE_TIMER_HIGH_RESOLUTION,
   426  		windows.SYNCHRONIZE|windows.TIMER_QUERY_STATE|windows.TIMER_MODIFY_STATE)
   427  }
   428  
   429  func initHighResTimer() {
   430  	h := createHighResTimer()
   431  	if h != 0 {
   432  		haveHighResTimer = true
   433  		haveHighResSleep = _NtCreateWaitCompletionPacket != nil
   434  		stdcall(_CloseHandle, h)
   435  	} else {
   436  		// Only load winmm.dll if we need it.
   437  		// This avoids a dependency on winmm.dll for Go programs
   438  		// that run on new Windows versions.
   439  		m32 := windowsLoadSystemLib(winmmdll[:])
   440  		if m32 == 0 {
   441  			print("runtime: LoadLibraryExW failed; errno=", getlasterror(), "\n")
   442  			throw("winmm.dll not found")
   443  		}
   444  		_timeBeginPeriod = windowsFindfunc(m32, []byte("timeBeginPeriod\000"))
   445  		_timeEndPeriod = windowsFindfunc(m32, []byte("timeEndPeriod\000"))
   446  		if _timeBeginPeriod == nil || _timeEndPeriod == nil {
   447  			print("runtime: GetProcAddress failed; errno=", getlasterror(), "\n")
   448  			throw("timeBegin/EndPeriod not found")
   449  		}
   450  	}
   451  }
   452  
   453  //go:linkname canUseLongPaths internal/syscall/windows.CanUseLongPaths
   454  var canUseLongPaths bool
   455  
   456  // initLongPathSupport enables long path support.
   457  func initLongPathSupport() {
   458  	const (
   459  		IsLongPathAwareProcess = 0x80
   460  		PebBitFieldOffset      = 3
   461  	)
   462  
   463  	// Check that we're ≥ 10.0.15063.
   464  	info := windows.OSVERSIONINFOW{}
   465  	info.OSVersionInfoSize = uint32(unsafe.Sizeof(info))
   466  	stdcall(_RtlGetVersion, uintptr(unsafe.Pointer(&info)))
   467  	if info.MajorVersion < 10 || (info.MajorVersion == 10 && info.MinorVersion == 0 && info.BuildNumber < 15063) {
   468  		return
   469  	}
   470  
   471  	// Set the IsLongPathAwareProcess flag of the PEB's bit field.
   472  	// This flag is not documented, but it's known to be used
   473  	// by Windows to enable long path support.
   474  	bitField := (*byte)(unsafe.Pointer(stdcall(_RtlGetCurrentPeb) + PebBitFieldOffset))
   475  	*bitField |= IsLongPathAwareProcess
   476  
   477  	canUseLongPaths = true
   478  }
   479  
   480  func osinit() {
   481  	asmstdcallAddr = unsafe.Pointer(windows.AsmStdCallAddr())
   482  
   483  	loadOptionalSyscalls()
   484  
   485  	preventErrorDialogs()
   486  
   487  	initExceptionHandler()
   488  
   489  	initHighResTimer()
   490  	timeBeginPeriodRetValue = osRelax(false)
   491  
   492  	initSysDirectory()
   493  	initLongPathSupport()
   494  
   495  	numCPUStartup = getCPUCount()
   496  
   497  	physPageSize = getPageSize()
   498  
   499  	// Windows dynamic priority boosting assumes that a process has different types
   500  	// of dedicated threads -- GUI, IO, computational, etc. Go processes use
   501  	// equivalent threads that all do a mix of GUI, IO, computations, etc.
   502  	// In such context dynamic priority boosting does nothing but harm, so we turn it off.
   503  	stdcall(_SetProcessPriorityBoost, windows.CurrentProcess, 1)
   504  }
   505  
   506  //go:nosplit
   507  func readRandom(r []byte) int {
   508  	n := 0
   509  	if stdcall(_ProcessPrng, uintptr(unsafe.Pointer(&r[0])), uintptr(len(r)))&0xff != 0 {
   510  		n = len(r)
   511  	}
   512  	return n
   513  }
   514  
   515  func goenvs() {
   516  	// strings is a pointer to environment variable pairs in the form:
   517  	//     "envA=valA\x00envB=valB\x00\x00" (in UTF-16)
   518  	// Two consecutive zero bytes end the list.
   519  	strings := unsafe.Pointer(stdcall(_GetEnvironmentStringsW))
   520  	p := (*[1 << 24]uint16)(strings)[:]
   521  
   522  	n := 0
   523  	for from, i := 0, 0; true; i++ {
   524  		if p[i] == 0 {
   525  			// empty string marks the end
   526  			if i == from {
   527  				break
   528  			}
   529  			from = i + 1
   530  			n++
   531  		}
   532  	}
   533  	envs = make([]string, n)
   534  
   535  	for i := range envs {
   536  		envs[i] = gostringw(&p[0])
   537  		for p[0] != 0 {
   538  			p = p[1:]
   539  		}
   540  		p = p[1:] // skip nil byte
   541  	}
   542  
   543  	stdcall(_FreeEnvironmentStringsW, uintptr(strings))
   544  
   545  	// We call these all the way here, late in init, so that malloc works
   546  	// for the callback functions these generate.
   547  	var fn any = ctrlHandler
   548  	ctrlHandlerPC := compileCallback(*efaceOf(&fn), true)
   549  	stdcall(_SetConsoleCtrlHandler, ctrlHandlerPC, 1)
   550  
   551  	monitorSuspendResume()
   552  }
   553  
   554  // exiting is set to non-zero when the process is exiting.
   555  var exiting uint32
   556  
   557  //go:nosplit
   558  func exit(code int32) {
   559  	// Disallow thread suspension for preemption. Otherwise,
   560  	// ExitProcess and SuspendThread can race: SuspendThread
   561  	// queues a suspension request for this thread, ExitProcess
   562  	// kills the suspending thread, and then this thread suspends.
   563  	lock(&suspendLock)
   564  	atomic.Store(&exiting, 1)
   565  	stdcall(_ExitProcess, uintptr(code))
   566  }
   567  
   568  // write1 must be nosplit because it's used as a last resort in
   569  // functions like badmorestackg0. In such cases, we'll always take the
   570  // ASCII path.
   571  //
   572  //go:nosplit
   573  func write1(fd uintptr, buf unsafe.Pointer, n int32) int32 {
   574  	const (
   575  		_STD_OUTPUT_HANDLE = ^uintptr(10) // -11
   576  		_STD_ERROR_HANDLE  = ^uintptr(11) // -12
   577  	)
   578  	var handle uintptr
   579  	switch fd {
   580  	case 1:
   581  		handle = stdcall(_GetStdHandle, _STD_OUTPUT_HANDLE)
   582  	case 2:
   583  		handle = stdcall(_GetStdHandle, _STD_ERROR_HANDLE)
   584  	default:
   585  		// assume fd is real windows handle.
   586  		handle = fd
   587  	}
   588  	isASCII := true
   589  	b := (*[1 << 30]byte)(buf)[:n]
   590  	for _, x := range b {
   591  		if x >= 0x80 {
   592  			isASCII = false
   593  			break
   594  		}
   595  	}
   596  
   597  	if !isASCII {
   598  		var m uint32
   599  		isConsole := stdcall(_GetConsoleMode, handle, uintptr(unsafe.Pointer(&m))) != 0
   600  		// If this is a console output, various non-unicode code pages can be in use.
   601  		// Use the dedicated WriteConsole call to ensure unicode is printed correctly.
   602  		if isConsole {
   603  			return int32(writeConsole(handle, buf, n))
   604  		}
   605  	}
   606  	var written uint32
   607  	stdcall(_WriteFile, handle, uintptr(buf), uintptr(n), uintptr(unsafe.Pointer(&written)), 0)
   608  	return int32(written)
   609  }
   610  
   611  var (
   612  	utf16ConsoleBack     [1000]uint16
   613  	utf16ConsoleBackLock mutex
   614  )
   615  
   616  // writeConsole writes bufLen bytes from buf to the console File.
   617  // It returns the number of bytes written.
   618  func writeConsole(handle uintptr, buf unsafe.Pointer, bufLen int32) int {
   619  	const surr2 = (surrogateMin + surrogateMax + 1) / 2
   620  
   621  	// Do not use defer for unlock. May cause issues when printing a panic.
   622  	lock(&utf16ConsoleBackLock)
   623  
   624  	b := (*[1 << 30]byte)(buf)[:bufLen]
   625  	s := *(*string)(unsafe.Pointer(&b))
   626  
   627  	utf16tmp := utf16ConsoleBack[:]
   628  
   629  	total := len(s)
   630  	w := 0
   631  	for _, r := range s {
   632  		if w >= len(utf16tmp)-2 {
   633  			writeConsoleUTF16(handle, utf16tmp[:w])
   634  			w = 0
   635  		}
   636  		if r < 0x10000 {
   637  			utf16tmp[w] = uint16(r)
   638  			w++
   639  		} else {
   640  			r -= 0x10000
   641  			utf16tmp[w] = surrogateMin + uint16(r>>10)&0x3ff
   642  			utf16tmp[w+1] = surr2 + uint16(r)&0x3ff
   643  			w += 2
   644  		}
   645  	}
   646  	writeConsoleUTF16(handle, utf16tmp[:w])
   647  	unlock(&utf16ConsoleBackLock)
   648  	return total
   649  }
   650  
   651  // writeConsoleUTF16 is the dedicated windows calls that correctly prints
   652  // to the console regardless of the current code page. Input is utf-16 code points.
   653  // The handle must be a console handle.
   654  func writeConsoleUTF16(handle uintptr, b []uint16) {
   655  	l := uint32(len(b))
   656  	if l == 0 {
   657  		return
   658  	}
   659  	var written uint32
   660  	stdcall(_WriteConsoleW,
   661  		handle,
   662  		uintptr(unsafe.Pointer(&b[0])),
   663  		uintptr(l),
   664  		uintptr(unsafe.Pointer(&written)),
   665  		0,
   666  	)
   667  	return
   668  }
   669  
   670  //go:nosplit
   671  func semasleep(ns int64) int32 {
   672  	const (
   673  		_WAIT_ABANDONED = 0x00000080
   674  		_WAIT_OBJECT_0  = 0x00000000
   675  		_WAIT_TIMEOUT   = 0x00000102
   676  		_WAIT_FAILED    = 0xFFFFFFFF
   677  	)
   678  
   679  	var result uintptr
   680  	if ns < 0 {
   681  		result = stdcall(_WaitForSingleObject, getg().m.waitsema, uintptr(windows.INFINITE))
   682  	} else {
   683  		start := nanotime()
   684  		elapsed := int64(0)
   685  		for {
   686  			ms := (ns - elapsed) / 1000000
   687  			if ms == 0 {
   688  				ms = 1
   689  			}
   690  			result = stdcall(_WaitForMultipleObjects, 2,
   691  				uintptr(unsafe.Pointer(&[2]uintptr{getg().m.waitsema, getg().m.resumesema})),
   692  				0, uintptr(ms))
   693  			if result != _WAIT_OBJECT_0+1 {
   694  				// Not a suspend/resume event
   695  				break
   696  			}
   697  			elapsed = nanotime() - start
   698  			if elapsed >= ns {
   699  				return -1
   700  			}
   701  		}
   702  	}
   703  	switch result {
   704  	case _WAIT_OBJECT_0: // Signaled
   705  		return 0
   706  
   707  	case _WAIT_TIMEOUT:
   708  		return -1
   709  
   710  	case _WAIT_ABANDONED:
   711  		systemstack(func() {
   712  			throw("runtime.semasleep wait_abandoned")
   713  		})
   714  
   715  	case _WAIT_FAILED:
   716  		systemstack(func() {
   717  			print("runtime: waitforsingleobject wait_failed; errno=", getlasterror(), "\n")
   718  			throw("runtime.semasleep wait_failed")
   719  		})
   720  
   721  	default:
   722  		systemstack(func() {
   723  			print("runtime: waitforsingleobject unexpected; result=", result, "\n")
   724  			throw("runtime.semasleep unexpected")
   725  		})
   726  	}
   727  
   728  	return -1 // unreachable
   729  }
   730  
   731  //go:nosplit
   732  func semawakeup(mp *m) {
   733  	if stdcall(_SetEvent, mp.waitsema) == 0 {
   734  		systemstack(func() {
   735  			print("runtime: setevent failed; errno=", getlasterror(), "\n")
   736  			throw("runtime.semawakeup")
   737  		})
   738  	}
   739  }
   740  
   741  //go:nosplit
   742  func semacreate(mp *m) {
   743  	if mp.waitsema != 0 {
   744  		return
   745  	}
   746  	mp.waitsema = stdcall(_CreateEventA, 0, 0, 0, 0)
   747  	if mp.waitsema == 0 {
   748  		systemstack(func() {
   749  			print("runtime: createevent failed; errno=", getlasterror(), "\n")
   750  			throw("runtime.semacreate")
   751  		})
   752  	}
   753  	mp.resumesema = stdcall(_CreateEventA, 0, 0, 0, 0)
   754  	if mp.resumesema == 0 {
   755  		systemstack(func() {
   756  			print("runtime: createevent failed; errno=", getlasterror(), "\n")
   757  			throw("runtime.semacreate")
   758  		})
   759  		stdcall(_CloseHandle, mp.waitsema)
   760  		mp.waitsema = 0
   761  	}
   762  }
   763  
   764  // May run with m.p==nil, so write barriers are not allowed.
   765  //
   766  //go:nowritebarrierrec
   767  func newosproc(mp *m) {
   768  	// LockOSThread can reach newosproc on a goroutine stack when starting
   769  	// the template thread. createThread does not switch stacks itself.
   770  	var thandle uintptr
   771  	var err uint32
   772  	systemstack(func() {
   773  		thandle, err = createThread(0, unsafe.Pointer(abi.FuncPCABI0(tstart_stdcall)), unsafe.Pointer(mp))
   774  	})
   775  	if thandle == 0 {
   776  		if atomic.Load(&exiting) != 0 {
   777  			// CreateThread may fail if called
   778  			// concurrently with ExitProcess. If this
   779  			// happens, just freeze this thread and let
   780  			// the process exit. See issue #18253.
   781  			lock(&deadlock)
   782  			lock(&deadlock)
   783  		}
   784  		print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", err, ")\n")
   785  		throw("runtime.newosproc")
   786  	}
   787  
   788  	// Close thandle to avoid leaking the thread object if it exits.
   789  	stdcall(_CloseHandle, thandle)
   790  }
   791  
   792  // poll_runtime_startThread starts fn on a new OS thread, locked for the entire
   793  // lifetime of fn. The thread never runs another goroutine and exits when fn
   794  // returns. fn must not call UnlockOSThread.
   795  //
   796  // Unlike a Windows callback, this can run before package initialization has
   797  // finished. Unlike go followed by LockOSThread, it cannot acquire an existing
   798  // thread with outstanding I/O from an earlier goroutine.
   799  //
   800  //go:linkname poll_runtime_startThread internal/poll.runtime_startThread
   801  func poll_runtime_startThread(fn func()) {
   802  	// A locked worker may need the template thread to replace its M when it
   803  	// parks. Set this up before publishing a worker, as LockOSThread does.
   804  	if atomic.Load(&newmHandoff.haveTemplateThread) == 0 {
   805  		startTemplateThread()
   806  	}
   807  	gp := getg()
   808  	pc := sys.GetCallerPC()
   809  	systemstack(func() {
   810  		mp := acquirem()
   811  		worker := newproc1(*(**funcval)(unsafe.Pointer(&fn)), gp, pc, false, waitReasonZero)
   812  		// A shared worker must not retain its creator's profiling labels.
   813  		worker.labels = nil
   814  		thread := allocm(nil, pollStartThread, -1)
   815  		thread.syncIOWorker = true
   816  		thread.sigmask = initSigmask
   817  		thread.lockedExt = 1
   818  		thread.lockedg.set(worker)
   819  		worker.lockedm.set(thread)
   820  		newm1(thread)
   821  		releasem(mp)
   822  	})
   823  }
   824  
   825  // poll_runtime_threadHandle lends the current worker's thread handle to poll.
   826  // minit initialized it before the worker started. The worker is permanently
   827  // locked to its thread, so unminit cannot close the handle until it returns.
   828  //
   829  //go:linkname poll_runtime_threadHandle internal/poll.runtime_threadHandle
   830  //go:nosplit
   831  func poll_runtime_threadHandle() uintptr {
   832  	gp := getg()
   833  	if !gp.m.syncIOWorker || gp.lockedm.ptr() != gp.m {
   834  		throw("runtime: not a synchronous I/O worker")
   835  	}
   836  	return gp.m.thread
   837  }
   838  
   839  // pollStartThread starts without a P. Publish the locked goroutine and wait
   840  // for the scheduler to hand a P to this thread, as in stoplockedm.
   841  func pollStartThread() {
   842  	mp := getg().m
   843  	lock(&sched.lock)
   844  	// No checkdead is needed: this balances the increase in mcount from
   845  	// allocating this M, so there is no net decrease in running Ms.
   846  	sched.nmidlelocked++
   847  	globrunqput(mp.lockedg.ptr())
   848  	startm(nil, false, true)
   849  	unlock(&sched.lock)
   850  	mPark()
   851  	acquirep(mp.nextp.ptr())
   852  	mp.nextp = 0
   853  	execute(mp.lockedg.ptr(), false)
   854  }
   855  
   856  // Used by the C library build mode. On Linux this function would allocate a
   857  // stack, but that's not necessary for Windows. No stack guards are present
   858  // and the GC has not been initialized, so write barriers will fail.
   859  //
   860  //go:nowritebarrierrec
   861  //go:nosplit
   862  func newosproc0(stacksize uintptr, fn unsafe.Pointer) {
   863  	thandle, err := createThread(stacksize, fn, nil)
   864  	if thandle == 0 {
   865  		print("runtime: failed to create new OS thread (errno=", err, ")\n")
   866  		throw("runtime: failed to create new OS thread\n")
   867  	}
   868  	stdcall_no_g(_CloseHandle, thandle)
   869  }
   870  
   871  // createThread calls CreateThread to create a new thread.
   872  //
   873  //go:nowritebarrierrec
   874  //go:nosplit
   875  func createThread(stackSize uintptr, fn unsafe.Pointer, arg unsafe.Pointer) (handle uintptr, err uint32) {
   876  	for tries := range 20 {
   877  		// We pass 0 for the stack size to use the default for this binary.
   878  		handle = stdcall_no_g(_CreateThread, 0, stackSize, uintptr(fn), uintptr(arg), 0, 0)
   879  		if handle == 0 {
   880  			err = getlasterror()
   881  		}
   882  		if handle == 0 && err == windows.ERROR_ACCESS_DENIED {
   883  			// "Insufficient resources". Yield, then back off a bit before retrying.
   884  			usleep_no_g(uint32(tries) * 1000)
   885  			continue
   886  		}
   887  		break
   888  	}
   889  	return handle, err
   890  }
   891  
   892  //go:nosplit
   893  //go:nowritebarrierrec
   894  func libpreinit() {}
   895  
   896  func exitThread(wait *atomic.Uint32) {
   897  	// We should never reach exitThread on Windows because we let
   898  	// the OS clean up threads.
   899  	throw("exitThread")
   900  }
   901  
   902  // Called to initialize a new m (including the bootstrap m).
   903  // Called on the parent thread (main thread in case of bootstrap), can allocate memory.
   904  func mpreinit(mp *m) {
   905  }
   906  
   907  //go:nosplit
   908  func sigsave(p *sigset) {
   909  }
   910  
   911  //go:nosplit
   912  func msigrestore(sigmask sigset) {
   913  }
   914  
   915  //go:nosplit
   916  //go:nowritebarrierrec
   917  func clearSignalHandlers() {
   918  }
   919  
   920  //go:nosplit
   921  func sigblock(exiting bool) {
   922  }
   923  
   924  // Called to initialize a new m (including the bootstrap m).
   925  // Called on the new thread, cannot allocate Go memory.
   926  func minit() {
   927  	var thandle uintptr
   928  	if stdcall(_DuplicateHandle, windows.CurrentProcess, windows.CurrentThread, windows.CurrentProcess, uintptr(unsafe.Pointer(&thandle)), 0, 0, windows.DUPLICATE_SAME_ACCESS) == 0 {
   929  		print("runtime.minit: duplicatehandle failed; errno=", getlasterror(), "\n")
   930  		throw("runtime.minit: duplicatehandle failed")
   931  	}
   932  
   933  	mp := getg().m
   934  	lock(&mp.threadLock)
   935  	mp.thread = thandle
   936  	mp.procid = uint64(stdcall(_GetCurrentThreadId))
   937  
   938  	// Configure usleep timer, if possible.
   939  	if mp.highResTimer == 0 && haveHighResTimer {
   940  		mp.highResTimer = createHighResTimer()
   941  		if mp.highResTimer == 0 {
   942  			print("runtime: CreateWaitableTimerEx failed; errno=", getlasterror(), "\n")
   943  			throw("CreateWaitableTimerEx when creating timer failed")
   944  		}
   945  	}
   946  	if mp.waitIocpHandle == 0 && haveHighResSleep {
   947  		mp.waitIocpTimer = createHighResTimer()
   948  		if mp.waitIocpTimer == 0 {
   949  			print("runtime: CreateWaitableTimerEx failed; errno=", getlasterror(), "\n")
   950  			throw("CreateWaitableTimerEx when creating timer failed")
   951  		}
   952  		const GENERIC_ALL = 0x10000000
   953  		errno := stdcall(_NtCreateWaitCompletionPacket, uintptr(unsafe.Pointer(&mp.waitIocpHandle)), GENERIC_ALL, 0)
   954  		if mp.waitIocpHandle == 0 {
   955  			print("runtime: NtCreateWaitCompletionPacket failed; errno=", errno, "\n")
   956  			throw("NtCreateWaitCompletionPacket failed")
   957  		}
   958  	}
   959  	unlock(&mp.threadLock)
   960  
   961  	// Query the true stack base from the OS. Currently we're
   962  	// running on a small assumed stack.
   963  	var mbi windows.MemoryBasicInformation
   964  	res := stdcall(_VirtualQuery, uintptr(unsafe.Pointer(&mbi)), uintptr(unsafe.Pointer(&mbi)), unsafe.Sizeof(mbi))
   965  	if res == 0 {
   966  		print("runtime: VirtualQuery failed; errno=", getlasterror(), "\n")
   967  		throw("VirtualQuery for stack base failed")
   968  	}
   969  	// The system leaves an 8K PAGE_GUARD region at the bottom of
   970  	// the stack (in theory VirtualQuery isn't supposed to include
   971  	// that, but it does). Add an additional 8K of slop for
   972  	// calling C functions that don't have stack checks and for
   973  	// lastcontinuehandler. We shouldn't be anywhere near this
   974  	// bound anyway.
   975  	base := mbi.AllocationBase + 16<<10
   976  	// Sanity check the stack bounds.
   977  	g0 := getg()
   978  	if base > g0.stack.hi || g0.stack.hi-base > 64<<20 {
   979  		print("runtime: g0 stack [", hex(base), ",", hex(g0.stack.hi), ")\n")
   980  		throw("bad g0 stack")
   981  	}
   982  	g0.stack.lo = base
   983  	g0.stackguard0 = g0.stack.lo + stackGuard
   984  	g0.stackguard1 = g0.stackguard0
   985  	// Sanity check the SP.
   986  	stackcheck()
   987  }
   988  
   989  // Called from dropm to undo the effect of an minit.
   990  //
   991  //go:nosplit
   992  func unminit() {
   993  	mp := getg().m
   994  	lock(&mp.threadLock)
   995  	if mp.thread != 0 {
   996  		stdcall(_CloseHandle, mp.thread)
   997  		mp.thread = 0
   998  	}
   999  	unlock(&mp.threadLock)
  1000  
  1001  	mp.procid = 0
  1002  }
  1003  
  1004  // Called from mexit, but not from dropm, to undo the effect of thread-owned
  1005  // resources in minit, semacreate, or elsewhere. Do not take locks after calling this.
  1006  //
  1007  // This always runs without a P, so //go:nowritebarrierrec is required.
  1008  //
  1009  //go:nowritebarrierrec
  1010  //go:nosplit
  1011  func mdestroy(mp *m) {
  1012  	if mp.highResTimer != 0 {
  1013  		stdcall(_CloseHandle, mp.highResTimer)
  1014  		mp.highResTimer = 0
  1015  	}
  1016  	if mp.waitIocpTimer != 0 {
  1017  		stdcall(_CloseHandle, mp.waitIocpTimer)
  1018  		mp.waitIocpTimer = 0
  1019  	}
  1020  	if mp.waitIocpHandle != 0 {
  1021  		stdcall(_CloseHandle, mp.waitIocpHandle)
  1022  		mp.waitIocpHandle = 0
  1023  	}
  1024  	if mp.waitsema != 0 {
  1025  		stdcall(_CloseHandle, mp.waitsema)
  1026  		mp.waitsema = 0
  1027  	}
  1028  	if mp.resumesema != 0 {
  1029  		stdcall(_CloseHandle, mp.resumesema)
  1030  		mp.resumesema = 0
  1031  	}
  1032  }
  1033  
  1034  // stdcall_no_g is like [stdcall] but can be called without a G.
  1035  //
  1036  //go:nowritebarrier
  1037  //go:nosplit
  1038  //go:uintptrkeepalive
  1039  func stdcall_no_g(fn stdFunction, args ...uintptr) uintptr {
  1040  	call := windows.StdCallInfo{
  1041  		Fn: uintptr(unsafe.Pointer(fn)),
  1042  		N:  uintptr(len(args)),
  1043  	}
  1044  	if len(args) > 0 {
  1045  		call.Args = uintptr(abi.NoEscape(unsafe.Pointer(&args[0])))
  1046  	}
  1047  	windows.StdCall(&call)
  1048  	return call.R1
  1049  }
  1050  
  1051  // stdcall calls fn with the given arguments using the stdcall calling convention.
  1052  // Must be called from the system stack.
  1053  // May run during STW, so write barriers are not allowed.
  1054  //
  1055  //go:nowritebarrier
  1056  //go:nosplit
  1057  //go:uintptrkeepalive
  1058  func stdcall(fn stdFunction, args ...uintptr) uintptr {
  1059  	gp := getg()
  1060  	mp := gp.m
  1061  	mp.stdCallInfo.Fn = uintptr(unsafe.Pointer(fn))
  1062  	mp.stdCallInfo.N = uintptr(len(args))
  1063  	if len(args) > 0 {
  1064  		mp.stdCallInfo.Args = uintptr(abi.NoEscape(unsafe.Pointer(&args[0])))
  1065  	}
  1066  	resetLibcall := false
  1067  	if mp.profilehz != 0 && mp.libcallsp == 0 {
  1068  		// leave pc/sp for cpu profiler
  1069  		mp.libcallg.set(gp)
  1070  		mp.libcallpc = sys.GetCallerPC()
  1071  		// sp must be the last, because once async cpu profiler finds
  1072  		// all three values to be non-zero, it will use them
  1073  		mp.libcallsp = sys.GetCallerSP()
  1074  		resetLibcall = true // See comment in sys_darwin.go:libcCall
  1075  	}
  1076  	asmcgocall(asmstdcallAddr, unsafe.Pointer(&mp.stdCallInfo))
  1077  	if resetLibcall {
  1078  		mp.libcallsp = 0
  1079  	}
  1080  	return mp.stdCallInfo.R1
  1081  }
  1082  
  1083  // These must run on the system stack only.
  1084  
  1085  //go:nosplit
  1086  func osyield_no_g() {
  1087  	stdcall_no_g(_SwitchToThread)
  1088  }
  1089  
  1090  //go:nosplit
  1091  func osyield() {
  1092  	systemstack(func() {
  1093  		stdcall(_SwitchToThread)
  1094  	})
  1095  }
  1096  
  1097  //go:nosplit
  1098  func usleep_no_g(us uint32) {
  1099  	timeout := uintptr(us) / 1000 // ms units
  1100  	stdcall_no_g(_WaitForSingleObject, windows.INVALID_HANDLE_VALUE, timeout)
  1101  }
  1102  
  1103  //go:nosplit
  1104  func usleep(us uint32) {
  1105  	systemstack(func() {
  1106  		var h, timeout uintptr
  1107  		// If the high-res timer is available and its handle has been allocated for this m, use it.
  1108  		// Otherwise fall back to the low-res one, which doesn't need a handle.
  1109  		if haveHighResTimer && getg().m.highResTimer != 0 {
  1110  			h = getg().m.highResTimer
  1111  			dt := -10 * int64(us) // relative sleep (negative), 100ns units
  1112  			stdcall(_SetWaitableTimer, h, uintptr(unsafe.Pointer(&dt)), 0, 0, 0, 0)
  1113  			timeout = windows.INFINITE
  1114  		} else {
  1115  			h = windows.INVALID_HANDLE_VALUE
  1116  			timeout = uintptr(us) / 1000 // ms units
  1117  		}
  1118  		stdcall(_WaitForSingleObject, h, timeout)
  1119  	})
  1120  }
  1121  
  1122  func ctrlHandler(_type uint32) uintptr {
  1123  	var s uint32
  1124  
  1125  	switch _type {
  1126  	case windows.CTRL_C_EVENT, windows.CTRL_BREAK_EVENT:
  1127  		s = windows.SIGINT
  1128  	case windows.CTRL_CLOSE_EVENT, windows.CTRL_LOGOFF_EVENT, windows.CTRL_SHUTDOWN_EVENT:
  1129  		s = windows.SIGTERM
  1130  	default:
  1131  		return 0
  1132  	}
  1133  
  1134  	if sigsend(s) {
  1135  		if s == windows.SIGTERM {
  1136  			// Windows terminates the process after this handler returns.
  1137  			// Block indefinitely to give signal handlers a chance to clean up,
  1138  			// but make sure to be properly parked first, so the rest of the
  1139  			// program can continue executing.
  1140  			block()
  1141  		}
  1142  		return 1
  1143  	}
  1144  	return 0
  1145  }
  1146  
  1147  // called from zcallback_windows_*.s to sys_windows_*.s
  1148  func callbackasm1()
  1149  
  1150  var profiletimer uintptr
  1151  
  1152  func profilem(mp *m, thread uintptr) {
  1153  	// Align Context to 16 bytes.
  1154  	var c *windows.Context
  1155  	var cbuf [unsafe.Sizeof(*c) + 15]byte
  1156  	c = (*windows.Context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15))
  1157  
  1158  	c.ContextFlags = windows.CONTEXT_CONTROL
  1159  	stdcall(_GetThreadContext, thread, uintptr(unsafe.Pointer(c)))
  1160  
  1161  	gp := gFromSP(mp, c.SP())
  1162  
  1163  	sigprof(c.PC(), c.SP(), c.LR(), gp, mp)
  1164  }
  1165  
  1166  func gFromSP(mp *m, sp uintptr) *g {
  1167  	if gp := mp.g0; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi {
  1168  		return gp
  1169  	}
  1170  	if gp := mp.gsignal; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi {
  1171  		return gp
  1172  	}
  1173  	if gp := mp.curg; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi {
  1174  		return gp
  1175  	}
  1176  	return nil
  1177  }
  1178  
  1179  func profileLoop() {
  1180  	stdcall(_SetThreadPriority, windows.CurrentThread, windows.THREAD_PRIORITY_HIGHEST)
  1181  
  1182  	for {
  1183  		stdcall(_WaitForSingleObject, profiletimer, windows.INFINITE)
  1184  		first := (*m)(atomic.Loadp(unsafe.Pointer(&allm)))
  1185  		for mp := first; mp != nil; mp = mp.alllink {
  1186  			if mp == getg().m {
  1187  				// Don't profile ourselves.
  1188  				continue
  1189  			}
  1190  
  1191  			lock(&mp.threadLock)
  1192  			// Do not profile threads blocked on Notes,
  1193  			// this includes idle worker threads,
  1194  			// idle timer thread, idle heap scavenger, etc.
  1195  			if mp.thread == 0 || mp.profilehz == 0 || mp.blocked {
  1196  				unlock(&mp.threadLock)
  1197  				continue
  1198  			}
  1199  			// Acquire our own handle to the thread.
  1200  			var thread uintptr
  1201  			if stdcall(_DuplicateHandle, windows.CurrentProcess, mp.thread, windows.CurrentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, windows.DUPLICATE_SAME_ACCESS) == 0 {
  1202  				print("runtime: duplicatehandle failed; errno=", getlasterror(), "\n")
  1203  				throw("duplicatehandle failed")
  1204  			}
  1205  			unlock(&mp.threadLock)
  1206  
  1207  			// mp may exit between the DuplicateHandle
  1208  			// above and the SuspendThread. The handle
  1209  			// will remain valid, but SuspendThread may
  1210  			// fail.
  1211  			if int32(stdcall(_SuspendThread, thread)) == -1 {
  1212  				// The thread no longer exists.
  1213  				stdcall(_CloseHandle, thread)
  1214  				continue
  1215  			}
  1216  			if mp.profilehz != 0 && !mp.blocked {
  1217  				// Pass the thread handle in case mp
  1218  				// was in the process of shutting down.
  1219  				profilem(mp, thread)
  1220  			}
  1221  			stdcall(_ResumeThread, thread)
  1222  			stdcall(_CloseHandle, thread)
  1223  		}
  1224  	}
  1225  }
  1226  
  1227  func setProcessCPUProfiler(hz int32) {
  1228  	if profiletimer == 0 {
  1229  		var timer uintptr
  1230  		if haveHighResTimer {
  1231  			timer = createHighResTimer()
  1232  			if timer != 0 && !profileTimerSupportsPeriod(timer) {
  1233  				// Some Windows builds (10 1803/1809, Server 2019) create
  1234  				// CREATE_WAITABLE_TIMER_HIGH_RESOLUTION timers but reject
  1235  				// a periodic SetWaitableTimer on them with
  1236  				// ERROR_INVALID_PARAMETER. setThreadCPUProfiler needs a
  1237  				// periodic timer; without the fallback profileLoop never
  1238  				// wakes and CPU profiles contain no samples.
  1239  				stdcall(_CloseHandle, timer)
  1240  				timer = 0
  1241  			}
  1242  		}
  1243  		if timer == 0 {
  1244  			timer = stdcall(_CreateWaitableTimerA, 0, 0, 0)
  1245  		}
  1246  		atomic.Storeuintptr(&profiletimer, timer)
  1247  		newm(profileLoop, nil, -1)
  1248  	}
  1249  }
  1250  
  1251  // profileTimerSupportsPeriod reports whether timer accepts a periodic
  1252  // SetWaitableTimer, which the CPU profiler relies on. On success the
  1253  // timer is left armed with a 1ms period; setThreadCPUProfiler re-arms
  1254  // it with the real period right after.
  1255  func profileTimerSupportsPeriod(timer uintptr) bool {
  1256  	due := int64(-10000) // 1ms, relative
  1257  	return stdcall(_SetWaitableTimer, timer, uintptr(unsafe.Pointer(&due)), 1, 0, 0, 0) != 0
  1258  }
  1259  
  1260  func setThreadCPUProfiler(hz int32) {
  1261  	ms := int32(0)
  1262  	due := ^int64(^uint64(1 << 63))
  1263  	if hz > 0 {
  1264  		ms = 1000 / hz
  1265  		if ms == 0 {
  1266  			ms = 1
  1267  		}
  1268  		due = int64(ms) * -10000
  1269  	}
  1270  	stdcall(_SetWaitableTimer, profiletimer, uintptr(unsafe.Pointer(&due)), uintptr(ms), 0, 0, 0)
  1271  	atomic.Store((*uint32)(unsafe.Pointer(&getg().m.profilehz)), uint32(hz))
  1272  }
  1273  
  1274  const preemptMSupported = true
  1275  
  1276  // suspendLock protects simultaneous SuspendThread operations from
  1277  // suspending each other.
  1278  var suspendLock mutex
  1279  
  1280  // threadPausedLockRank warns the runtime against acquiring new locks on this
  1281  // thread while it has another thread paused. The holder of this lock rank is
  1282  // effectively a signal handler for a paused thread, which may be holding
  1283  // arbitrary locks.
  1284  //
  1285  // The warning takes two concrete forms: First, unlock2 will see that this
  1286  // thread is holding other locks, so won't try to flush the mutex contention
  1287  // buffer (which can involve acquiring another lock). Second, builds with
  1288  // GOEXPERIMENT=staticlockranking (currently broken, https://go.dev/issue/76058)
  1289  // will report a rank violation.
  1290  //
  1291  // suspendLock serializes the transition into "suspended", but it's fine for
  1292  // multiple threads to be in the suspended state concurrently, with each of
  1293  // their sponsors independently using threadPausedLockRank to discourage
  1294  // themselves from acquiring new locks.
  1295  const threadPausedLockRank = lockRankLeafRank
  1296  
  1297  // preemptM requests preemption of the M running gp.
  1298  // gp.preempt must be set, and gp must be in _Gscanrunning, with its _Gscan
  1299  // bit held by the caller.
  1300  // preemptM releases the _Gscan bit before returning.
  1301  func preemptM(gp *g) {
  1302  	mp := gp.m
  1303  	if mp == getg().m {
  1304  		throw("self-preempt")
  1305  	}
  1306  
  1307  	// Keep gp in _Gscanrunning so it can't enter _Gsyscall and external
  1308  	// code before SuspendThread takes effect. External code may call
  1309  	// ExitProcess, which can otherwise kill this thread before the target
  1310  	// is suspended.
  1311  
  1312  	// Acquire our own handle to mp's thread.
  1313  	lock(&mp.threadLock)
  1314  	if mp.thread == 0 {
  1315  		// The M hasn't been minit'd yet (or was just unminit'd).
  1316  		unlock(&mp.threadLock)
  1317  		casfrom_Gscanstatus(gp, _Gscanrunning, _Grunning)
  1318  		mp.preemptGen.Add(1)
  1319  		return
  1320  	}
  1321  	var thread uintptr
  1322  	if stdcall(_DuplicateHandle, windows.CurrentProcess, mp.thread, windows.CurrentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, windows.DUPLICATE_SAME_ACCESS) == 0 {
  1323  		print("runtime.preemptM: duplicatehandle failed; errno=", getlasterror(), "\n")
  1324  		throw("runtime.preemptM: duplicatehandle failed")
  1325  	}
  1326  	unlock(&mp.threadLock)
  1327  
  1328  	// Prepare thread context buffer. This must be aligned to 16 bytes.
  1329  	var c *windows.Context
  1330  	var cbuf [unsafe.Sizeof(*c) + 15]byte
  1331  	c = (*windows.Context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15))
  1332  	// CONTEXT_EXCEPTION_REQUEST reports whether the thread is in the kernel.
  1333  	c.ContextFlags = windows.CONTEXT_CONTROL | windows.CONTEXT_EXCEPTION_REQUEST
  1334  
  1335  	// Serialize thread suspension. SuspendThread is asynchronous,
  1336  	// so it's otherwise possible for two threads to suspend each
  1337  	// other and deadlock. We must hold this lock until after
  1338  	// GetThreadContext, since that blocks until the thread is
  1339  	// actually suspended.
  1340  	lock(&suspendLock)
  1341  
  1342  	// Signal handler rules are in effect; don't acquire new locks.
  1343  	acquireLockRankAndM(threadPausedLockRank)
  1344  
  1345  	// Suspend the thread.
  1346  	if int32(stdcall(_SuspendThread, thread)) == -1 {
  1347  		unlock(&suspendLock)
  1348  		stdcall(_CloseHandle, thread)
  1349  		// The thread no longer exists. This shouldn't be
  1350  		// possible, but just acknowledge the request.
  1351  		mp.preemptGen.Add(1)
  1352  		releaseLockRankAndM(threadPausedLockRank)
  1353  		casfrom_Gscanstatus(gp, _Gscanrunning, _Grunning)
  1354  		return
  1355  	}
  1356  
  1357  	// We have to be very careful between this point and once
  1358  	// we've shown mp is at an async safe-point. This is like a
  1359  	// signal handler in the sense that mp could have been doing
  1360  	// anything when we stopped it, including holding arbitrary
  1361  	// locks.
  1362  
  1363  	// We have to get the thread context before inspecting the M
  1364  	// because SuspendThread only requests a suspend.
  1365  	// GetThreadContext actually blocks until it's suspended.
  1366  	stdcall(_GetThreadContext, thread, uintptr(unsafe.Pointer(c)))
  1367  
  1368  	unlock(&suspendLock)
  1369  
  1370  	// Don't redirect a thread that is in the kernel handling an exception
  1371  	// or system call; Windows may not apply the new context correctly
  1372  	// (#79249). Without CONTEXT_EXCEPTION_REPORTING we can't tell.
  1373  	safe := c.ContextFlags&windows.CONTEXT_EXCEPTION_REPORTING != 0 &&
  1374  		c.ContextFlags&(windows.CONTEXT_EXCEPTION_ACTIVE|windows.CONTEXT_SERVICE_ACTIVE) == 0
  1375  	c.ContextFlags = windows.CONTEXT_CONTROL
  1376  
  1377  	// Is it safe to preempt?
  1378  	if safe && gFromSP(mp, c.SP()) == gp {
  1379  		if ok, resumePC := isAsyncSafePoint(gp, c.PC(), c.SP(), c.LR()); ok {
  1380  			// Inject call to asyncPreempt
  1381  			targetPC := abi.FuncPCABI0(asyncPreempt)
  1382  			c.PushCall(targetPC, resumePC)
  1383  			stdcall(_SetThreadContext, thread, uintptr(unsafe.Pointer(c)))
  1384  		}
  1385  	}
  1386  
  1387  	// Acknowledge the preemption.
  1388  	mp.preemptGen.Add(1)
  1389  
  1390  	stdcall(_ResumeThread, thread)
  1391  	stdcall(_CloseHandle, thread)
  1392  
  1393  	releaseLockRankAndM(threadPausedLockRank)
  1394  	casfrom_Gscanstatus(gp, _Gscanrunning, _Grunning)
  1395  }
  1396  

View as plain text