netpoll_windows.go

     1  // Copyright 2013 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/goarch"
     9  	"internal/runtime/atomic"
    10  	"internal/runtime/syscall/windows"
    11  	"unsafe"
    12  )
    13  
    14  // Sources are used to identify the event that created an overlapped entry.
    15  // The source values are arbitrary. There is no risk of collision with user
    16  // defined values because the only way to set the key of an overlapped entry
    17  // is using the iocphandle, which is not accessible to user code.
    18  const (
    19  	netpollSourceReady = iota + 1
    20  	netpollSourceBreak
    21  	netpollSourceTimer
    22  )
    23  
    24  const (
    25  	// sourceBits is the number of bits needed to represent a source.
    26  	// 4 bits can hold 16 different sources, which is more than enough.
    27  	// It is set to a low value so the overlapped entry key can
    28  	// contain as much bits as possible for the pollDesc pointer.
    29  	sourceBits  = 4 // 4 bits can hold 16 different sources, which is more than enough.
    30  	sourceMasks = 1<<sourceBits - 1
    31  )
    32  
    33  // packNetpollKey creates a key from a source and a tag.
    34  // Bits that don't fit in the result are discarded.
    35  func packNetpollKey(source uint8, pd *pollDesc) uintptr {
    36  	// TODO: Consider combining the source with pd.fdseq to detect stale pollDescs.
    37  	if source > (1<<sourceBits)-1 {
    38  		// Also fail on 64-bit systems, even though it can hold more bits.
    39  		throw("runtime: source value is too large")
    40  	}
    41  	if goarch.PtrSize == 4 {
    42  		return uintptr(unsafe.Pointer(pd))<<sourceBits | uintptr(source)
    43  	}
    44  	return uintptr(taggedPointerPack(unsafe.Pointer(pd), uintptr(source)))
    45  }
    46  
    47  // unpackNetpollSource returns the source packed key.
    48  func unpackNetpollSource(key uintptr) uint8 {
    49  	if goarch.PtrSize == 4 {
    50  		return uint8(key & sourceMasks)
    51  	}
    52  	return uint8(taggedPointer(key).tag())
    53  }
    54  
    55  // pollOperation must be the same as beginning of internal/poll.operation.
    56  // Keep these in sync.
    57  type pollOperation struct {
    58  	// used by windows
    59  	_ windows.Overlapped
    60  	// used by netpoll
    61  	pd   *pollDesc
    62  	mode int32
    63  }
    64  
    65  // pollOperationFromOverlappedEntry returns the pollOperation contained in
    66  // e. It can return nil if the entry is not from internal/poll.
    67  // See go.dev/issue/58870
    68  func pollOperationFromOverlappedEntry(e *overlappedEntry) *pollOperation {
    69  	if e.ov == nil {
    70  		return nil
    71  	}
    72  	op := (*pollOperation)(unsafe.Pointer(e.ov))
    73  	// Check that the key matches the pollDesc pointer.
    74  	var keyMatch bool
    75  	if goarch.PtrSize == 4 {
    76  		keyMatch = e.key&^sourceMasks == uintptr(unsafe.Pointer(op.pd))<<sourceBits
    77  	} else {
    78  		keyMatch = (*pollDesc)(taggedPointer(e.key).pointer()) == op.pd
    79  	}
    80  	if !keyMatch {
    81  		return nil
    82  	}
    83  	return op
    84  }
    85  
    86  // overlappedEntry contains the information returned by a call to GetQueuedCompletionStatusEx.
    87  // https://learn.microsoft.com/en-us/windows/win32/api/minwinbase/ns-minwinbase-overlapped_entry
    88  type overlappedEntry struct {
    89  	key      uintptr
    90  	ov       *windows.Overlapped
    91  	internal uintptr
    92  	qty      uint32
    93  }
    94  
    95  var (
    96  	iocphandle uintptr = windows.INVALID_HANDLE_VALUE // completion port io handle
    97  
    98  	netpollWakeSig atomic.Uint32 // used to avoid duplicate calls of netpollBreak
    99  )
   100  
   101  func netpollinit() {
   102  	iocphandle = stdcall(_CreateIoCompletionPort, windows.INVALID_HANDLE_VALUE, 0, 0, windows.DWORD_MAX)
   103  	if iocphandle == 0 {
   104  		println("runtime: CreateIoCompletionPort failed (errno=", getlasterror(), ")")
   105  		throw("runtime: netpollinit failed")
   106  	}
   107  }
   108  
   109  func netpollIsPollDescriptor(fd uintptr) bool {
   110  	return fd == iocphandle
   111  }
   112  
   113  func netpollopen(fd uintptr, pd *pollDesc) int32 {
   114  	key := packNetpollKey(netpollSourceReady, pd)
   115  	if stdcall(_CreateIoCompletionPort, fd, iocphandle, key, 0) == 0 {
   116  		return int32(getlasterror())
   117  	}
   118  	return 0
   119  }
   120  
   121  func netpollclose(fd uintptr) int32 {
   122  	// nothing to do
   123  	return 0
   124  }
   125  
   126  func netpollarm(pd *pollDesc, mode int) {
   127  	throw("runtime: unused")
   128  }
   129  
   130  func netpollBreak() {
   131  	// Failing to cas indicates there is an in-flight wakeup, so we're done here.
   132  	if !netpollWakeSig.CompareAndSwap(0, 1) {
   133  		return
   134  	}
   135  
   136  	key := packNetpollKey(netpollSourceBreak, nil)
   137  	if stdcall(_PostQueuedCompletionStatus, iocphandle, 0, key, 0) == 0 {
   138  		println("runtime: netpoll: PostQueuedCompletionStatus failed (errno=", getlasterror(), ")")
   139  		throw("runtime: netpoll: PostQueuedCompletionStatus failed")
   140  	}
   141  }
   142  
   143  // netpoll checks for ready network connections.
   144  // Returns a list of goroutines that become runnable,
   145  // and a delta to add to netpollWaiters.
   146  // This must never return an empty list with a non-zero delta.
   147  //
   148  // delay < 0: blocks indefinitely
   149  // delay == 0: does not block, just polls
   150  // delay > 0: block for up to that many nanoseconds
   151  func netpoll(delay int64) (gList, int32) {
   152  	if iocphandle == windows.INVALID_HANDLE_VALUE {
   153  		return gList{}, 0
   154  	}
   155  
   156  	var entries [64]overlappedEntry
   157  	var wait uint32
   158  	var toRun gList
   159  	mp := getg().m
   160  
   161  	if delay >= 1e15 {
   162  		// An arbitrary cap on how long to wait for a timer.
   163  		// 1e15 ns == ~11.5 days.
   164  		delay = 1e15
   165  	}
   166  
   167  	if delay > 0 && mp.waitIocpHandle != 0 {
   168  		// GetQueuedCompletionStatusEx doesn't use a high resolution timer internally,
   169  		// so we use a separate higher resolution timer associated with a wait completion
   170  		// packet to wake up the poller. Note that the completion packet can be delivered
   171  		// to another thread, and the Go scheduler expects netpoll to only block up to delay,
   172  		// so we still need to use a timeout with GetQueuedCompletionStatusEx.
   173  		// TODO: Improve the Go scheduler to support non-blocking timers.
   174  		signaled := netpollQueueTimer(delay)
   175  		if signaled {
   176  			// There is a small window between the SetWaitableTimer and the NtAssociateWaitCompletionPacket
   177  			// where the timer can expire. We can return immediately in this case.
   178  			return gList{}, 0
   179  		}
   180  	}
   181  	if delay < 0 {
   182  		wait = windows.INFINITE
   183  	} else if delay == 0 {
   184  		wait = 0
   185  	} else if delay < 1e6 {
   186  		wait = 1
   187  	} else {
   188  		wait = uint32(delay / 1e6)
   189  	}
   190  	n := len(entries) / int(gomaxprocs)
   191  	if n < 8 {
   192  		n = 8
   193  	}
   194  	if delay != 0 {
   195  		mp.blocked = true
   196  	}
   197  	if stdcall(_GetQueuedCompletionStatusEx, iocphandle, uintptr(unsafe.Pointer(&entries[0])), uintptr(n), uintptr(unsafe.Pointer(&n)), uintptr(wait), 0) == 0 {
   198  		mp.blocked = false
   199  		errno := getlasterror()
   200  		if errno == windows.WAIT_TIMEOUT {
   201  			return gList{}, 0
   202  		}
   203  		println("runtime: GetQueuedCompletionStatusEx failed (errno=", errno, ")")
   204  		throw("runtime: netpoll failed")
   205  	}
   206  	mp.blocked = false
   207  	delta := int32(0)
   208  	for i := 0; i < n; i++ {
   209  		e := &entries[i]
   210  		switch unpackNetpollSource(e.key) {
   211  		case netpollSourceReady:
   212  			op := pollOperationFromOverlappedEntry(e)
   213  			if op == nil {
   214  				// Entry from outside the Go runtime and internal/poll, ignore.
   215  				continue
   216  			}
   217  			// Entry from internal/poll.
   218  			mode := op.mode
   219  			if mode != 'r' && mode != 'w' {
   220  				println("runtime: GetQueuedCompletionStatusEx returned net_op with invalid mode=", mode)
   221  				throw("runtime: netpoll failed")
   222  			}
   223  			delta += netpollready(&toRun, op.pd, mode)
   224  		case netpollSourceBreak:
   225  			netpollWakeSig.Store(0)
   226  			if delay == 0 {
   227  				// Forward the notification to the blocked poller.
   228  				netpollBreak()
   229  			}
   230  		case netpollSourceTimer:
   231  			// TODO: We could avoid calling NtCancelWaitCompletionPacket for expired wait completion packets.
   232  		default:
   233  			println("runtime: GetQueuedCompletionStatusEx returned net_op with invalid key=", e.key)
   234  			throw("runtime: netpoll failed")
   235  		}
   236  	}
   237  	return toRun, delta
   238  }
   239  
   240  // netpollQueueTimer queues a timer to wake up the poller after the given delay.
   241  // It returns true if the timer expired during this call.
   242  func netpollQueueTimer(delay int64) (signaled bool) {
   243  	mp := getg().m
   244  	// A wait completion packet can only be associated with one timer at a time,
   245  	// so we need to cancel the previous one if it exists. This wouldn't be necessary
   246  	// if the poller would only be woken up by the timer, in which case the association
   247  	// would be automatically canceled, but it can also be woken up by other events,
   248  	// such as a netpollBreak, so we can get to this point with a timer that hasn't
   249  	// expired yet. In this case, the completion packet can still be picked up by
   250  	// another thread, so defer the cancellation until it is really necessary.
   251  	errno := stdcall(_NtCancelWaitCompletionPacket, mp.waitIocpHandle, 1)
   252  	switch errno {
   253  	case windows.STATUS_CANCELLED:
   254  		// STATUS_CANCELLED is returned when the associated timer has already expired,
   255  		// in which automatically cancels the wait completion packet.
   256  		fallthrough
   257  	case windows.STATUS_SUCCESS:
   258  		dt := -delay / 100 // relative sleep (negative), 100ns units
   259  		if stdcall(_SetWaitableTimer, mp.waitIocpTimer, uintptr(unsafe.Pointer(&dt)), 0, 0, 0, 0) == 0 {
   260  			println("runtime: SetWaitableTimer failed; errno=", getlasterror())
   261  			throw("runtime: netpoll failed")
   262  		}
   263  		key := packNetpollKey(netpollSourceTimer, nil)
   264  		if errno := stdcall(_NtAssociateWaitCompletionPacket, mp.waitIocpHandle, iocphandle, mp.waitIocpTimer, key, 0, 0, 0, uintptr(unsafe.Pointer(&signaled))); errno != 0 {
   265  			println("runtime: NtAssociateWaitCompletionPacket failed; errno=", errno)
   266  			throw("runtime: netpoll failed")
   267  		}
   268  	case windows.STATUS_PENDING:
   269  		// STATUS_PENDING is returned if the wait operation can't be canceled yet.
   270  		// This can happen if this thread was woken up by another event, such as a netpollBreak,
   271  		// and the timer expired just while calling NtCancelWaitCompletionPacket, in which case
   272  		// this call fails to cancel the association to avoid a race condition.
   273  		// This is a rare case, so we can just avoid using the high resolution timer this time.
   274  	default:
   275  		println("runtime: NtCancelWaitCompletionPacket failed; errno=", errno)
   276  		throw("runtime: netpoll failed")
   277  	}
   278  	return signaled
   279  }
   280
View as plain text