netpoll.go

Documentation: runtime

     1  // Copyright 2013 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build unix || (js && wasm) || windows
     6  
     7  package runtime
     8  
     9  import (
    10  	"runtime/internal/atomic"
    11  	"unsafe"
    12  )
    13  
    14  // Integrated network poller (platform-independent part).
    15  // A particular implementation (epoll/kqueue/port/AIX/Windows)
    16  // must define the following functions:
    17  //
    18  // func netpollinit()
    19  //     Initialize the poller. Only called once.
    20  //
    21  // func netpollopen(fd uintptr, pd *pollDesc) int32
    22  //     Arm edge-triggered notifications for fd. The pd argument is to pass
    23  //     back to netpollready when fd is ready. Return an errno value.
    24  //
    25  // func netpollclose(fd uintptr) int32
    26  //     Disable notifications for fd. Return an errno value.
    27  //
    28  // func netpoll(delta int64) gList
    29  //     Poll the network. If delta < 0, block indefinitely. If delta == 0,
    30  //     poll without blocking. If delta > 0, block for up to delta nanoseconds.
    31  //     Return a list of goroutines built by calling netpollready.
    32  //
    33  // func netpollBreak()
    34  //     Wake up the network poller, assumed to be blocked in netpoll.
    35  //
    36  // func netpollIsPollDescriptor(fd uintptr) bool
    37  //     Reports whether fd is a file descriptor used by the poller.
    38  
    39  // Error codes returned by runtime_pollReset and runtime_pollWait.
    40  // These must match the values in internal/poll/fd_poll_runtime.go.
    41  const (
    42  	pollNoError        = 0 // no error
    43  	pollErrClosing     = 1 // descriptor is closed
    44  	pollErrTimeout     = 2 // I/O timeout
    45  	pollErrNotPollable = 3 // general error polling descriptor
    46  )
    47  
    48  // pollDesc contains 2 binary semaphores, rg and wg, to park reader and writer
    49  // goroutines respectively. The semaphore can be in the following states:
    50  //
    51  //	pdReady - io readiness notification is pending;
    52  //	          a goroutine consumes the notification by changing the state to nil.
    53  //	pdWait - a goroutine prepares to park on the semaphore, but not yet parked;
    54  //	         the goroutine commits to park by changing the state to G pointer,
    55  //	         or, alternatively, concurrent io notification changes the state to pdReady,
    56  //	         or, alternatively, concurrent timeout/close changes the state to nil.
    57  //	G pointer - the goroutine is blocked on the semaphore;
    58  //	            io notification or timeout/close changes the state to pdReady or nil respectively
    59  //	            and unparks the goroutine.
    60  //	nil - none of the above.
    61  const (
    62  	pdReady uintptr = 1
    63  	pdWait  uintptr = 2
    64  )
    65  
    66  const pollBlockSize = 4 * 1024
    67  
    68  // Network poller descriptor.
    69  //
    70  // No heap pointers.
    71  //
    72  //go:notinheap
    73  type pollDesc struct {
    74  	link *pollDesc // in pollcache, protected by pollcache.lock
    75  	fd   uintptr   // constant for pollDesc usage lifetime
    76  
    77  	// atomicInfo holds bits from closing, rd, and wd,
    78  	// which are only ever written while holding the lock,
    79  	// summarized for use by netpollcheckerr,
    80  	// which cannot acquire the lock.
    81  	// After writing these fields under lock in a way that
    82  	// might change the summary, code must call publishInfo
    83  	// before releasing the lock.
    84  	// Code that changes fields and then calls netpollunblock
    85  	// (while still holding the lock) must call publishInfo
    86  	// before calling netpollunblock, because publishInfo is what
    87  	// stops netpollblock from blocking anew
    88  	// (by changing the result of netpollcheckerr).
    89  	// atomicInfo also holds the eventErr bit,
    90  	// recording whether a poll event on the fd got an error;
    91  	// atomicInfo is the only source of truth for that bit.
    92  	atomicInfo atomic.Uint32 // atomic pollInfo
    93  
    94  	// rg, wg are accessed atomically and hold g pointers.
    95  	// (Using atomic.Uintptr here is similar to using guintptr elsewhere.)
    96  	rg atomic.Uintptr // pdReady, pdWait, G waiting for read or nil
    97  	wg atomic.Uintptr // pdReady, pdWait, G waiting for write or nil
    98  
    99  	lock    mutex // protects the following fields
   100  	closing bool
   101  	user    uint32    // user settable cookie
   102  	rseq    uintptr   // protects from stale read timers
   103  	rt      timer     // read deadline timer (set if rt.f != nil)
   104  	rd      int64     // read deadline (a nanotime in the future, -1 when expired)
   105  	wseq    uintptr   // protects from stale write timers
   106  	wt      timer     // write deadline timer
   107  	wd      int64     // write deadline (a nanotime in the future, -1 when expired)
   108  	self    *pollDesc // storage for indirect interface. See (*pollDesc).makeArg.
   109  }
   110  
   111  // pollInfo is the bits needed by netpollcheckerr, stored atomically,
   112  // mostly duplicating state that is manipulated under lock in pollDesc.
   113  // The one exception is the pollEventErr bit, which is maintained only
   114  // in the pollInfo.
   115  type pollInfo uint32
   116  
   117  const (
   118  	pollClosing = 1 << iota
   119  	pollEventErr
   120  	pollExpiredReadDeadline
   121  	pollExpiredWriteDeadline
   122  )
   123  
   124  func (i pollInfo) closing() bool              { return i&pollClosing != 0 }
   125  func (i pollInfo) eventErr() bool             { return i&pollEventErr != 0 }
   126  func (i pollInfo) expiredReadDeadline() bool  { return i&pollExpiredReadDeadline != 0 }
   127  func (i pollInfo) expiredWriteDeadline() bool { return i&pollExpiredWriteDeadline != 0 }
   128  
   129  // info returns the pollInfo corresponding to pd.
   130  func (pd *pollDesc) info() pollInfo {
   131  	return pollInfo(pd.atomicInfo.Load())
   132  }
   133  
   134  // publishInfo updates pd.atomicInfo (returned by pd.info)
   135  // using the other values in pd.
   136  // It must be called while holding pd.lock,
   137  // and it must be called after changing anything
   138  // that might affect the info bits.
   139  // In practice this means after changing closing
   140  // or changing rd or wd from < 0 to >= 0.
   141  func (pd *pollDesc) publishInfo() {
   142  	var info uint32
   143  	if pd.closing {
   144  		info |= pollClosing
   145  	}
   146  	if pd.rd < 0 {
   147  		info |= pollExpiredReadDeadline
   148  	}
   149  	if pd.wd < 0 {
   150  		info |= pollExpiredWriteDeadline
   151  	}
   152  
   153  	// Set all of x except the pollEventErr bit.
   154  	x := pd.atomicInfo.Load()
   155  	for !pd.atomicInfo.CompareAndSwap(x, (x&pollEventErr)|info) {
   156  		x = pd.atomicInfo.Load()
   157  	}
   158  }
   159  
   160  // setEventErr sets the result of pd.info().eventErr() to b.
   161  func (pd *pollDesc) setEventErr(b bool) {
   162  	x := pd.atomicInfo.Load()
   163  	for (x&pollEventErr != 0) != b && !pd.atomicInfo.CompareAndSwap(x, x^pollEventErr) {
   164  		x = pd.atomicInfo.Load()
   165  	}
   166  }
   167  
   168  type pollCache struct {
   169  	lock  mutex
   170  	first *pollDesc
   171  	// PollDesc objects must be type-stable,
   172  	// because we can get ready notification from epoll/kqueue
   173  	// after the descriptor is closed/reused.
   174  	// Stale notifications are detected using seq variable,
   175  	// seq is incremented when deadlines are changed or descriptor is reused.
   176  }
   177  
   178  var (
   179  	netpollInitLock mutex
   180  	netpollInited   uint32
   181  
   182  	pollcache      pollCache
   183  	netpollWaiters uint32
   184  )
   185  
   186  //go:linkname poll_runtime_pollServerInit internal/poll.runtime_pollServerInit
   187  func poll_runtime_pollServerInit() {
   188  	netpollGenericInit()
   189  }
   190  
   191  func netpollGenericInit() {
   192  	if atomic.Load(&netpollInited) == 0 {
   193  		lockInit(&netpollInitLock, lockRankNetpollInit)
   194  		lock(&netpollInitLock)
   195  		if netpollInited == 0 {
   196  			netpollinit()
   197  			atomic.Store(&netpollInited, 1)
   198  		}
   199  		unlock(&netpollInitLock)
   200  	}
   201  }
   202  
   203  func netpollinited() bool {
   204  	return atomic.Load(&netpollInited) != 0
   205  }
   206  
   207  //go:linkname poll_runtime_isPollServerDescriptor internal/poll.runtime_isPollServerDescriptor
   208  
   209  // poll_runtime_isPollServerDescriptor reports whether fd is a
   210  // descriptor being used by netpoll.
   211  func poll_runtime_isPollServerDescriptor(fd uintptr) bool {
   212  	return netpollIsPollDescriptor(fd)
   213  }
   214  
   215  //go:linkname poll_runtime_pollOpen internal/poll.runtime_pollOpen
   216  func poll_runtime_pollOpen(fd uintptr) (*pollDesc, int) {
   217  	pd := pollcache.alloc()
   218  	lock(&pd.lock)
   219  	wg := pd.wg.Load()
   220  	if wg != 0 && wg != pdReady {
   221  		throw("runtime: blocked write on free polldesc")
   222  	}
   223  	rg := pd.rg.Load()
   224  	if rg != 0 && rg != pdReady {
   225  		throw("runtime: blocked read on free polldesc")
   226  	}
   227  	pd.fd = fd
   228  	pd.closing = false
   229  	pd.setEventErr(false)
   230  	pd.rseq++
   231  	pd.rg.Store(0)
   232  	pd.rd = 0
   233  	pd.wseq++
   234  	pd.wg.Store(0)
   235  	pd.wd = 0
   236  	pd.self = pd
   237  	pd.publishInfo()
   238  	unlock(&pd.lock)
   239  
   240  	errno := netpollopen(fd, pd)
   241  	if errno != 0 {
   242  		pollcache.free(pd)
   243  		return nil, int(errno)
   244  	}
   245  	return pd, 0
   246  }
   247  
   248  //go:linkname poll_runtime_pollClose internal/poll.runtime_pollClose
   249  func poll_runtime_pollClose(pd *pollDesc) {
   250  	if !pd.closing {
   251  		throw("runtime: close polldesc w/o unblock")
   252  	}
   253  	wg := pd.wg.Load()
   254  	if wg != 0 && wg != pdReady {
   255  		throw("runtime: blocked write on closing polldesc")
   256  	}
   257  	rg := pd.rg.Load()
   258  	if rg != 0 && rg != pdReady {
   259  		throw("runtime: blocked read on closing polldesc")
   260  	}
   261  	netpollclose(pd.fd)
   262  	pollcache.free(pd)
   263  }
   264  
   265  func (c *pollCache) free(pd *pollDesc) {
   266  	lock(&c.lock)
   267  	pd.link = c.first
   268  	c.first = pd
   269  	unlock(&c.lock)
   270  }
   271  
   272  // poll_runtime_pollReset, which is internal/poll.runtime_pollReset,
   273  // prepares a descriptor for polling in mode, which is 'r' or 'w'.
   274  // This returns an error code; the codes are defined above.
   275  //
   276  //go:linkname poll_runtime_pollReset internal/poll.runtime_pollReset
   277  func poll_runtime_pollReset(pd *pollDesc, mode int) int {
   278  	errcode := netpollcheckerr(pd, int32(mode))
   279  	if errcode != pollNoError {
   280  		return errcode
   281  	}
   282  	if mode == 'r' {
   283  		pd.rg.Store(0)
   284  	} else if mode == 'w' {
   285  		pd.wg.Store(0)
   286  	}
   287  	return pollNoError
   288  }
   289  
   290  // poll_runtime_pollWait, which is internal/poll.runtime_pollWait,
   291  // waits for a descriptor to be ready for reading or writing,
   292  // according to mode, which is 'r' or 'w'.
   293  // This returns an error code; the codes are defined above.
   294  //
   295  //go:linkname poll_runtime_pollWait internal/poll.runtime_pollWait
   296  func poll_runtime_pollWait(pd *pollDesc, mode int) int {
   297  	errcode := netpollcheckerr(pd, int32(mode))
   298  	if errcode != pollNoError {
   299  		return errcode
   300  	}
   301  	// As for now only Solaris, illumos, and AIX use level-triggered IO.
   302  	if GOOS == "solaris" || GOOS == "illumos" || GOOS == "aix" {
   303  		netpollarm(pd, mode)
   304  	}
   305  	for !netpollblock(pd, int32(mode), false) {
   306  		errcode = netpollcheckerr(pd, int32(mode))
   307  		if errcode != pollNoError {
   308  			return errcode
   309  		}
   310  		// Can happen if timeout has fired and unblocked us,
   311  		// but before we had a chance to run, timeout has been reset.
   312  		// Pretend it has not happened and retry.
   313  	}
   314  	return pollNoError
   315  }
   316  
   317  //go:linkname poll_runtime_pollWaitCanceled internal/poll.runtime_pollWaitCanceled
   318  func poll_runtime_pollWaitCanceled(pd *pollDesc, mode int) {
   319  	// This function is used only on windows after a failed attempt to cancel
   320  	// a pending async IO operation. Wait for ioready, ignore closing or timeouts.
   321  	for !netpollblock(pd, int32(mode), true) {
   322  	}
   323  }
   324  
   325  //go:linkname poll_runtime_pollSetDeadline internal/poll.runtime_pollSetDeadline
   326  func poll_runtime_pollSetDeadline(pd *pollDesc, d int64, mode int) {
   327  	lock(&pd.lock)
   328  	if pd.closing {
   329  		unlock(&pd.lock)
   330  		return
   331  	}
   332  	rd0, wd0 := pd.rd, pd.wd
   333  	combo0 := rd0 > 0 && rd0 == wd0
   334  	if d > 0 {
   335  		d += nanotime()
   336  		if d <= 0 {
   337  			// If the user has a deadline in the future, but the delay calculation
   338  			// overflows, then set the deadline to the maximum possible value.
   339  			d = 1<<63 - 1
   340  		}
   341  	}
   342  	if mode == 'r' || mode == 'r'+'w' {
   343  		pd.rd = d
   344  	}
   345  	if mode == 'w' || mode == 'r'+'w' {
   346  		pd.wd = d
   347  	}
   348  	pd.publishInfo()
   349  	combo := pd.rd > 0 && pd.rd == pd.wd
   350  	rtf := netpollReadDeadline
   351  	if combo {
   352  		rtf = netpollDeadline
   353  	}
   354  	if pd.rt.f == nil {
   355  		if pd.rd > 0 {
   356  			pd.rt.f = rtf
   357  			// Copy current seq into the timer arg.
   358  			// Timer func will check the seq against current descriptor seq,
   359  			// if they differ the descriptor was reused or timers were reset.
   360  			pd.rt.arg = pd.makeArg()
   361  			pd.rt.seq = pd.rseq
   362  			resettimer(&pd.rt, pd.rd)
   363  		}
   364  	} else if pd.rd != rd0 || combo != combo0 {
   365  		pd.rseq++ // invalidate current timers
   366  		if pd.rd > 0 {
   367  			modtimer(&pd.rt, pd.rd, 0, rtf, pd.makeArg(), pd.rseq)
   368  		} else {
   369  			deltimer(&pd.rt)
   370  			pd.rt.f = nil
   371  		}
   372  	}
   373  	if pd.wt.f == nil {
   374  		if pd.wd > 0 && !combo {
   375  			pd.wt.f = netpollWriteDeadline
   376  			pd.wt.arg = pd.makeArg()
   377  			pd.wt.seq = pd.wseq
   378  			resettimer(&pd.wt, pd.wd)
   379  		}
   380  	} else if pd.wd != wd0 || combo != combo0 {
   381  		pd.wseq++ // invalidate current timers
   382  		if pd.wd > 0 && !combo {
   383  			modtimer(&pd.wt, pd.wd, 0, netpollWriteDeadline, pd.makeArg(), pd.wseq)
   384  		} else {
   385  			deltimer(&pd.wt)
   386  			pd.wt.f = nil
   387  		}
   388  	}
   389  	// If we set the new deadline in the past, unblock currently pending IO if any.
   390  	// Note that pd.publishInfo has already been called, above, immediately after modifying rd and wd.
   391  	var rg, wg *g
   392  	if pd.rd < 0 {
   393  		rg = netpollunblock(pd, 'r', false)
   394  	}
   395  	if pd.wd < 0 {
   396  		wg = netpollunblock(pd, 'w', false)
   397  	}
   398  	unlock(&pd.lock)
   399  	if rg != nil {
   400  		netpollgoready(rg, 3)
   401  	}
   402  	if wg != nil {
   403  		netpollgoready(wg, 3)
   404  	}
   405  }
   406  
   407  //go:linkname poll_runtime_pollUnblock internal/poll.runtime_pollUnblock
   408  func poll_runtime_pollUnblock(pd *pollDesc) {
   409  	lock(&pd.lock)
   410  	if pd.closing {
   411  		throw("runtime: unblock on closing polldesc")
   412  	}
   413  	pd.closing = true
   414  	pd.rseq++
   415  	pd.wseq++
   416  	var rg, wg *g
   417  	pd.publishInfo()
   418  	rg = netpollunblock(pd, 'r', false)
   419  	wg = netpollunblock(pd, 'w', false)
   420  	if pd.rt.f != nil {
   421  		deltimer(&pd.rt)
   422  		pd.rt.f = nil
   423  	}
   424  	if pd.wt.f != nil {
   425  		deltimer(&pd.wt)
   426  		pd.wt.f = nil
   427  	}
   428  	unlock(&pd.lock)
   429  	if rg != nil {
   430  		netpollgoready(rg, 3)
   431  	}
   432  	if wg != nil {
   433  		netpollgoready(wg, 3)
   434  	}
   435  }
   436  
   437  // netpollready is called by the platform-specific netpoll function.
   438  // It declares that the fd associated with pd is ready for I/O.
   439  // The toRun argument is used to build a list of goroutines to return
   440  // from netpoll. The mode argument is 'r', 'w', or 'r'+'w' to indicate
   441  // whether the fd is ready for reading or writing or both.
   442  //
   443  // This may run while the world is stopped, so write barriers are not allowed.
   444  //
   445  //go:nowritebarrier
   446  func netpollready(toRun *gList, pd *pollDesc, mode int32) {
   447  	var rg, wg *g
   448  	if mode == 'r' || mode == 'r'+'w' {
   449  		rg = netpollunblock(pd, 'r', true)
   450  	}
   451  	if mode == 'w' || mode == 'r'+'w' {
   452  		wg = netpollunblock(pd, 'w', true)
   453  	}
   454  	if rg != nil {
   455  		toRun.push(rg)
   456  	}
   457  	if wg != nil {
   458  		toRun.push(wg)
   459  	}
   460  }
   461  
   462  func netpollcheckerr(pd *pollDesc, mode int32) int {
   463  	info := pd.info()
   464  	if info.closing() {
   465  		return pollErrClosing
   466  	}
   467  	if (mode == 'r' && info.expiredReadDeadline()) || (mode == 'w' && info.expiredWriteDeadline()) {
   468  		return pollErrTimeout
   469  	}
   470  	// Report an event scanning error only on a read event.
   471  	// An error on a write event will be captured in a subsequent
   472  	// write call that is able to report a more specific error.
   473  	if mode == 'r' && info.eventErr() {
   474  		return pollErrNotPollable
   475  	}
   476  	return pollNoError
   477  }
   478  
   479  func netpollblockcommit(gp *g, gpp unsafe.Pointer) bool {
   480  	r := atomic.Casuintptr((*uintptr)(gpp), pdWait, uintptr(unsafe.Pointer(gp)))
   481  	if r {
   482  		// Bump the count of goroutines waiting for the poller.
   483  		// The scheduler uses this to decide whether to block
   484  		// waiting for the poller if there is nothing else to do.
   485  		atomic.Xadd(&netpollWaiters, 1)
   486  	}
   487  	return r
   488  }
   489  
   490  func netpollgoready(gp *g, traceskip int) {
   491  	atomic.Xadd(&netpollWaiters, -1)
   492  	goready(gp, traceskip+1)
   493  }
   494  
   495  // returns true if IO is ready, or false if timedout or closed
   496  // waitio - wait only for completed IO, ignore errors
   497  // Concurrent calls to netpollblock in the same mode are forbidden, as pollDesc
   498  // can hold only a single waiting goroutine for each mode.
   499  func netpollblock(pd *pollDesc, mode int32, waitio bool) bool {
   500  	gpp := &pd.rg
   501  	if mode == 'w' {
   502  		gpp = &pd.wg
   503  	}
   504  
   505  	// set the gpp semaphore to pdWait
   506  	for {
   507  		// Consume notification if already ready.
   508  		if gpp.CompareAndSwap(pdReady, 0) {
   509  			return true
   510  		}
   511  		if gpp.CompareAndSwap(0, pdWait) {
   512  			break
   513  		}
   514  
   515  		// Double check that this isn't corrupt; otherwise we'd loop
   516  		// forever.
   517  		if v := gpp.Load(); v != pdReady && v != 0 {
   518  			throw("runtime: double wait")
   519  		}
   520  	}
   521  
   522  	// need to recheck error states after setting gpp to pdWait
   523  	// this is necessary because runtime_pollUnblock/runtime_pollSetDeadline/deadlineimpl
   524  	// do the opposite: store to closing/rd/wd, publishInfo, load of rg/wg
   525  	if waitio || netpollcheckerr(pd, mode) == pollNoError {
   526  		gopark(netpollblockcommit, unsafe.Pointer(gpp), waitReasonIOWait, traceEvGoBlockNet, 5)
   527  	}
   528  	// be careful to not lose concurrent pdReady notification
   529  	old := gpp.Swap(0)
   530  	if old > pdWait {
   531  		throw("runtime: corrupted polldesc")
   532  	}
   533  	return old == pdReady
   534  }
   535  
   536  func netpollunblock(pd *pollDesc, mode int32, ioready bool) *g {
   537  	gpp := &pd.rg
   538  	if mode == 'w' {
   539  		gpp = &pd.wg
   540  	}
   541  
   542  	for {
   543  		old := gpp.Load()
   544  		if old == pdReady {
   545  			return nil
   546  		}
   547  		if old == 0 && !ioready {
   548  			// Only set pdReady for ioready. runtime_pollWait
   549  			// will check for timeout/cancel before waiting.
   550  			return nil
   551  		}
   552  		var new uintptr
   553  		if ioready {
   554  			new = pdReady
   555  		}
   556  		if gpp.CompareAndSwap(old, new) {
   557  			if old == pdWait {
   558  				old = 0
   559  			}
   560  			return (*g)(unsafe.Pointer(old))
   561  		}
   562  	}
   563  }
   564  
   565  func netpolldeadlineimpl(pd *pollDesc, seq uintptr, read, write bool) {
   566  	lock(&pd.lock)
   567  	// Seq arg is seq when the timer was set.
   568  	// If it's stale, ignore the timer event.
   569  	currentSeq := pd.rseq
   570  	if !read {
   571  		currentSeq = pd.wseq
   572  	}
   573  	if seq != currentSeq {
   574  		// The descriptor was reused or timers were reset.
   575  		unlock(&pd.lock)
   576  		return
   577  	}
   578  	var rg *g
   579  	if read {
   580  		if pd.rd <= 0 || pd.rt.f == nil {
   581  			throw("runtime: inconsistent read deadline")
   582  		}
   583  		pd.rd = -1
   584  		pd.publishInfo()
   585  		rg = netpollunblock(pd, 'r', false)
   586  	}
   587  	var wg *g
   588  	if write {
   589  		if pd.wd <= 0 || pd.wt.f == nil && !read {
   590  			throw("runtime: inconsistent write deadline")
   591  		}
   592  		pd.wd = -1
   593  		pd.publishInfo()
   594  		wg = netpollunblock(pd, 'w', false)
   595  	}
   596  	unlock(&pd.lock)
   597  	if rg != nil {
   598  		netpollgoready(rg, 0)
   599  	}
   600  	if wg != nil {
   601  		netpollgoready(wg, 0)
   602  	}
   603  }
   604  
   605  func netpollDeadline(arg any, seq uintptr) {
   606  	netpolldeadlineimpl(arg.(*pollDesc), seq, true, true)
   607  }
   608  
   609  func netpollReadDeadline(arg any, seq uintptr) {
   610  	netpolldeadlineimpl(arg.(*pollDesc), seq, true, false)
   611  }
   612  
   613  func netpollWriteDeadline(arg any, seq uintptr) {
   614  	netpolldeadlineimpl(arg.(*pollDesc), seq, false, true)
   615  }
   616  
   617  func (c *pollCache) alloc() *pollDesc {
   618  	lock(&c.lock)
   619  	if c.first == nil {
   620  		const pdSize = unsafe.Sizeof(pollDesc{})
   621  		n := pollBlockSize / pdSize
   622  		if n == 0 {
   623  			n = 1
   624  		}
   625  		// Must be in non-GC memory because can be referenced
   626  		// only from epoll/kqueue internals.
   627  		mem := persistentalloc(n*pdSize, 0, &memstats.other_sys)
   628  		for i := uintptr(0); i < n; i++ {
   629  			pd := (*pollDesc)(add(mem, i*pdSize))
   630  			pd.link = c.first
   631  			c.first = pd
   632  		}
   633  	}
   634  	pd := c.first
   635  	c.first = pd.link
   636  	lockInit(&pd.lock, lockRankPollDesc)
   637  	unlock(&c.lock)
   638  	return pd
   639  }
   640  
   641  // makeArg converts pd to an interface{}.
   642  // makeArg does not do any allocation. Normally, such
   643  // a conversion requires an allocation because pointers to
   644  // go:notinheap types (which pollDesc is) must be stored
   645  // in interfaces indirectly. See issue 42076.
   646  func (pd *pollDesc) makeArg() (i any) {
   647  	x := (*eface)(unsafe.Pointer(&i))
   648  	x._type = pdType
   649  	x.data = unsafe.Pointer(&pd.self)
   650  	return
   651  }
   652  
   653  var (
   654  	pdEface any    = (*pollDesc)(nil)
   655  	pdType  *_type = efaceOf(&pdEface)._type
   656  )
   657
View as plain text