...
Run Format

Source file src/runtime/lock_futex.go

Documentation: runtime

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // +build dragonfly freebsd linux
     6  
     7  package runtime
     8  
     9  import (
    10  	"runtime/internal/atomic"
    11  	"unsafe"
    12  )
    13  
    14  // This implementation depends on OS-specific implementations of
    15  //
    16  //	futexsleep(addr *uint32, val uint32, ns int64)
    17  //		Atomically,
    18  //			if *addr == val { sleep }
    19  //		Might be woken up spuriously; that's allowed.
    20  //		Don't sleep longer than ns; ns < 0 means forever.
    21  //
    22  //	futexwakeup(addr *uint32, cnt uint32)
    23  //		If any procs are sleeping on addr, wake up at most cnt.
    24  
    25  const (
    26  	mutex_unlocked = 0
    27  	mutex_locked   = 1
    28  	mutex_sleeping = 2
    29  
    30  	active_spin     = 4
    31  	active_spin_cnt = 30
    32  	passive_spin    = 1
    33  )
    34  
    35  // Possible lock states are mutex_unlocked, mutex_locked and mutex_sleeping.
    36  // mutex_sleeping means that there is presumably at least one sleeping thread.
    37  // Note that there can be spinning threads during all states - they do not
    38  // affect mutex's state.
    39  
    40  // We use the uintptr mutex.key and note.key as a uint32.
    41  //go:nosplit
    42  func key32(p *uintptr) *uint32 {
    43  	return (*uint32)(unsafe.Pointer(p))
    44  }
    45  
    46  func lock(l *mutex) {
    47  	gp := getg()
    48  
    49  	if gp.m.locks < 0 {
    50  		throw("runtime·lock: lock count")
    51  	}
    52  	gp.m.locks++
    53  
    54  	// Speculative grab for lock.
    55  	v := atomic.Xchg(key32(&l.key), mutex_locked)
    56  	if v == mutex_unlocked {
    57  		return
    58  	}
    59  
    60  	// wait is either MUTEX_LOCKED or MUTEX_SLEEPING
    61  	// depending on whether there is a thread sleeping
    62  	// on this mutex. If we ever change l->key from
    63  	// MUTEX_SLEEPING to some other value, we must be
    64  	// careful to change it back to MUTEX_SLEEPING before
    65  	// returning, to ensure that the sleeping thread gets
    66  	// its wakeup call.
    67  	wait := v
    68  
    69  	// On uniprocessors, no point spinning.
    70  	// On multiprocessors, spin for ACTIVE_SPIN attempts.
    71  	spin := 0
    72  	if ncpu > 1 {
    73  		spin = active_spin
    74  	}
    75  	for {
    76  		// Try for lock, spinning.
    77  		for i := 0; i < spin; i++ {
    78  			for l.key == mutex_unlocked {
    79  				if atomic.Cas(key32(&l.key), mutex_unlocked, wait) {
    80  					return
    81  				}
    82  			}
    83  			procyield(active_spin_cnt)
    84  		}
    85  
    86  		// Try for lock, rescheduling.
    87  		for i := 0; i < passive_spin; i++ {
    88  			for l.key == mutex_unlocked {
    89  				if atomic.Cas(key32(&l.key), mutex_unlocked, wait) {
    90  					return
    91  				}
    92  			}
    93  			osyield()
    94  		}
    95  
    96  		// Sleep.
    97  		v = atomic.Xchg(key32(&l.key), mutex_sleeping)
    98  		if v == mutex_unlocked {
    99  			return
   100  		}
   101  		wait = mutex_sleeping
   102  		futexsleep(key32(&l.key), mutex_sleeping, -1)
   103  	}
   104  }
   105  
   106  func unlock(l *mutex) {
   107  	v := atomic.Xchg(key32(&l.key), mutex_unlocked)
   108  	if v == mutex_unlocked {
   109  		throw("unlock of unlocked lock")
   110  	}
   111  	if v == mutex_sleeping {
   112  		futexwakeup(key32(&l.key), 1)
   113  	}
   114  
   115  	gp := getg()
   116  	gp.m.locks--
   117  	if gp.m.locks < 0 {
   118  		throw("runtime·unlock: lock count")
   119  	}
   120  	if gp.m.locks == 0 && gp.preempt { // restore the preemption request in case we've cleared it in newstack
   121  		gp.stackguard0 = stackPreempt
   122  	}
   123  }
   124  
   125  // One-time notifications.
   126  func noteclear(n *note) {
   127  	n.key = 0
   128  }
   129  
   130  func notewakeup(n *note) {
   131  	old := atomic.Xchg(key32(&n.key), 1)
   132  	if old != 0 {
   133  		print("notewakeup - double wakeup (", old, ")\n")
   134  		throw("notewakeup - double wakeup")
   135  	}
   136  	futexwakeup(key32(&n.key), 1)
   137  }
   138  
   139  func notesleep(n *note) {
   140  	gp := getg()
   141  	if gp != gp.m.g0 {
   142  		throw("notesleep not on g0")
   143  	}
   144  	ns := int64(-1)
   145  	if *cgo_yield != nil {
   146  		// Sleep for an arbitrary-but-moderate interval to poll libc interceptors.
   147  		ns = 10e6
   148  	}
   149  	for atomic.Load(key32(&n.key)) == 0 {
   150  		gp.m.blocked = true
   151  		futexsleep(key32(&n.key), 0, ns)
   152  		if *cgo_yield != nil {
   153  			asmcgocall(*cgo_yield, nil)
   154  		}
   155  		gp.m.blocked = false
   156  	}
   157  }
   158  
   159  // May run with m.p==nil if called from notetsleep, so write barriers
   160  // are not allowed.
   161  //
   162  //go:nosplit
   163  //go:nowritebarrier
   164  func notetsleep_internal(n *note, ns int64) bool {
   165  	gp := getg()
   166  
   167  	if ns < 0 {
   168  		if *cgo_yield != nil {
   169  			// Sleep for an arbitrary-but-moderate interval to poll libc interceptors.
   170  			ns = 10e6
   171  		}
   172  		for atomic.Load(key32(&n.key)) == 0 {
   173  			gp.m.blocked = true
   174  			futexsleep(key32(&n.key), 0, ns)
   175  			if *cgo_yield != nil {
   176  				asmcgocall(*cgo_yield, nil)
   177  			}
   178  			gp.m.blocked = false
   179  		}
   180  		return true
   181  	}
   182  
   183  	if atomic.Load(key32(&n.key)) != 0 {
   184  		return true
   185  	}
   186  
   187  	deadline := nanotime() + ns
   188  	for {
   189  		if *cgo_yield != nil && ns > 10e6 {
   190  			ns = 10e6
   191  		}
   192  		gp.m.blocked = true
   193  		futexsleep(key32(&n.key), 0, ns)
   194  		if *cgo_yield != nil {
   195  			asmcgocall(*cgo_yield, nil)
   196  		}
   197  		gp.m.blocked = false
   198  		if atomic.Load(key32(&n.key)) != 0 {
   199  			break
   200  		}
   201  		now := nanotime()
   202  		if now >= deadline {
   203  			break
   204  		}
   205  		ns = deadline - now
   206  	}
   207  	return atomic.Load(key32(&n.key)) != 0
   208  }
   209  
   210  func notetsleep(n *note, ns int64) bool {
   211  	gp := getg()
   212  	if gp != gp.m.g0 && gp.m.preemptoff != "" {
   213  		throw("notetsleep not on g0")
   214  	}
   215  
   216  	return notetsleep_internal(n, ns)
   217  }
   218  
   219  // same as runtime·notetsleep, but called on user g (not g0)
   220  // calls only nosplit functions between entersyscallblock/exitsyscall
   221  func notetsleepg(n *note, ns int64) bool {
   222  	gp := getg()
   223  	if gp == gp.m.g0 {
   224  		throw("notetsleepg on g0")
   225  	}
   226  
   227  	entersyscallblock()
   228  	ok := notetsleep_internal(n, ns)
   229  	exitsyscall()
   230  	return ok
   231  }
   232  
   233  func pauseSchedulerUntilCallback() bool {
   234  	return false
   235  }
   236  
   237  func checkTimeouts() {}
   238  

View as plain text