Black Lives Matter. Support the Equal Justice Initiative.

Source file src/runtime/os_linux.go

Documentation: runtime

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"runtime/internal/atomic"
     9  	"runtime/internal/sys"
    10  	"unsafe"
    11  )
    12  
    13  type mOS struct{}
    14  
    15  //go:noescape
    16  func futex(addr unsafe.Pointer, op int32, val uint32, ts, addr2 unsafe.Pointer, val3 uint32) int32
    17  
    18  // Linux futex.
    19  //
    20  //	futexsleep(uint32 *addr, uint32 val)
    21  //	futexwakeup(uint32 *addr)
    22  //
    23  // Futexsleep atomically checks if *addr == val and if so, sleeps on addr.
    24  // Futexwakeup wakes up threads sleeping on addr.
    25  // Futexsleep is allowed to wake up spuriously.
    26  
    27  const (
    28  	_FUTEX_PRIVATE_FLAG = 128
    29  	_FUTEX_WAIT_PRIVATE = 0 | _FUTEX_PRIVATE_FLAG
    30  	_FUTEX_WAKE_PRIVATE = 1 | _FUTEX_PRIVATE_FLAG
    31  )
    32  
    33  // Atomically,
    34  //	if(*addr == val) sleep
    35  // Might be woken up spuriously; that's allowed.
    36  // Don't sleep longer than ns; ns < 0 means forever.
    37  //go:nosplit
    38  func futexsleep(addr *uint32, val uint32, ns int64) {
    39  	// Some Linux kernels have a bug where futex of
    40  	// FUTEX_WAIT returns an internal error code
    41  	// as an errno. Libpthread ignores the return value
    42  	// here, and so can we: as it says a few lines up,
    43  	// spurious wakeups are allowed.
    44  	if ns < 0 {
    45  		futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, nil, nil, 0)
    46  		return
    47  	}
    48  
    49  	var ts timespec
    50  	ts.setNsec(ns)
    51  	futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, unsafe.Pointer(&ts), nil, 0)
    52  }
    53  
    54  // If any procs are sleeping on addr, wake up at most cnt.
    55  //go:nosplit
    56  func futexwakeup(addr *uint32, cnt uint32) {
    57  	ret := futex(unsafe.Pointer(addr), _FUTEX_WAKE_PRIVATE, cnt, nil, nil, 0)
    58  	if ret >= 0 {
    59  		return
    60  	}
    61  
    62  	// I don't know that futex wakeup can return
    63  	// EAGAIN or EINTR, but if it does, it would be
    64  	// safe to loop and call futex again.
    65  	systemstack(func() {
    66  		print("futexwakeup addr=", addr, " returned ", ret, "\n")
    67  	})
    68  
    69  	*(*int32)(unsafe.Pointer(uintptr(0x1006))) = 0x1006
    70  }
    71  
    72  func getproccount() int32 {
    73  	// This buffer is huge (8 kB) but we are on the system stack
    74  	// and there should be plenty of space (64 kB).
    75  	// Also this is a leaf, so we're not holding up the memory for long.
    76  	// See golang.org/issue/11823.
    77  	// The suggested behavior here is to keep trying with ever-larger
    78  	// buffers, but we don't have a dynamic memory allocator at the
    79  	// moment, so that's a bit tricky and seems like overkill.
    80  	const maxCPUs = 64 * 1024
    81  	var buf [maxCPUs / 8]byte
    82  	r := sched_getaffinity(0, unsafe.Sizeof(buf), &buf[0])
    83  	if r < 0 {
    84  		return 1
    85  	}
    86  	n := int32(0)
    87  	for _, v := range buf[:r] {
    88  		for v != 0 {
    89  			n += int32(v & 1)
    90  			v >>= 1
    91  		}
    92  	}
    93  	if n == 0 {
    94  		n = 1
    95  	}
    96  	return n
    97  }
    98  
    99  // Clone, the Linux rfork.
   100  const (
   101  	_CLONE_VM             = 0x100
   102  	_CLONE_FS             = 0x200
   103  	_CLONE_FILES          = 0x400
   104  	_CLONE_SIGHAND        = 0x800
   105  	_CLONE_PTRACE         = 0x2000
   106  	_CLONE_VFORK          = 0x4000
   107  	_CLONE_PARENT         = 0x8000
   108  	_CLONE_THREAD         = 0x10000
   109  	_CLONE_NEWNS          = 0x20000
   110  	_CLONE_SYSVSEM        = 0x40000
   111  	_CLONE_SETTLS         = 0x80000
   112  	_CLONE_PARENT_SETTID  = 0x100000
   113  	_CLONE_CHILD_CLEARTID = 0x200000
   114  	_CLONE_UNTRACED       = 0x800000
   115  	_CLONE_CHILD_SETTID   = 0x1000000
   116  	_CLONE_STOPPED        = 0x2000000
   117  	_CLONE_NEWUTS         = 0x4000000
   118  	_CLONE_NEWIPC         = 0x8000000
   119  
   120  	// As of QEMU 2.8.0 (5ea2fc84d), user emulation requires all six of these
   121  	// flags to be set when creating a thread; attempts to share the other
   122  	// five but leave SYSVSEM unshared will fail with -EINVAL.
   123  	//
   124  	// In non-QEMU environments CLONE_SYSVSEM is inconsequential as we do not
   125  	// use System V semaphores.
   126  
   127  	cloneFlags = _CLONE_VM | /* share memory */
   128  		_CLONE_FS | /* share cwd, etc */
   129  		_CLONE_FILES | /* share fd table */
   130  		_CLONE_SIGHAND | /* share sig handler table */
   131  		_CLONE_SYSVSEM | /* share SysV semaphore undo lists (see issue #20763) */
   132  		_CLONE_THREAD /* revisit - okay for now */
   133  )
   134  
   135  //go:noescape
   136  func clone(flags int32, stk, mp, gp, fn unsafe.Pointer) int32
   137  
   138  // May run with m.p==nil, so write barriers are not allowed.
   139  //go:nowritebarrier
   140  func newosproc(mp *m) {
   141  	stk := unsafe.Pointer(mp.g0.stack.hi)
   142  	/*
   143  	 * note: strace gets confused if we use CLONE_PTRACE here.
   144  	 */
   145  	if false {
   146  		print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " clone=", funcPC(clone), " id=", mp.id, " ostk=", &mp, "\n")
   147  	}
   148  
   149  	// Disable signals during clone, so that the new thread starts
   150  	// with signals disabled. It will enable them in minit.
   151  	var oset sigset
   152  	sigprocmask(_SIG_SETMASK, &sigset_all, &oset)
   153  	ret := clone(cloneFlags, stk, unsafe.Pointer(mp), unsafe.Pointer(mp.g0), unsafe.Pointer(funcPC(mstart)))
   154  	sigprocmask(_SIG_SETMASK, &oset, nil)
   155  
   156  	if ret < 0 {
   157  		print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", -ret, ")\n")
   158  		if ret == -_EAGAIN {
   159  			println("runtime: may need to increase max user processes (ulimit -u)")
   160  		}
   161  		throw("newosproc")
   162  	}
   163  }
   164  
   165  // Version of newosproc that doesn't require a valid G.
   166  //go:nosplit
   167  func newosproc0(stacksize uintptr, fn unsafe.Pointer) {
   168  	stack := sysAlloc(stacksize, &memstats.stacks_sys)
   169  	if stack == nil {
   170  		write(2, unsafe.Pointer(&failallocatestack[0]), int32(len(failallocatestack)))
   171  		exit(1)
   172  	}
   173  	ret := clone(cloneFlags, unsafe.Pointer(uintptr(stack)+stacksize), nil, nil, fn)
   174  	if ret < 0 {
   175  		write(2, unsafe.Pointer(&failthreadcreate[0]), int32(len(failthreadcreate)))
   176  		exit(1)
   177  	}
   178  }
   179  
   180  var failallocatestack = []byte("runtime: failed to allocate stack for the new OS thread\n")
   181  var failthreadcreate = []byte("runtime: failed to create new OS thread\n")
   182  
   183  const (
   184  	_AT_NULL   = 0  // End of vector
   185  	_AT_PAGESZ = 6  // System physical page size
   186  	_AT_HWCAP  = 16 // hardware capability bit vector
   187  	_AT_RANDOM = 25 // introduced in 2.6.29
   188  	_AT_HWCAP2 = 26 // hardware capability bit vector 2
   189  )
   190  
   191  var procAuxv = []byte("/proc/self/auxv\x00")
   192  
   193  var addrspace_vec [1]byte
   194  
   195  func mincore(addr unsafe.Pointer, n uintptr, dst *byte) int32
   196  
   197  func sysargs(argc int32, argv **byte) {
   198  	n := argc + 1
   199  
   200  	// skip over argv, envp to get to auxv
   201  	for argv_index(argv, n) != nil {
   202  		n++
   203  	}
   204  
   205  	// skip NULL separator
   206  	n++
   207  
   208  	// now argv+n is auxv
   209  	auxv := (*[1 << 28]uintptr)(add(unsafe.Pointer(argv), uintptr(n)*sys.PtrSize))
   210  	if sysauxv(auxv[:]) != 0 {
   211  		return
   212  	}
   213  	// In some situations we don't get a loader-provided
   214  	// auxv, such as when loaded as a library on Android.
   215  	// Fall back to /proc/self/auxv.
   216  	fd := open(&procAuxv[0], 0 /* O_RDONLY */, 0)
   217  	if fd < 0 {
   218  		// On Android, /proc/self/auxv might be unreadable (issue 9229), so we fallback to
   219  		// try using mincore to detect the physical page size.
   220  		// mincore should return EINVAL when address is not a multiple of system page size.
   221  		const size = 256 << 10 // size of memory region to allocate
   222  		p, err := mmap(nil, size, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
   223  		if err != 0 {
   224  			return
   225  		}
   226  		var n uintptr
   227  		for n = 4 << 10; n < size; n <<= 1 {
   228  			err := mincore(unsafe.Pointer(uintptr(p)+n), 1, &addrspace_vec[0])
   229  			if err == 0 {
   230  				physPageSize = n
   231  				break
   232  			}
   233  		}
   234  		if physPageSize == 0 {
   235  			physPageSize = size
   236  		}
   237  		munmap(p, size)
   238  		return
   239  	}
   240  	var buf [128]uintptr
   241  	n = read(fd, noescape(unsafe.Pointer(&buf[0])), int32(unsafe.Sizeof(buf)))
   242  	closefd(fd)
   243  	if n < 0 {
   244  		return
   245  	}
   246  	// Make sure buf is terminated, even if we didn't read
   247  	// the whole file.
   248  	buf[len(buf)-2] = _AT_NULL
   249  	sysauxv(buf[:])
   250  }
   251  
   252  func sysauxv(auxv []uintptr) int {
   253  	var i int
   254  	for ; auxv[i] != _AT_NULL; i += 2 {
   255  		tag, val := auxv[i], auxv[i+1]
   256  		switch tag {
   257  		case _AT_RANDOM:
   258  			// The kernel provides a pointer to 16-bytes
   259  			// worth of random data.
   260  			startupRandomData = (*[16]byte)(unsafe.Pointer(val))[:]
   261  
   262  		case _AT_PAGESZ:
   263  			physPageSize = val
   264  		}
   265  
   266  		archauxv(tag, val)
   267  		vdsoauxv(tag, val)
   268  	}
   269  	return i / 2
   270  }
   271  
   272  var sysTHPSizePath = []byte("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size\x00")
   273  
   274  func getHugePageSize() uintptr {
   275  	var numbuf [20]byte
   276  	fd := open(&sysTHPSizePath[0], 0 /* O_RDONLY */, 0)
   277  	if fd < 0 {
   278  		return 0
   279  	}
   280  	ptr := noescape(unsafe.Pointer(&numbuf[0]))
   281  	n := read(fd, ptr, int32(len(numbuf)))
   282  	closefd(fd)
   283  	if n <= 0 {
   284  		return 0
   285  	}
   286  	n-- // remove trailing newline
   287  	v, ok := atoi(slicebytetostringtmp((*byte)(ptr), int(n)))
   288  	if !ok || v < 0 {
   289  		v = 0
   290  	}
   291  	if v&(v-1) != 0 {
   292  		// v is not a power of 2
   293  		return 0
   294  	}
   295  	return uintptr(v)
   296  }
   297  
   298  func osinit() {
   299  	ncpu = getproccount()
   300  	physHugePageSize = getHugePageSize()
   301  	osArchInit()
   302  }
   303  
   304  var urandom_dev = []byte("/dev/urandom\x00")
   305  
   306  func getRandomData(r []byte) {
   307  	if startupRandomData != nil {
   308  		n := copy(r, startupRandomData)
   309  		extendRandom(r, n)
   310  		return
   311  	}
   312  	fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0)
   313  	n := read(fd, unsafe.Pointer(&r[0]), int32(len(r)))
   314  	closefd(fd)
   315  	extendRandom(r, int(n))
   316  }
   317  
   318  func goenvs() {
   319  	goenvs_unix()
   320  }
   321  
   322  // Called to do synchronous initialization of Go code built with
   323  // -buildmode=c-archive or -buildmode=c-shared.
   324  // None of the Go runtime is initialized.
   325  //go:nosplit
   326  //go:nowritebarrierrec
   327  func libpreinit() {
   328  	initsig(true)
   329  }
   330  
   331  // gsignalInitQuirk, if non-nil, is called for every allocated gsignal G.
   332  //
   333  // TODO(austin): Remove this after Go 1.15 when we remove the
   334  // mlockGsignal workaround.
   335  var gsignalInitQuirk func(gsignal *g)
   336  
   337  // Called to initialize a new m (including the bootstrap m).
   338  // Called on the parent thread (main thread in case of bootstrap), can allocate memory.
   339  func mpreinit(mp *m) {
   340  	mp.gsignal = malg(32 * 1024) // Linux wants >= 2K
   341  	mp.gsignal.m = mp
   342  	if gsignalInitQuirk != nil {
   343  		gsignalInitQuirk(mp.gsignal)
   344  	}
   345  }
   346  
   347  func gettid() uint32
   348  
   349  // Called to initialize a new m (including the bootstrap m).
   350  // Called on the new thread, cannot allocate memory.
   351  func minit() {
   352  	minitSignals()
   353  
   354  	// Cgo-created threads and the bootstrap m are missing a
   355  	// procid. We need this for asynchronous preemption and it's
   356  	// useful in debuggers.
   357  	getg().m.procid = uint64(gettid())
   358  }
   359  
   360  // Called from dropm to undo the effect of an minit.
   361  //go:nosplit
   362  func unminit() {
   363  	unminitSignals()
   364  }
   365  
   366  //#ifdef GOARCH_386
   367  //#define sa_handler k_sa_handler
   368  //#endif
   369  
   370  func sigreturn()
   371  func sigtramp(sig uint32, info *siginfo, ctx unsafe.Pointer)
   372  func cgoSigtramp()
   373  
   374  //go:noescape
   375  func sigaltstack(new, old *stackt)
   376  
   377  //go:noescape
   378  func setitimer(mode int32, new, old *itimerval)
   379  
   380  //go:noescape
   381  func rtsigprocmask(how int32, new, old *sigset, size int32)
   382  
   383  //go:nosplit
   384  //go:nowritebarrierrec
   385  func sigprocmask(how int32, new, old *sigset) {
   386  	rtsigprocmask(how, new, old, int32(unsafe.Sizeof(*new)))
   387  }
   388  
   389  func raise(sig uint32)
   390  func raiseproc(sig uint32)
   391  
   392  //go:noescape
   393  func sched_getaffinity(pid, len uintptr, buf *byte) int32
   394  func osyield()
   395  
   396  func pipe() (r, w int32, errno int32)
   397  func pipe2(flags int32) (r, w int32, errno int32)
   398  func setNonblock(fd int32)
   399  
   400  //go:nosplit
   401  //go:nowritebarrierrec
   402  func setsig(i uint32, fn uintptr) {
   403  	var sa sigactiont
   404  	sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK | _SA_RESTORER | _SA_RESTART
   405  	sigfillset(&sa.sa_mask)
   406  	// Although Linux manpage says "sa_restorer element is obsolete and
   407  	// should not be used". x86_64 kernel requires it. Only use it on
   408  	// x86.
   409  	if GOARCH == "386" || GOARCH == "amd64" {
   410  		sa.sa_restorer = funcPC(sigreturn)
   411  	}
   412  	if fn == funcPC(sighandler) {
   413  		if iscgo {
   414  			fn = funcPC(cgoSigtramp)
   415  		} else {
   416  			fn = funcPC(sigtramp)
   417  		}
   418  	}
   419  	sa.sa_handler = fn
   420  	sigaction(i, &sa, nil)
   421  }
   422  
   423  //go:nosplit
   424  //go:nowritebarrierrec
   425  func setsigstack(i uint32) {
   426  	var sa sigactiont
   427  	sigaction(i, nil, &sa)
   428  	if sa.sa_flags&_SA_ONSTACK != 0 {
   429  		return
   430  	}
   431  	sa.sa_flags |= _SA_ONSTACK
   432  	sigaction(i, &sa, nil)
   433  }
   434  
   435  //go:nosplit
   436  //go:nowritebarrierrec
   437  func getsig(i uint32) uintptr {
   438  	var sa sigactiont
   439  	sigaction(i, nil, &sa)
   440  	return sa.sa_handler
   441  }
   442  
   443  // setSignaltstackSP sets the ss_sp field of a stackt.
   444  //go:nosplit
   445  func setSignalstackSP(s *stackt, sp uintptr) {
   446  	*(*uintptr)(unsafe.Pointer(&s.ss_sp)) = sp
   447  }
   448  
   449  //go:nosplit
   450  func (c *sigctxt) fixsigcode(sig uint32) {
   451  }
   452  
   453  // sysSigaction calls the rt_sigaction system call.
   454  //go:nosplit
   455  func sysSigaction(sig uint32, new, old *sigactiont) {
   456  	if rt_sigaction(uintptr(sig), new, old, unsafe.Sizeof(sigactiont{}.sa_mask)) != 0 {
   457  		// Workaround for bugs in QEMU user mode emulation.
   458  		//
   459  		// QEMU turns calls to the sigaction system call into
   460  		// calls to the C library sigaction call; the C
   461  		// library call rejects attempts to call sigaction for
   462  		// SIGCANCEL (32) or SIGSETXID (33).
   463  		//
   464  		// QEMU rejects calling sigaction on SIGRTMAX (64).
   465  		//
   466  		// Just ignore the error in these case. There isn't
   467  		// anything we can do about it anyhow.
   468  		if sig != 32 && sig != 33 && sig != 64 {
   469  			// Use system stack to avoid split stack overflow on ppc64/ppc64le.
   470  			systemstack(func() {
   471  				throw("sigaction failed")
   472  			})
   473  		}
   474  	}
   475  }
   476  
   477  // rt_sigaction is implemented in assembly.
   478  //go:noescape
   479  func rt_sigaction(sig uintptr, new, old *sigactiont, size uintptr) int32
   480  
   481  func getpid() int
   482  func tgkill(tgid, tid, sig int)
   483  
   484  // touchStackBeforeSignal stores an errno value. If non-zero, it means
   485  // that we should touch the signal stack before sending a signal.
   486  // This is used on systems that have a bug when the signal stack must
   487  // be faulted in.  See #35777 and #37436.
   488  //
   489  // This is accessed atomically as it is set and read in different threads.
   490  //
   491  // TODO(austin): Remove this after Go 1.15 when we remove the
   492  // mlockGsignal workaround.
   493  var touchStackBeforeSignal uint32
   494  
   495  // signalM sends a signal to mp.
   496  func signalM(mp *m, sig int) {
   497  	if atomic.Load(&touchStackBeforeSignal) != 0 {
   498  		atomic.Cas((*uint32)(unsafe.Pointer(mp.gsignal.stack.hi-4)), 0, 0)
   499  	}
   500  	tgkill(getpid(), int(mp.procid), sig)
   501  }
   502  

View as plain text