Black Lives Matter. Support the Equal Justice Initiative.

Source file src/runtime/stubs.go

Documentation: runtime

     1  // Copyright 2014 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import "unsafe"
     8  
     9  // Should be a built-in for unsafe.Pointer?
    10  //go:nosplit
    11  func add(p unsafe.Pointer, x uintptr) unsafe.Pointer {
    12  	return unsafe.Pointer(uintptr(p) + x)
    13  }
    14  
    15  // getg returns the pointer to the current g.
    16  // The compiler rewrites calls to this function into instructions
    17  // that fetch the g directly (from TLS or from the dedicated register).
    18  func getg() *g
    19  
    20  // mcall switches from the g to the g0 stack and invokes fn(g),
    21  // where g is the goroutine that made the call.
    22  // mcall saves g's current PC/SP in g->sched so that it can be restored later.
    23  // It is up to fn to arrange for that later execution, typically by recording
    24  // g in a data structure, causing something to call ready(g) later.
    25  // mcall returns to the original goroutine g later, when g has been rescheduled.
    26  // fn must not return at all; typically it ends by calling schedule, to let the m
    27  // run other goroutines.
    28  //
    29  // mcall can only be called from g stacks (not g0, not gsignal).
    30  //
    31  // This must NOT be go:noescape: if fn is a stack-allocated closure,
    32  // fn puts g on a run queue, and g executes before fn returns, the
    33  // closure will be invalidated while it is still executing.
    34  func mcall(fn func(*g))
    35  
    36  // systemstack runs fn on a system stack.
    37  // If systemstack is called from the per-OS-thread (g0) stack, or
    38  // if systemstack is called from the signal handling (gsignal) stack,
    39  // systemstack calls fn directly and returns.
    40  // Otherwise, systemstack is being called from the limited stack
    41  // of an ordinary goroutine. In this case, systemstack switches
    42  // to the per-OS-thread stack, calls fn, and switches back.
    43  // It is common to use a func literal as the argument, in order
    44  // to share inputs and outputs with the code around the call
    45  // to system stack:
    46  //
    47  //	... set up y ...
    48  //	systemstack(func() {
    49  //		x = bigcall(y)
    50  //	})
    51  //	... use x ...
    52  //
    53  //go:noescape
    54  func systemstack(fn func())
    55  
    56  var badsystemstackMsg = "fatal: systemstack called from unexpected goroutine"
    57  
    58  //go:nosplit
    59  //go:nowritebarrierrec
    60  func badsystemstack() {
    61  	sp := stringStructOf(&badsystemstackMsg)
    62  	write(2, sp.str, int32(sp.len))
    63  }
    64  
    65  // memclrNoHeapPointers clears n bytes starting at ptr.
    66  //
    67  // Usually you should use typedmemclr. memclrNoHeapPointers should be
    68  // used only when the caller knows that *ptr contains no heap pointers
    69  // because either:
    70  //
    71  // *ptr is initialized memory and its type is pointer-free, or
    72  //
    73  // *ptr is uninitialized memory (e.g., memory that's being reused
    74  // for a new allocation) and hence contains only "junk".
    75  //
    76  // memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n
    77  // is a multiple of the pointer size, then any pointer-aligned,
    78  // pointer-sized portion is cleared atomically. Despite the function
    79  // name, this is necessary because this function is the underlying
    80  // implementation of typedmemclr and memclrHasPointers. See the doc of
    81  // memmove for more details.
    82  //
    83  // The (CPU-specific) implementations of this function are in memclr_*.s.
    84  //
    85  //go:noescape
    86  func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)
    87  
    88  //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers
    89  func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) {
    90  	memclrNoHeapPointers(ptr, n)
    91  }
    92  
    93  // memmove copies n bytes from "from" to "to".
    94  //
    95  // memmove ensures that any pointer in "from" is written to "to" with
    96  // an indivisible write, so that racy reads cannot observe a
    97  // half-written pointer. This is necessary to prevent the garbage
    98  // collector from observing invalid pointers, and differs from memmove
    99  // in unmanaged languages. However, memmove is only required to do
   100  // this if "from" and "to" may contain pointers, which can only be the
   101  // case if "from", "to", and "n" are all be word-aligned.
   102  //
   103  // Implementations are in memmove_*.s.
   104  //
   105  //go:noescape
   106  func memmove(to, from unsafe.Pointer, n uintptr)
   107  
   108  //go:linkname reflect_memmove reflect.memmove
   109  func reflect_memmove(to, from unsafe.Pointer, n uintptr) {
   110  	memmove(to, from, n)
   111  }
   112  
   113  // exported value for testing
   114  var hashLoad = float32(loadFactorNum) / float32(loadFactorDen)
   115  
   116  //go:nosplit
   117  func fastrand() uint32 {
   118  	mp := getg().m
   119  	// Implement xorshift64+: 2 32-bit xorshift sequences added together.
   120  	// Shift triplet [17,7,16] was calculated as indicated in Marsaglia's
   121  	// Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf
   122  	// This generator passes the SmallCrush suite, part of TestU01 framework:
   123  	// http://simul.iro.umontreal.ca/testu01/tu01.html
   124  	s1, s0 := mp.fastrand[0], mp.fastrand[1]
   125  	s1 ^= s1 << 17
   126  	s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16
   127  	mp.fastrand[0], mp.fastrand[1] = s0, s1
   128  	return s0 + s1
   129  }
   130  
   131  //go:nosplit
   132  func fastrandn(n uint32) uint32 {
   133  	// This is similar to fastrand() % n, but faster.
   134  	// See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
   135  	return uint32(uint64(fastrand()) * uint64(n) >> 32)
   136  }
   137  
   138  //go:linkname sync_fastrand sync.fastrand
   139  func sync_fastrand() uint32 { return fastrand() }
   140  
   141  //go:linkname net_fastrand net.fastrand
   142  func net_fastrand() uint32 { return fastrand() }
   143  
   144  //go:linkname os_fastrand os.fastrand
   145  func os_fastrand() uint32 { return fastrand() }
   146  
   147  // in internal/bytealg/equal_*.s
   148  //go:noescape
   149  func memequal(a, b unsafe.Pointer, size uintptr) bool
   150  
   151  // noescape hides a pointer from escape analysis.  noescape is
   152  // the identity function but escape analysis doesn't think the
   153  // output depends on the input.  noescape is inlined and currently
   154  // compiles down to zero instructions.
   155  // USE CAREFULLY!
   156  //go:nosplit
   157  func noescape(p unsafe.Pointer) unsafe.Pointer {
   158  	x := uintptr(p)
   159  	return unsafe.Pointer(x ^ 0)
   160  }
   161  
   162  // Not all cgocallback frames are actually cgocallback,
   163  // so not all have these arguments. Mark them uintptr so that the GC
   164  // does not misinterpret memory when the arguments are not present.
   165  // cgocallback is not called from Go, only from crosscall2.
   166  // This in turn calls cgocallbackg, which is where we'll find
   167  // pointer-declared arguments.
   168  func cgocallback(fn, frame, ctxt uintptr)
   169  func gogo(buf *gobuf)
   170  func gosave(buf *gobuf)
   171  
   172  //go:noescape
   173  func jmpdefer(fv *funcval, argp uintptr)
   174  func asminit()
   175  func setg(gg *g)
   176  func breakpoint()
   177  
   178  // reflectcall calls fn with a copy of the n argument bytes pointed at by arg.
   179  // After fn returns, reflectcall copies n-retoffset result bytes
   180  // back into arg+retoffset before returning. If copying result bytes back,
   181  // the caller should pass the argument frame type as argtype, so that
   182  // call can execute appropriate write barriers during the copy.
   183  //
   184  // Package reflect always passes a frame type. In package runtime,
   185  // Windows callbacks are the only use of this that copies results
   186  // back, and those cannot have pointers in their results, so runtime
   187  // passes nil for the frame type.
   188  //
   189  // Package reflect accesses this symbol through a linkname.
   190  func reflectcall(argtype *_type, fn, arg unsafe.Pointer, argsize uint32, retoffset uint32)
   191  
   192  func procyield(cycles uint32)
   193  
   194  type neverCallThisFunction struct{}
   195  
   196  // goexit is the return stub at the top of every goroutine call stack.
   197  // Each goroutine stack is constructed as if goexit called the
   198  // goroutine's entry point function, so that when the entry point
   199  // function returns, it will return to goexit, which will call goexit1
   200  // to perform the actual exit.
   201  //
   202  // This function must never be called directly. Call goexit1 instead.
   203  // gentraceback assumes that goexit terminates the stack. A direct
   204  // call on the stack will cause gentraceback to stop walking the stack
   205  // prematurely and if there is leftover state it may panic.
   206  func goexit(neverCallThisFunction)
   207  
   208  // publicationBarrier performs a store/store barrier (a "publication"
   209  // or "export" barrier). Some form of synchronization is required
   210  // between initializing an object and making that object accessible to
   211  // another processor. Without synchronization, the initialization
   212  // writes and the "publication" write may be reordered, allowing the
   213  // other processor to follow the pointer and observe an uninitialized
   214  // object. In general, higher-level synchronization should be used,
   215  // such as locking or an atomic pointer write. publicationBarrier is
   216  // for when those aren't an option, such as in the implementation of
   217  // the memory manager.
   218  //
   219  // There's no corresponding barrier for the read side because the read
   220  // side naturally has a data dependency order. All architectures that
   221  // Go supports or seems likely to ever support automatically enforce
   222  // data dependency ordering.
   223  func publicationBarrier()
   224  
   225  // getcallerpc returns the program counter (PC) of its caller's caller.
   226  // getcallersp returns the stack pointer (SP) of its caller's caller.
   227  // The implementation may be a compiler intrinsic; there is not
   228  // necessarily code implementing this on every platform.
   229  //
   230  // For example:
   231  //
   232  //	func f(arg1, arg2, arg3 int) {
   233  //		pc := getcallerpc()
   234  //		sp := getcallersp()
   235  //	}
   236  //
   237  // These two lines find the PC and SP immediately following
   238  // the call to f (where f will return).
   239  //
   240  // The call to getcallerpc and getcallersp must be done in the
   241  // frame being asked about.
   242  //
   243  // The result of getcallersp is correct at the time of the return,
   244  // but it may be invalidated by any subsequent call to a function
   245  // that might relocate the stack in order to grow or shrink it.
   246  // A general rule is that the result of getcallersp should be used
   247  // immediately and can only be passed to nosplit functions.
   248  
   249  //go:noescape
   250  func getcallerpc() uintptr
   251  
   252  //go:noescape
   253  func getcallersp() uintptr // implemented as an intrinsic on all platforms
   254  
   255  // getclosureptr returns the pointer to the current closure.
   256  // getclosureptr can only be used in an assignment statement
   257  // at the entry of a function. Moreover, go:nosplit directive
   258  // must be specified at the declaration of caller function,
   259  // so that the function prolog does not clobber the closure register.
   260  // for example:
   261  //
   262  //	//go:nosplit
   263  //	func f(arg1, arg2, arg3 int) {
   264  //		dx := getclosureptr()
   265  //	}
   266  //
   267  // The compiler rewrites calls to this function into instructions that fetch the
   268  // pointer from a well-known register (DX on x86 architecture, etc.) directly.
   269  func getclosureptr() uintptr
   270  
   271  //go:noescape
   272  func asmcgocall(fn, arg unsafe.Pointer) int32
   273  
   274  func morestack()
   275  func morestack_noctxt()
   276  func rt0_go()
   277  
   278  // return0 is a stub used to return 0 from deferproc.
   279  // It is called at the very end of deferproc to signal
   280  // the calling Go function that it should not jump
   281  // to deferreturn.
   282  // in asm_*.s
   283  func return0()
   284  
   285  // in asm_*.s
   286  // not called directly; definitions here supply type information for traceback.
   287  func call16(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   288  func call32(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   289  func call64(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   290  func call128(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   291  func call256(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   292  func call512(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   293  func call1024(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   294  func call2048(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   295  func call4096(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   296  func call8192(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   297  func call16384(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   298  func call32768(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   299  func call65536(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   300  func call131072(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   301  func call262144(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   302  func call524288(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   303  func call1048576(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   304  func call2097152(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   305  func call4194304(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   306  func call8388608(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   307  func call16777216(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   308  func call33554432(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   309  func call67108864(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   310  func call134217728(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   311  func call268435456(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   312  func call536870912(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   313  func call1073741824(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   314  
   315  func systemstack_switch()
   316  
   317  // alignUp rounds n up to a multiple of a. a must be a power of 2.
   318  func alignUp(n, a uintptr) uintptr {
   319  	return (n + a - 1) &^ (a - 1)
   320  }
   321  
   322  // alignDown rounds n down to a multiple of a. a must be a power of 2.
   323  func alignDown(n, a uintptr) uintptr {
   324  	return n &^ (a - 1)
   325  }
   326  
   327  // divRoundUp returns ceil(n / a).
   328  func divRoundUp(n, a uintptr) uintptr {
   329  	// a is generally a power of two. This will get inlined and
   330  	// the compiler will optimize the division.
   331  	return (n + a - 1) / a
   332  }
   333  
   334  // checkASM reports whether assembly runtime checks have passed.
   335  func checkASM() bool
   336  
   337  func memequal_varlen(a, b unsafe.Pointer) bool
   338  
   339  // bool2int returns 0 if x is false or 1 if x is true.
   340  func bool2int(x bool) int {
   341  	// Avoid branches. In the SSA compiler, this compiles to
   342  	// exactly what you would want it to.
   343  	return int(uint8(*(*uint8)(unsafe.Pointer(&x))))
   344  }
   345  
   346  // abort crashes the runtime in situations where even throw might not
   347  // work. In general it should do something a debugger will recognize
   348  // (e.g., an INT3 on x86). A crash in abort is recognized by the
   349  // signal handler, which will attempt to tear down the runtime
   350  // immediately.
   351  func abort()
   352  
   353  // Called from compiled code; declared for vet; do NOT call from Go.
   354  func gcWriteBarrier()
   355  func duffzero()
   356  func duffcopy()
   357  
   358  // Called from linker-generated .initarray; declared for go vet; do NOT call from Go.
   359  func addmoduledata()
   360  

View as plain text