...
Run Format

Source file src/runtime/stubs.go

Documentation: runtime

     1  // Copyright 2014 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import "unsafe"
     8  
     9  // Should be a built-in for unsafe.Pointer?
    10  //go:nosplit
    11  func add(p unsafe.Pointer, x uintptr) unsafe.Pointer {
    12  	return unsafe.Pointer(uintptr(p) + x)
    13  }
    14  
    15  // getg returns the pointer to the current g.
    16  // The compiler rewrites calls to this function into instructions
    17  // that fetch the g directly (from TLS or from the dedicated register).
    18  func getg() *g
    19  
    20  // mcall switches from the g to the g0 stack and invokes fn(g),
    21  // where g is the goroutine that made the call.
    22  // mcall saves g's current PC/SP in g->sched so that it can be restored later.
    23  // It is up to fn to arrange for that later execution, typically by recording
    24  // g in a data structure, causing something to call ready(g) later.
    25  // mcall returns to the original goroutine g later, when g has been rescheduled.
    26  // fn must not return at all; typically it ends by calling schedule, to let the m
    27  // run other goroutines.
    28  //
    29  // mcall can only be called from g stacks (not g0, not gsignal).
    30  //
    31  // This must NOT be go:noescape: if fn is a stack-allocated closure,
    32  // fn puts g on a run queue, and g executes before fn returns, the
    33  // closure will be invalidated while it is still executing.
    34  func mcall(fn func(*g))
    35  
    36  // systemstack runs fn on a system stack.
    37  // If systemstack is called from the per-OS-thread (g0) stack, or
    38  // if systemstack is called from the signal handling (gsignal) stack,
    39  // systemstack calls fn directly and returns.
    40  // Otherwise, systemstack is being called from the limited stack
    41  // of an ordinary goroutine. In this case, systemstack switches
    42  // to the per-OS-thread stack, calls fn, and switches back.
    43  // It is common to use a func literal as the argument, in order
    44  // to share inputs and outputs with the code around the call
    45  // to system stack:
    46  //
    47  //	... set up y ...
    48  //	systemstack(func() {
    49  //		x = bigcall(y)
    50  //	})
    51  //	... use x ...
    52  //
    53  //go:noescape
    54  func systemstack(fn func())
    55  
    56  var badsystemstackMsg = "fatal: systemstack called from unexpected goroutine"
    57  
    58  //go:nosplit
    59  //go:nowritebarrierrec
    60  func badsystemstack() {
    61  	sp := stringStructOf(&badsystemstackMsg)
    62  	write(2, sp.str, int32(sp.len))
    63  }
    64  
    65  // memclrNoHeapPointers clears n bytes starting at ptr.
    66  //
    67  // Usually you should use typedmemclr. memclrNoHeapPointers should be
    68  // used only when the caller knows that *ptr contains no heap pointers
    69  // because either:
    70  //
    71  // *ptr is initialized memory and its type is pointer-free, or
    72  //
    73  // *ptr is uninitialized memory (e.g., memory that's being reused
    74  // for a new allocation) and hence contains only "junk".
    75  //
    76  // The (CPU-specific) implementations of this function are in memclr_*.s.
    77  //go:noescape
    78  func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)
    79  
    80  //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers
    81  func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) {
    82  	memclrNoHeapPointers(ptr, n)
    83  }
    84  
    85  // memmove copies n bytes from "from" to "to".
    86  // in memmove_*.s
    87  //go:noescape
    88  func memmove(to, from unsafe.Pointer, n uintptr)
    89  
    90  //go:linkname reflect_memmove reflect.memmove
    91  func reflect_memmove(to, from unsafe.Pointer, n uintptr) {
    92  	memmove(to, from, n)
    93  }
    94  
    95  // exported value for testing
    96  var hashLoad = float32(loadFactorNum) / float32(loadFactorDen)
    97  
    98  //go:nosplit
    99  func fastrand() uint32 {
   100  	mp := getg().m
   101  	// Implement xorshift64+: 2 32-bit xorshift sequences added together.
   102  	// Shift triplet [17,7,16] was calculated as indicated in Marsaglia's
   103  	// Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf
   104  	// This generator passes the SmallCrush suite, part of TestU01 framework:
   105  	// http://simul.iro.umontreal.ca/testu01/tu01.html
   106  	s1, s0 := mp.fastrand[0], mp.fastrand[1]
   107  	s1 ^= s1 << 17
   108  	s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16
   109  	mp.fastrand[0], mp.fastrand[1] = s0, s1
   110  	return s0 + s1
   111  }
   112  
   113  //go:nosplit
   114  func fastrandn(n uint32) uint32 {
   115  	// This is similar to fastrand() % n, but faster.
   116  	// See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
   117  	return uint32(uint64(fastrand()) * uint64(n) >> 32)
   118  }
   119  
   120  //go:linkname sync_fastrand sync.fastrand
   121  func sync_fastrand() uint32 { return fastrand() }
   122  
   123  // in asm_*.s
   124  //go:noescape
   125  func memequal(a, b unsafe.Pointer, size uintptr) bool
   126  
   127  // noescape hides a pointer from escape analysis.  noescape is
   128  // the identity function but escape analysis doesn't think the
   129  // output depends on the input.  noescape is inlined and currently
   130  // compiles down to zero instructions.
   131  // USE CAREFULLY!
   132  //go:nosplit
   133  func noescape(p unsafe.Pointer) unsafe.Pointer {
   134  	x := uintptr(p)
   135  	return unsafe.Pointer(x ^ 0)
   136  }
   137  
   138  func cgocallback(fn, frame unsafe.Pointer, framesize, ctxt uintptr)
   139  func gogo(buf *gobuf)
   140  func gosave(buf *gobuf)
   141  
   142  //go:noescape
   143  func jmpdefer(fv *funcval, argp uintptr)
   144  func asminit()
   145  func setg(gg *g)
   146  func breakpoint()
   147  
   148  // reflectcall calls fn with a copy of the n argument bytes pointed at by arg.
   149  // After fn returns, reflectcall copies n-retoffset result bytes
   150  // back into arg+retoffset before returning. If copying result bytes back,
   151  // the caller should pass the argument frame type as argtype, so that
   152  // call can execute appropriate write barriers during the copy.
   153  // Package reflect passes a frame type. In package runtime, there is only
   154  // one call that copies results back, in cgocallbackg1, and it does NOT pass a
   155  // frame type, meaning there are no write barriers invoked. See that call
   156  // site for justification.
   157  //
   158  // Package reflect accesses this symbol through a linkname.
   159  func reflectcall(argtype *_type, fn, arg unsafe.Pointer, argsize uint32, retoffset uint32)
   160  
   161  func procyield(cycles uint32)
   162  
   163  type neverCallThisFunction struct{}
   164  
   165  // goexit is the return stub at the top of every goroutine call stack.
   166  // Each goroutine stack is constructed as if goexit called the
   167  // goroutine's entry point function, so that when the entry point
   168  // function returns, it will return to goexit, which will call goexit1
   169  // to perform the actual exit.
   170  //
   171  // This function must never be called directly. Call goexit1 instead.
   172  // gentraceback assumes that goexit terminates the stack. A direct
   173  // call on the stack will cause gentraceback to stop walking the stack
   174  // prematurely and if there is leftover state it may panic.
   175  func goexit(neverCallThisFunction)
   176  
   177  // Not all cgocallback_gofunc frames are actually cgocallback_gofunc,
   178  // so not all have these arguments. Mark them uintptr so that the GC
   179  // does not misinterpret memory when the arguments are not present.
   180  // cgocallback_gofunc is not called from go, only from cgocallback,
   181  // so the arguments will be found via cgocallback's pointer-declared arguments.
   182  // See the assembly implementations for more details.
   183  func cgocallback_gofunc(fv, frame, framesize, ctxt uintptr)
   184  
   185  // publicationBarrier performs a store/store barrier (a "publication"
   186  // or "export" barrier). Some form of synchronization is required
   187  // between initializing an object and making that object accessible to
   188  // another processor. Without synchronization, the initialization
   189  // writes and the "publication" write may be reordered, allowing the
   190  // other processor to follow the pointer and observe an uninitialized
   191  // object. In general, higher-level synchronization should be used,
   192  // such as locking or an atomic pointer write. publicationBarrier is
   193  // for when those aren't an option, such as in the implementation of
   194  // the memory manager.
   195  //
   196  // There's no corresponding barrier for the read side because the read
   197  // side naturally has a data dependency order. All architectures that
   198  // Go supports or seems likely to ever support automatically enforce
   199  // data dependency ordering.
   200  func publicationBarrier()
   201  
   202  // getcallerpc returns the program counter (PC) of its caller's caller.
   203  // getcallersp returns the stack pointer (SP) of its caller's caller.
   204  // The implementation may be a compiler intrinsic; there is not
   205  // necessarily code implementing this on every platform.
   206  //
   207  // For example:
   208  //
   209  //	func f(arg1, arg2, arg3 int) {
   210  //		pc := getcallerpc()
   211  //		sp := getcallersp()
   212  //	}
   213  //
   214  // These two lines find the PC and SP immediately following
   215  // the call to f (where f will return).
   216  //
   217  // The call to getcallerpc and getcallersp must be done in the
   218  // frame being asked about.
   219  //
   220  // The result of getcallersp is correct at the time of the return,
   221  // but it may be invalidated by any subsequent call to a function
   222  // that might relocate the stack in order to grow or shrink it.
   223  // A general rule is that the result of getcallersp should be used
   224  // immediately and can only be passed to nosplit functions.
   225  
   226  //go:noescape
   227  func getcallerpc() uintptr
   228  
   229  //go:noescape
   230  func getcallersp() uintptr // implemented as an intrinsic on all platforms
   231  
   232  // getclosureptr returns the pointer to the current closure.
   233  // getclosureptr can only be used in an assignment statement
   234  // at the entry of a function. Moreover, go:nosplit directive
   235  // must be specified at the declaration of caller function,
   236  // so that the function prolog does not clobber the closure register.
   237  // for example:
   238  //
   239  //	//go:nosplit
   240  //	func f(arg1, arg2, arg3 int) {
   241  //		dx := getclosureptr()
   242  //	}
   243  //
   244  // The compiler rewrites calls to this function into instructions that fetch the
   245  // pointer from a well-known register (DX on x86 architecture, etc.) directly.
   246  func getclosureptr() uintptr
   247  
   248  //go:noescape
   249  func asmcgocall(fn, arg unsafe.Pointer) int32
   250  
   251  // argp used in Defer structs when there is no argp.
   252  const _NoArgs = ^uintptr(0)
   253  
   254  func morestack()
   255  func morestack_noctxt()
   256  func rt0_go()
   257  
   258  // return0 is a stub used to return 0 from deferproc.
   259  // It is called at the very end of deferproc to signal
   260  // the calling Go function that it should not jump
   261  // to deferreturn.
   262  // in asm_*.s
   263  func return0()
   264  
   265  // in asm_*.s
   266  // not called directly; definitions here supply type information for traceback.
   267  func call32(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   268  func call64(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   269  func call128(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   270  func call256(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   271  func call512(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   272  func call1024(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   273  func call2048(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   274  func call4096(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   275  func call8192(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   276  func call16384(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   277  func call32768(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   278  func call65536(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   279  func call131072(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   280  func call262144(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   281  func call524288(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   282  func call1048576(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   283  func call2097152(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   284  func call4194304(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   285  func call8388608(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   286  func call16777216(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   287  func call33554432(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   288  func call67108864(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   289  func call134217728(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   290  func call268435456(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   291  func call536870912(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   292  func call1073741824(typ, fn, arg unsafe.Pointer, n, retoffset uint32)
   293  
   294  func systemstack_switch()
   295  
   296  // round n up to a multiple of a.  a must be a power of 2.
   297  func round(n, a uintptr) uintptr {
   298  	return (n + a - 1) &^ (a - 1)
   299  }
   300  
   301  // checkASM reports whether assembly runtime checks have passed.
   302  func checkASM() bool
   303  
   304  func memequal_varlen(a, b unsafe.Pointer) bool
   305  
   306  // bool2int returns 0 if x is false or 1 if x is true.
   307  func bool2int(x bool) int {
   308  	// Avoid branches. In the SSA compiler, this compiles to
   309  	// exactly what you would want it to.
   310  	return int(uint8(*(*uint8)(unsafe.Pointer(&x))))
   311  }
   312  
   313  // abort crashes the runtime in situations where even throw might not
   314  // work. In general it should do something a debugger will recognize
   315  // (e.g., an INT3 on x86). A crash in abort is recognized by the
   316  // signal handler, which will attempt to tear down the runtime
   317  // immediately.
   318  func abort()
   319  

View as plain text