Source file src/runtime/stubs.go

     1  // Copyright 2014 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/abi"
     9  	"unsafe"
    10  )
    11  
    12  // Should be a built-in for unsafe.Pointer?
    13  //
    14  //go:nosplit
    15  func add(p unsafe.Pointer, x uintptr) unsafe.Pointer {
    16  	return unsafe.Pointer(uintptr(p) + x)
    17  }
    18  
    19  // getg returns the pointer to the current g.
    20  // The compiler rewrites calls to this function into instructions
    21  // that fetch the g directly (from TLS or from the dedicated register).
    22  func getg() *g
    23  
    24  // mcall switches from the g to the g0 stack and invokes fn(g),
    25  // where g is the goroutine that made the call.
    26  // mcall saves g's current PC/SP in g->sched so that it can be restored later.
    27  // It is up to fn to arrange for that later execution, typically by recording
    28  // g in a data structure, causing something to call ready(g) later.
    29  // mcall returns to the original goroutine g later, when g has been rescheduled.
    30  // fn must not return at all; typically it ends by calling schedule, to let the m
    31  // run other goroutines.
    32  //
    33  // mcall can only be called from g stacks (not g0, not gsignal).
    34  //
    35  // This must NOT be go:noescape: if fn is a stack-allocated closure,
    36  // fn puts g on a run queue, and g executes before fn returns, the
    37  // closure will be invalidated while it is still executing.
    38  func mcall(fn func(*g))
    39  
    40  // systemstack runs fn on a system stack.
    41  // If systemstack is called from the per-OS-thread (g0) stack, or
    42  // if systemstack is called from the signal handling (gsignal) stack,
    43  // systemstack calls fn directly and returns.
    44  // Otherwise, systemstack is being called from the limited stack
    45  // of an ordinary goroutine. In this case, systemstack switches
    46  // to the per-OS-thread stack, calls fn, and switches back.
    47  // It is common to use a func literal as the argument, in order
    48  // to share inputs and outputs with the code around the call
    49  // to system stack:
    50  //
    51  //	... set up y ...
    52  //	systemstack(func() {
    53  //		x = bigcall(y)
    54  //	})
    55  //	... use x ...
    56  //
    57  //go:noescape
    58  func systemstack(fn func())
    59  
    60  //go:nosplit
    61  //go:nowritebarrierrec
    62  func badsystemstack() {
    63  	writeErrStr("fatal: systemstack called from unexpected goroutine")
    64  }
    65  
    66  // memclrNoHeapPointers clears n bytes starting at ptr.
    67  //
    68  // Usually you should use typedmemclr. memclrNoHeapPointers should be
    69  // used only when the caller knows that *ptr contains no heap pointers
    70  // because either:
    71  //
    72  // *ptr is initialized memory and its type is pointer-free, or
    73  //
    74  // *ptr is uninitialized memory (e.g., memory that's being reused
    75  // for a new allocation) and hence contains only "junk".
    76  //
    77  // memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n
    78  // is a multiple of the pointer size, then any pointer-aligned,
    79  // pointer-sized portion is cleared atomically. Despite the function
    80  // name, this is necessary because this function is the underlying
    81  // implementation of typedmemclr and memclrHasPointers. See the doc of
    82  // memmove for more details.
    83  //
    84  // The (CPU-specific) implementations of this function are in memclr_*.s.
    85  //
    86  //go:noescape
    87  func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)
    88  
    89  //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers
    90  func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) {
    91  	memclrNoHeapPointers(ptr, n)
    92  }
    93  
    94  // memmove copies n bytes from "from" to "to".
    95  //
    96  // memmove ensures that any pointer in "from" is written to "to" with
    97  // an indivisible write, so that racy reads cannot observe a
    98  // half-written pointer. This is necessary to prevent the garbage
    99  // collector from observing invalid pointers, and differs from memmove
   100  // in unmanaged languages. However, memmove is only required to do
   101  // this if "from" and "to" may contain pointers, which can only be the
   102  // case if "from", "to", and "n" are all be word-aligned.
   103  //
   104  // Implementations are in memmove_*.s.
   105  //
   106  //go:noescape
   107  func memmove(to, from unsafe.Pointer, n uintptr)
   108  
   109  // Outside assembly calls memmove. Make sure it has ABI wrappers.
   110  //
   111  //go:linkname memmove
   112  
   113  //go:linkname reflect_memmove reflect.memmove
   114  func reflect_memmove(to, from unsafe.Pointer, n uintptr) {
   115  	memmove(to, from, n)
   116  }
   117  
   118  // exported value for testing
   119  const hashLoad = float32(loadFactorNum) / float32(loadFactorDen)
   120  
   121  // in internal/bytealg/equal_*.s
   122  //
   123  //go:noescape
   124  func memequal(a, b unsafe.Pointer, size uintptr) bool
   125  
   126  // noescape hides a pointer from escape analysis.  noescape is
   127  // the identity function but escape analysis doesn't think the
   128  // output depends on the input.  noescape is inlined and currently
   129  // compiles down to zero instructions.
   130  // USE CAREFULLY!
   131  //
   132  //go:nosplit
   133  func noescape(p unsafe.Pointer) unsafe.Pointer {
   134  	x := uintptr(p)
   135  	return unsafe.Pointer(x ^ 0)
   136  }
   137  
   138  // noEscapePtr hides a pointer from escape analysis. See noescape.
   139  // USE CAREFULLY!
   140  //
   141  //go:nosplit
   142  func noEscapePtr[T any](p *T) *T {
   143  	x := uintptr(unsafe.Pointer(p))
   144  	return (*T)(unsafe.Pointer(x ^ 0))
   145  }
   146  
   147  // Not all cgocallback frames are actually cgocallback,
   148  // so not all have these arguments. Mark them uintptr so that the GC
   149  // does not misinterpret memory when the arguments are not present.
   150  // cgocallback is not called from Go, only from crosscall2.
   151  // This in turn calls cgocallbackg, which is where we'll find
   152  // pointer-declared arguments.
   153  //
   154  // When fn is nil (frame is saved g), call dropm instead,
   155  // this is used when the C thread is exiting.
   156  func cgocallback(fn, frame, ctxt uintptr)
   157  
   158  func gogo(buf *gobuf)
   159  
   160  func asminit()
   161  func setg(gg *g)
   162  func breakpoint()
   163  
   164  // reflectcall calls fn with arguments described by stackArgs, stackArgsSize,
   165  // frameSize, and regArgs.
   166  //
   167  // Arguments passed on the stack and space for return values passed on the stack
   168  // must be laid out at the space pointed to by stackArgs (with total length
   169  // stackArgsSize) according to the ABI.
   170  //
   171  // stackRetOffset must be some value <= stackArgsSize that indicates the
   172  // offset within stackArgs where the return value space begins.
   173  //
   174  // frameSize is the total size of the argument frame at stackArgs and must
   175  // therefore be >= stackArgsSize. It must include additional space for spilling
   176  // register arguments for stack growth and preemption.
   177  //
   178  // TODO(mknyszek): Once we don't need the additional spill space, remove frameSize,
   179  // since frameSize will be redundant with stackArgsSize.
   180  //
   181  // Arguments passed in registers must be laid out in regArgs according to the ABI.
   182  // regArgs will hold any return values passed in registers after the call.
   183  //
   184  // reflectcall copies stack arguments from stackArgs to the goroutine stack, and
   185  // then copies back stackArgsSize-stackRetOffset bytes back to the return space
   186  // in stackArgs once fn has completed. It also "unspills" argument registers from
   187  // regArgs before calling fn, and spills them back into regArgs immediately
   188  // following the call to fn. If there are results being returned on the stack,
   189  // the caller should pass the argument frame type as stackArgsType so that
   190  // reflectcall can execute appropriate write barriers during the copy.
   191  //
   192  // reflectcall expects regArgs.ReturnIsPtr to be populated indicating which
   193  // registers on the return path will contain Go pointers. It will then store
   194  // these pointers in regArgs.Ptrs such that they are visible to the GC.
   195  //
   196  // Package reflect passes a frame type. In package runtime, there is only
   197  // one call that copies results back, in callbackWrap in syscall_windows.go, and it
   198  // does NOT pass a frame type, meaning there are no write barriers invoked. See that
   199  // call site for justification.
   200  //
   201  // Package reflect accesses this symbol through a linkname.
   202  //
   203  // Arguments passed through to reflectcall do not escape. The type is used
   204  // only in a very limited callee of reflectcall, the stackArgs are copied, and
   205  // regArgs is only used in the reflectcall frame.
   206  //
   207  //go:noescape
   208  func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   209  
   210  func procyield(cycles uint32)
   211  
   212  type neverCallThisFunction struct{}
   213  
   214  // goexit is the return stub at the top of every goroutine call stack.
   215  // Each goroutine stack is constructed as if goexit called the
   216  // goroutine's entry point function, so that when the entry point
   217  // function returns, it will return to goexit, which will call goexit1
   218  // to perform the actual exit.
   219  //
   220  // This function must never be called directly. Call goexit1 instead.
   221  // gentraceback assumes that goexit terminates the stack. A direct
   222  // call on the stack will cause gentraceback to stop walking the stack
   223  // prematurely and if there is leftover state it may panic.
   224  func goexit(neverCallThisFunction)
   225  
   226  // publicationBarrier performs a store/store barrier (a "publication"
   227  // or "export" barrier). Some form of synchronization is required
   228  // between initializing an object and making that object accessible to
   229  // another processor. Without synchronization, the initialization
   230  // writes and the "publication" write may be reordered, allowing the
   231  // other processor to follow the pointer and observe an uninitialized
   232  // object. In general, higher-level synchronization should be used,
   233  // such as locking or an atomic pointer write. publicationBarrier is
   234  // for when those aren't an option, such as in the implementation of
   235  // the memory manager.
   236  //
   237  // There's no corresponding barrier for the read side because the read
   238  // side naturally has a data dependency order. All architectures that
   239  // Go supports or seems likely to ever support automatically enforce
   240  // data dependency ordering.
   241  func publicationBarrier()
   242  
   243  // getcallerpc returns the program counter (PC) of its caller's caller.
   244  // getcallersp returns the stack pointer (SP) of its caller's caller.
   245  // The implementation may be a compiler intrinsic; there is not
   246  // necessarily code implementing this on every platform.
   247  //
   248  // For example:
   249  //
   250  //	func f(arg1, arg2, arg3 int) {
   251  //		pc := getcallerpc()
   252  //		sp := getcallersp()
   253  //	}
   254  //
   255  // These two lines find the PC and SP immediately following
   256  // the call to f (where f will return).
   257  //
   258  // The call to getcallerpc and getcallersp must be done in the
   259  // frame being asked about.
   260  //
   261  // The result of getcallersp is correct at the time of the return,
   262  // but it may be invalidated by any subsequent call to a function
   263  // that might relocate the stack in order to grow or shrink it.
   264  // A general rule is that the result of getcallersp should be used
   265  // immediately and can only be passed to nosplit functions.
   266  
   267  //go:noescape
   268  func getcallerpc() uintptr
   269  
   270  //go:noescape
   271  func getcallersp() uintptr // implemented as an intrinsic on all platforms
   272  
   273  // getclosureptr returns the pointer to the current closure.
   274  // getclosureptr can only be used in an assignment statement
   275  // at the entry of a function. Moreover, go:nosplit directive
   276  // must be specified at the declaration of caller function,
   277  // so that the function prolog does not clobber the closure register.
   278  // for example:
   279  //
   280  //	//go:nosplit
   281  //	func f(arg1, arg2, arg3 int) {
   282  //		dx := getclosureptr()
   283  //	}
   284  //
   285  // The compiler rewrites calls to this function into instructions that fetch the
   286  // pointer from a well-known register (DX on x86 architecture, etc.) directly.
   287  //
   288  // WARNING: PGO-based devirtualization cannot detect that caller of
   289  // getclosureptr require closure context, and thus must maintain a list of
   290  // these functions, which is in
   291  // cmd/compile/internal/devirtualize/pgo.maybeDevirtualizeFunctionCall.
   292  func getclosureptr() uintptr
   293  
   294  //go:noescape
   295  func asmcgocall(fn, arg unsafe.Pointer) int32
   296  
   297  func morestack()
   298  func morestack_noctxt()
   299  func rt0_go()
   300  
   301  // return0 is a stub used to return 0 from deferproc.
   302  // It is called at the very end of deferproc to signal
   303  // the calling Go function that it should not jump
   304  // to deferreturn.
   305  // in asm_*.s
   306  func return0()
   307  
   308  // in asm_*.s
   309  // not called directly; definitions here supply type information for traceback.
   310  // These must have the same signature (arg pointer map) as reflectcall.
   311  func call16(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   312  func call32(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   313  func call64(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   314  func call128(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   315  func call256(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   316  func call512(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   317  func call1024(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   318  func call2048(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   319  func call4096(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   320  func call8192(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   321  func call16384(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   322  func call32768(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   323  func call65536(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   324  func call131072(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   325  func call262144(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   326  func call524288(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   327  func call1048576(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   328  func call2097152(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   329  func call4194304(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   330  func call8388608(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   331  func call16777216(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   332  func call33554432(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   333  func call67108864(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   334  func call134217728(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   335  func call268435456(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   336  func call536870912(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   337  func call1073741824(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   338  
   339  func systemstack_switch()
   340  
   341  // alignUp rounds n up to a multiple of a. a must be a power of 2.
   342  //
   343  //go:nosplit
   344  func alignUp(n, a uintptr) uintptr {
   345  	return (n + a - 1) &^ (a - 1)
   346  }
   347  
   348  // alignDown rounds n down to a multiple of a. a must be a power of 2.
   349  //
   350  //go:nosplit
   351  func alignDown(n, a uintptr) uintptr {
   352  	return n &^ (a - 1)
   353  }
   354  
   355  // divRoundUp returns ceil(n / a).
   356  func divRoundUp(n, a uintptr) uintptr {
   357  	// a is generally a power of two. This will get inlined and
   358  	// the compiler will optimize the division.
   359  	return (n + a - 1) / a
   360  }
   361  
   362  // checkASM reports whether assembly runtime checks have passed.
   363  func checkASM() bool
   364  
   365  func memequal_varlen(a, b unsafe.Pointer) bool
   366  
   367  // bool2int returns 0 if x is false or 1 if x is true.
   368  func bool2int(x bool) int {
   369  	// Avoid branches. In the SSA compiler, this compiles to
   370  	// exactly what you would want it to.
   371  	return int(*(*uint8)(unsafe.Pointer(&x)))
   372  }
   373  
   374  // abort crashes the runtime in situations where even throw might not
   375  // work. In general it should do something a debugger will recognize
   376  // (e.g., an INT3 on x86). A crash in abort is recognized by the
   377  // signal handler, which will attempt to tear down the runtime
   378  // immediately.
   379  func abort()
   380  
   381  // Called from compiled code; declared for vet; do NOT call from Go.
   382  func gcWriteBarrier1()
   383  func gcWriteBarrier2()
   384  func gcWriteBarrier3()
   385  func gcWriteBarrier4()
   386  func gcWriteBarrier5()
   387  func gcWriteBarrier6()
   388  func gcWriteBarrier7()
   389  func gcWriteBarrier8()
   390  func duffzero()
   391  func duffcopy()
   392  
   393  // Called from linker-generated .initarray; declared for go vet; do NOT call from Go.
   394  func addmoduledata()
   395  
   396  // Injected by the signal handler for panicking signals.
   397  // Initializes any registers that have fixed meaning at calls but
   398  // are scratch in bodies and calls sigpanic.
   399  // On many platforms it just jumps to sigpanic.
   400  func sigpanic0()
   401  
   402  // intArgRegs is used by the various register assignment
   403  // algorithm implementations in the runtime. These include:.
   404  // - Finalizers (mfinal.go)
   405  // - Windows callbacks (syscall_windows.go)
   406  //
   407  // Both are stripped-down versions of the algorithm since they
   408  // only have to deal with a subset of cases (finalizers only
   409  // take a pointer or interface argument, Go Windows callbacks
   410  // don't support floating point).
   411  //
   412  // It should be modified with care and are generally only
   413  // modified when testing this package.
   414  //
   415  // It should never be set higher than its internal/abi
   416  // constant counterparts, because the system relies on a
   417  // structure that is at least large enough to hold the
   418  // registers the system supports.
   419  //
   420  // Protected by finlock.
   421  var intArgRegs = abi.IntArgRegs
   422  

View as plain text