Source file src/runtime/cgocall.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Cgo call and callback support.
     6  //
     7  // To call into the C function f from Go, the cgo-generated code calls
     8  // runtime.cgocall(_cgo_Cfunc_f, frame), where _cgo_Cfunc_f is a
     9  // gcc-compiled function written by cgo.
    10  //
    11  // runtime.cgocall (below) calls entersyscall so as not to block
    12  // other goroutines or the garbage collector, and then calls
    13  // runtime.asmcgocall(_cgo_Cfunc_f, frame).
    14  //
    15  // runtime.asmcgocall (in asm_$GOARCH.s) switches to the m->g0 stack
    16  // (assumed to be an operating system-allocated stack, so safe to run
    17  // gcc-compiled code on) and calls _cgo_Cfunc_f(frame).
    18  //
    19  // _cgo_Cfunc_f invokes the actual C function f with arguments
    20  // taken from the frame structure, records the results in the frame,
    21  // and returns to runtime.asmcgocall.
    22  //
    23  // After it regains control, runtime.asmcgocall switches back to the
    24  // original g (m->curg)'s stack and returns to runtime.cgocall.
    25  //
    26  // After it regains control, runtime.cgocall calls exitsyscall, which blocks
    27  // until this m can run Go code without violating the $GOMAXPROCS limit,
    28  // and then unlocks g from m.
    29  //
    30  // The above description skipped over the possibility of the gcc-compiled
    31  // function f calling back into Go. If that happens, we continue down
    32  // the rabbit hole during the execution of f.
    33  //
    34  // To make it possible for gcc-compiled C code to call a Go function p.GoF,
    35  // cgo writes a gcc-compiled function named GoF (not p.GoF, since gcc doesn't
    36  // know about packages).  The gcc-compiled C function f calls GoF.
    37  //
    38  // GoF initializes "frame", a structure containing all of its
    39  // arguments and slots for p.GoF's results. It calls
    40  // crosscall2(_cgoexp_GoF, frame, framesize, ctxt) using the gcc ABI.
    41  //
    42  // crosscall2 (in cgo/asm_$GOARCH.s) is a four-argument adapter from
    43  // the gcc function call ABI to the gc function call ABI. At this
    44  // point we're in the Go runtime, but we're still running on m.g0's
    45  // stack and outside the $GOMAXPROCS limit. crosscall2 calls
    46  // runtime.cgocallback(_cgoexp_GoF, frame, ctxt) using the gc ABI.
    47  // (crosscall2's framesize argument is no longer used, but there's one
    48  // case where SWIG calls crosscall2 directly and expects to pass this
    49  // argument. See _cgo_panic.)
    50  //
    51  // runtime.cgocallback (in asm_$GOARCH.s) switches from m.g0's stack
    52  // to the original g (m.curg)'s stack, on which it calls
    53  // runtime.cgocallbackg(_cgoexp_GoF, frame, ctxt). As part of the
    54  // stack switch, runtime.cgocallback saves the current SP as
    55  // m.g0.sched.sp, so that any use of m.g0's stack during the execution
    56  // of the callback will be done below the existing stack frames.
    57  // Before overwriting m.g0.sched.sp, it pushes the old value on the
    58  // m.g0 stack, so that it can be restored later.
    59  //
    60  // runtime.cgocallbackg (below) is now running on a real goroutine
    61  // stack (not an m.g0 stack).  First it calls runtime.exitsyscall, which will
    62  // block until the $GOMAXPROCS limit allows running this goroutine.
    63  // Once exitsyscall has returned, it is safe to do things like call the memory
    64  // allocator or invoke the Go callback function.  runtime.cgocallbackg
    65  // first defers a function to unwind m.g0.sched.sp, so that if p.GoF
    66  // panics, m.g0.sched.sp will be restored to its old value: the m.g0 stack
    67  // and the m.curg stack will be unwound in lock step.
    68  // Then it calls _cgoexp_GoF(frame).
    69  //
    70  // _cgoexp_GoF, which was generated by cmd/cgo, unpacks the arguments
    71  // from frame, calls p.GoF, writes the results back to frame, and
    72  // returns. Now we start unwinding this whole process.
    73  //
    74  // runtime.cgocallbackg pops but does not execute the deferred
    75  // function to unwind m.g0.sched.sp, calls runtime.entersyscall, and
    76  // returns to runtime.cgocallback.
    77  //
    78  // After it regains control, runtime.cgocallback switches back to
    79  // m.g0's stack (the pointer is still in m.g0.sched.sp), restores the old
    80  // m.g0.sched.sp value from the stack, and returns to crosscall2.
    81  //
    82  // crosscall2 restores the callee-save registers for gcc and returns
    83  // to GoF, which unpacks any result values and returns to f.
    84  
    85  package runtime
    86  
    87  import (
    88  	"internal/goarch"
    89  	"internal/goexperiment"
    90  	"runtime/internal/sys"
    91  	"unsafe"
    92  )
    93  
    94  // Addresses collected in a cgo backtrace when crashing.
    95  // Length must match arg.Max in x_cgo_callers in runtime/cgo/gcc_traceback.c.
    96  type cgoCallers [32]uintptr
    97  
    98  // argset matches runtime/cgo/linux_syscall.c:argset_t
    99  type argset struct {
   100  	args   unsafe.Pointer
   101  	retval uintptr
   102  }
   103  
   104  // wrapper for syscall package to call cgocall for libc (cgo) calls.
   105  //
   106  //go:linkname syscall_cgocaller syscall.cgocaller
   107  //go:nosplit
   108  //go:uintptrescapes
   109  func syscall_cgocaller(fn unsafe.Pointer, args ...uintptr) uintptr {
   110  	as := argset{args: unsafe.Pointer(&args[0])}
   111  	cgocall(fn, unsafe.Pointer(&as))
   112  	return as.retval
   113  }
   114  
   115  var ncgocall uint64 // number of cgo calls in total for dead m
   116  
   117  // Call from Go to C.
   118  //
   119  // This must be nosplit because it's used for syscalls on some
   120  // platforms. Syscalls may have untyped arguments on the stack, so
   121  // it's not safe to grow or scan the stack.
   122  //
   123  //go:nosplit
   124  func cgocall(fn, arg unsafe.Pointer) int32 {
   125  	if !iscgo && GOOS != "solaris" && GOOS != "illumos" && GOOS != "windows" {
   126  		throw("cgocall unavailable")
   127  	}
   128  
   129  	if fn == nil {
   130  		throw("cgocall nil")
   131  	}
   132  
   133  	if raceenabled {
   134  		racereleasemerge(unsafe.Pointer(&racecgosync))
   135  	}
   136  
   137  	mp := getg().m
   138  	mp.ncgocall++
   139  
   140  	// Reset traceback.
   141  	mp.cgoCallers[0] = 0
   142  
   143  	// Announce we are entering a system call
   144  	// so that the scheduler knows to create another
   145  	// M to run goroutines while we are in the
   146  	// foreign code.
   147  	//
   148  	// The call to asmcgocall is guaranteed not to
   149  	// grow the stack and does not allocate memory,
   150  	// so it is safe to call while "in a system call", outside
   151  	// the $GOMAXPROCS accounting.
   152  	//
   153  	// fn may call back into Go code, in which case we'll exit the
   154  	// "system call", run the Go code (which may grow the stack),
   155  	// and then re-enter the "system call" reusing the PC and SP
   156  	// saved by entersyscall here.
   157  	entersyscall()
   158  
   159  	// Tell asynchronous preemption that we're entering external
   160  	// code. We do this after entersyscall because this may block
   161  	// and cause an async preemption to fail, but at this point a
   162  	// sync preemption will succeed (though this is not a matter
   163  	// of correctness).
   164  	osPreemptExtEnter(mp)
   165  
   166  	mp.incgo = true
   167  	// We use ncgo as a check during execution tracing for whether there is
   168  	// any C on the call stack, which there will be after this point. If
   169  	// there isn't, we can use frame pointer unwinding to collect call
   170  	// stacks efficiently. This will be the case for the first Go-to-C call
   171  	// on a stack, so it's preferable to update it here, after we emit a
   172  	// trace event in entersyscall above.
   173  	mp.ncgo++
   174  
   175  	errno := asmcgocall(fn, arg)
   176  
   177  	// Update accounting before exitsyscall because exitsyscall may
   178  	// reschedule us on to a different M.
   179  	mp.incgo = false
   180  	mp.ncgo--
   181  
   182  	osPreemptExtExit(mp)
   183  
   184  	exitsyscall()
   185  
   186  	// Note that raceacquire must be called only after exitsyscall has
   187  	// wired this M to a P.
   188  	if raceenabled {
   189  		raceacquire(unsafe.Pointer(&racecgosync))
   190  	}
   191  
   192  	// From the garbage collector's perspective, time can move
   193  	// backwards in the sequence above. If there's a callback into
   194  	// Go code, GC will see this function at the call to
   195  	// asmcgocall. When the Go call later returns to C, the
   196  	// syscall PC/SP is rolled back and the GC sees this function
   197  	// back at the call to entersyscall. Normally, fn and arg
   198  	// would be live at entersyscall and dead at asmcgocall, so if
   199  	// time moved backwards, GC would see these arguments as dead
   200  	// and then live. Prevent these undead arguments from crashing
   201  	// GC by forcing them to stay live across this time warp.
   202  	KeepAlive(fn)
   203  	KeepAlive(arg)
   204  	KeepAlive(mp)
   205  
   206  	return errno
   207  }
   208  
   209  // Set or reset the system stack bounds for a callback on sp.
   210  //
   211  // Must be nosplit because it is called by needm prior to fully initializing
   212  // the M.
   213  //
   214  //go:nosplit
   215  func callbackUpdateSystemStack(mp *m, sp uintptr, signal bool) {
   216  	g0 := mp.g0
   217  	if sp > g0.stack.lo && sp <= g0.stack.hi {
   218  		// Stack already in bounds, nothing to do.
   219  		return
   220  	}
   221  
   222  	if mp.ncgo > 0 {
   223  		// ncgo > 0 indicates that this M was in Go further up the stack
   224  		// (it called C and is now receiving a callback). It is not
   225  		// safe for the C call to change the stack out from under us.
   226  
   227  		// Note that this case isn't possible for signal == true, as
   228  		// that is always passing a new M from needm.
   229  
   230  		// Stack is bogus, but reset the bounds anyway so we can print.
   231  		hi := g0.stack.hi
   232  		lo := g0.stack.lo
   233  		g0.stack.hi = sp + 1024
   234  		g0.stack.lo = sp - 32*1024
   235  		g0.stackguard0 = g0.stack.lo + stackGuard
   236  		g0.stackguard1 = g0.stackguard0
   237  
   238  		print("M ", mp.id, " procid ", mp.procid, " runtime: cgocallback with sp=", hex(sp), " out of bounds [", hex(lo), ", ", hex(hi), "]")
   239  		print("\n")
   240  		exit(2)
   241  	}
   242  
   243  	// This M does not have Go further up the stack. However, it may have
   244  	// previously called into Go, initializing the stack bounds. Between
   245  	// that call returning and now the stack may have changed (perhaps the
   246  	// C thread is running a coroutine library). We need to update the
   247  	// stack bounds for this case.
   248  	//
   249  	// Set the stack bounds to match the current stack. If we don't
   250  	// actually know how big the stack is, like we don't know how big any
   251  	// scheduling stack is, but we assume there's at least 32 kB. If we
   252  	// can get a more accurate stack bound from pthread, use that, provided
   253  	// it actually contains SP..
   254  	g0.stack.hi = sp + 1024
   255  	g0.stack.lo = sp - 32*1024
   256  	if !signal && _cgo_getstackbound != nil {
   257  		// Don't adjust if called from the signal handler.
   258  		// We are on the signal stack, not the pthread stack.
   259  		// (We could get the stack bounds from sigaltstack, but
   260  		// we're getting out of the signal handler very soon
   261  		// anyway. Not worth it.)
   262  		var bounds [2]uintptr
   263  		asmcgocall(_cgo_getstackbound, unsafe.Pointer(&bounds))
   264  		// getstackbound is an unsupported no-op on Windows.
   265  		//
   266  		// Don't use these bounds if they don't contain SP. Perhaps we
   267  		// were called by something not using the standard thread
   268  		// stack.
   269  		if bounds[0] != 0 && sp > bounds[0] && sp <= bounds[1] {
   270  			g0.stack.lo = bounds[0]
   271  			g0.stack.hi = bounds[1]
   272  		}
   273  	}
   274  	g0.stackguard0 = g0.stack.lo + stackGuard
   275  	g0.stackguard1 = g0.stackguard0
   276  }
   277  
   278  // Call from C back to Go. fn must point to an ABIInternal Go entry-point.
   279  //
   280  //go:nosplit
   281  func cgocallbackg(fn, frame unsafe.Pointer, ctxt uintptr) {
   282  	gp := getg()
   283  	if gp != gp.m.curg {
   284  		println("runtime: bad g in cgocallback")
   285  		exit(2)
   286  	}
   287  
   288  	sp := gp.m.g0.sched.sp // system sp saved by cgocallback.
   289  	callbackUpdateSystemStack(gp.m, sp, false)
   290  
   291  	// The call from C is on gp.m's g0 stack, so we must ensure
   292  	// that we stay on that M. We have to do this before calling
   293  	// exitsyscall, since it would otherwise be free to move us to
   294  	// a different M. The call to unlockOSThread is in this function
   295  	// after cgocallbackg1, or in the case of panicking, in unwindm.
   296  	lockOSThread()
   297  
   298  	checkm := gp.m
   299  
   300  	// Save current syscall parameters, so m.syscall can be
   301  	// used again if callback decide to make syscall.
   302  	syscall := gp.m.syscall
   303  
   304  	// entersyscall saves the caller's SP to allow the GC to trace the Go
   305  	// stack. However, since we're returning to an earlier stack frame and
   306  	// need to pair with the entersyscall() call made by cgocall, we must
   307  	// save syscall* and let reentersyscall restore them.
   308  	savedsp := unsafe.Pointer(gp.syscallsp)
   309  	savedpc := gp.syscallpc
   310  	exitsyscall() // coming out of cgo call
   311  	gp.m.incgo = false
   312  	if gp.m.isextra {
   313  		gp.m.isExtraInC = false
   314  	}
   315  
   316  	osPreemptExtExit(gp.m)
   317  
   318  	if gp.nocgocallback {
   319  		panic("runtime: function marked with #cgo nocallback called back into Go")
   320  	}
   321  
   322  	cgocallbackg1(fn, frame, ctxt)
   323  
   324  	// At this point we're about to call unlockOSThread.
   325  	// The following code must not change to a different m.
   326  	// This is enforced by checking incgo in the schedule function.
   327  	gp.m.incgo = true
   328  	unlockOSThread()
   329  
   330  	if gp.m.isextra {
   331  		gp.m.isExtraInC = true
   332  	}
   333  
   334  	if gp.m != checkm {
   335  		throw("m changed unexpectedly in cgocallbackg")
   336  	}
   337  
   338  	osPreemptExtEnter(gp.m)
   339  
   340  	// going back to cgo call
   341  	reentersyscall(savedpc, uintptr(savedsp))
   342  
   343  	gp.m.syscall = syscall
   344  }
   345  
   346  func cgocallbackg1(fn, frame unsafe.Pointer, ctxt uintptr) {
   347  	gp := getg()
   348  
   349  	if gp.m.needextram || extraMWaiters.Load() > 0 {
   350  		gp.m.needextram = false
   351  		systemstack(newextram)
   352  	}
   353  
   354  	if ctxt != 0 {
   355  		s := append(gp.cgoCtxt, ctxt)
   356  
   357  		// Now we need to set gp.cgoCtxt = s, but we could get
   358  		// a SIGPROF signal while manipulating the slice, and
   359  		// the SIGPROF handler could pick up gp.cgoCtxt while
   360  		// tracing up the stack.  We need to ensure that the
   361  		// handler always sees a valid slice, so set the
   362  		// values in an order such that it always does.
   363  		p := (*slice)(unsafe.Pointer(&gp.cgoCtxt))
   364  		atomicstorep(unsafe.Pointer(&p.array), unsafe.Pointer(&s[0]))
   365  		p.cap = cap(s)
   366  		p.len = len(s)
   367  
   368  		defer func(gp *g) {
   369  			// Decrease the length of the slice by one, safely.
   370  			p := (*slice)(unsafe.Pointer(&gp.cgoCtxt))
   371  			p.len--
   372  		}(gp)
   373  	}
   374  
   375  	if gp.m.ncgo == 0 {
   376  		// The C call to Go came from a thread not currently running
   377  		// any Go. In the case of -buildmode=c-archive or c-shared,
   378  		// this call may be coming in before package initialization
   379  		// is complete. Wait until it is.
   380  		<-main_init_done
   381  	}
   382  
   383  	// Check whether the profiler needs to be turned on or off; this route to
   384  	// run Go code does not use runtime.execute, so bypasses the check there.
   385  	hz := sched.profilehz
   386  	if gp.m.profilehz != hz {
   387  		setThreadCPUProfiler(hz)
   388  	}
   389  
   390  	// Add entry to defer stack in case of panic.
   391  	restore := true
   392  	defer unwindm(&restore)
   393  
   394  	if raceenabled {
   395  		raceacquire(unsafe.Pointer(&racecgosync))
   396  	}
   397  
   398  	// Invoke callback. This function is generated by cmd/cgo and
   399  	// will unpack the argument frame and call the Go function.
   400  	var cb func(frame unsafe.Pointer)
   401  	cbFV := funcval{uintptr(fn)}
   402  	*(*unsafe.Pointer)(unsafe.Pointer(&cb)) = noescape(unsafe.Pointer(&cbFV))
   403  	cb(frame)
   404  
   405  	if raceenabled {
   406  		racereleasemerge(unsafe.Pointer(&racecgosync))
   407  	}
   408  
   409  	// Do not unwind m->g0->sched.sp.
   410  	// Our caller, cgocallback, will do that.
   411  	restore = false
   412  }
   413  
   414  func unwindm(restore *bool) {
   415  	if *restore {
   416  		// Restore sp saved by cgocallback during
   417  		// unwind of g's stack (see comment at top of file).
   418  		mp := acquirem()
   419  		sched := &mp.g0.sched
   420  		sched.sp = *(*uintptr)(unsafe.Pointer(sched.sp + alignUp(sys.MinFrameSize, sys.StackAlign)))
   421  
   422  		// Do the accounting that cgocall will not have a chance to do
   423  		// during an unwind.
   424  		//
   425  		// In the case where a Go call originates from C, ncgo is 0
   426  		// and there is no matching cgocall to end.
   427  		if mp.ncgo > 0 {
   428  			mp.incgo = false
   429  			mp.ncgo--
   430  			osPreemptExtExit(mp)
   431  		}
   432  
   433  		// Undo the call to lockOSThread in cgocallbackg, only on the
   434  		// panicking path. In normal return case cgocallbackg will call
   435  		// unlockOSThread, ensuring no preemption point after the unlock.
   436  		// Here we don't need to worry about preemption, because we're
   437  		// panicking out of the callback and unwinding the g0 stack,
   438  		// instead of reentering cgo (which requires the same thread).
   439  		unlockOSThread()
   440  
   441  		releasem(mp)
   442  	}
   443  }
   444  
   445  // called from assembly.
   446  func badcgocallback() {
   447  	throw("misaligned stack in cgocallback")
   448  }
   449  
   450  // called from (incomplete) assembly.
   451  func cgounimpl() {
   452  	throw("cgo not implemented")
   453  }
   454  
   455  var racecgosync uint64 // represents possible synchronization in C code
   456  
   457  // Pointer checking for cgo code.
   458  
   459  // We want to detect all cases where a program that does not use
   460  // unsafe makes a cgo call passing a Go pointer to memory that
   461  // contains an unpinned Go pointer. Here a Go pointer is defined as a
   462  // pointer to memory allocated by the Go runtime. Programs that use
   463  // unsafe can evade this restriction easily, so we don't try to catch
   464  // them. The cgo program will rewrite all possibly bad pointer
   465  // arguments to call cgoCheckPointer, where we can catch cases of a Go
   466  // pointer pointing to an unpinned Go pointer.
   467  
   468  // Complicating matters, taking the address of a slice or array
   469  // element permits the C program to access all elements of the slice
   470  // or array. In that case we will see a pointer to a single element,
   471  // but we need to check the entire data structure.
   472  
   473  // The cgoCheckPointer call takes additional arguments indicating that
   474  // it was called on an address expression. An additional argument of
   475  // true means that it only needs to check a single element. An
   476  // additional argument of a slice or array means that it needs to
   477  // check the entire slice/array, but nothing else. Otherwise, the
   478  // pointer could be anything, and we check the entire heap object,
   479  // which is conservative but safe.
   480  
   481  // When and if we implement a moving garbage collector,
   482  // cgoCheckPointer will pin the pointer for the duration of the cgo
   483  // call.  (This is necessary but not sufficient; the cgo program will
   484  // also have to change to pin Go pointers that cannot point to Go
   485  // pointers.)
   486  
   487  // cgoCheckPointer checks if the argument contains a Go pointer that
   488  // points to an unpinned Go pointer, and panics if it does.
   489  func cgoCheckPointer(ptr any, arg any) {
   490  	if !goexperiment.CgoCheck2 && debug.cgocheck == 0 {
   491  		return
   492  	}
   493  
   494  	ep := efaceOf(&ptr)
   495  	t := ep._type
   496  
   497  	top := true
   498  	if arg != nil && (t.Kind_&kindMask == kindPtr || t.Kind_&kindMask == kindUnsafePointer) {
   499  		p := ep.data
   500  		if t.Kind_&kindDirectIface == 0 {
   501  			p = *(*unsafe.Pointer)(p)
   502  		}
   503  		if p == nil || !cgoIsGoPointer(p) {
   504  			return
   505  		}
   506  		aep := efaceOf(&arg)
   507  		switch aep._type.Kind_ & kindMask {
   508  		case kindBool:
   509  			if t.Kind_&kindMask == kindUnsafePointer {
   510  				// We don't know the type of the element.
   511  				break
   512  			}
   513  			pt := (*ptrtype)(unsafe.Pointer(t))
   514  			cgoCheckArg(pt.Elem, p, true, false, cgoCheckPointerFail)
   515  			return
   516  		case kindSlice:
   517  			// Check the slice rather than the pointer.
   518  			ep = aep
   519  			t = ep._type
   520  		case kindArray:
   521  			// Check the array rather than the pointer.
   522  			// Pass top as false since we have a pointer
   523  			// to the array.
   524  			ep = aep
   525  			t = ep._type
   526  			top = false
   527  		default:
   528  			throw("can't happen")
   529  		}
   530  	}
   531  
   532  	cgoCheckArg(t, ep.data, t.Kind_&kindDirectIface == 0, top, cgoCheckPointerFail)
   533  }
   534  
   535  const cgoCheckPointerFail = "cgo argument has Go pointer to unpinned Go pointer"
   536  const cgoResultFail = "cgo result is unpinned Go pointer or points to unpinned Go pointer"
   537  
   538  // cgoCheckArg is the real work of cgoCheckPointer. The argument p
   539  // is either a pointer to the value (of type t), or the value itself,
   540  // depending on indir. The top parameter is whether we are at the top
   541  // level, where Go pointers are allowed. Go pointers to pinned objects are
   542  // allowed as long as they don't reference other unpinned pointers.
   543  func cgoCheckArg(t *_type, p unsafe.Pointer, indir, top bool, msg string) {
   544  	if t.PtrBytes == 0 || p == nil {
   545  		// If the type has no pointers there is nothing to do.
   546  		return
   547  	}
   548  
   549  	switch t.Kind_ & kindMask {
   550  	default:
   551  		throw("can't happen")
   552  	case kindArray:
   553  		at := (*arraytype)(unsafe.Pointer(t))
   554  		if !indir {
   555  			if at.Len != 1 {
   556  				throw("can't happen")
   557  			}
   558  			cgoCheckArg(at.Elem, p, at.Elem.Kind_&kindDirectIface == 0, top, msg)
   559  			return
   560  		}
   561  		for i := uintptr(0); i < at.Len; i++ {
   562  			cgoCheckArg(at.Elem, p, true, top, msg)
   563  			p = add(p, at.Elem.Size_)
   564  		}
   565  	case kindChan, kindMap:
   566  		// These types contain internal pointers that will
   567  		// always be allocated in the Go heap. It's never OK
   568  		// to pass them to C.
   569  		panic(errorString(msg))
   570  	case kindFunc:
   571  		if indir {
   572  			p = *(*unsafe.Pointer)(p)
   573  		}
   574  		if !cgoIsGoPointer(p) {
   575  			return
   576  		}
   577  		panic(errorString(msg))
   578  	case kindInterface:
   579  		it := *(**_type)(p)
   580  		if it == nil {
   581  			return
   582  		}
   583  		// A type known at compile time is OK since it's
   584  		// constant. A type not known at compile time will be
   585  		// in the heap and will not be OK.
   586  		if inheap(uintptr(unsafe.Pointer(it))) {
   587  			panic(errorString(msg))
   588  		}
   589  		p = *(*unsafe.Pointer)(add(p, goarch.PtrSize))
   590  		if !cgoIsGoPointer(p) {
   591  			return
   592  		}
   593  		if !top && !isPinned(p) {
   594  			panic(errorString(msg))
   595  		}
   596  		cgoCheckArg(it, p, it.Kind_&kindDirectIface == 0, false, msg)
   597  	case kindSlice:
   598  		st := (*slicetype)(unsafe.Pointer(t))
   599  		s := (*slice)(p)
   600  		p = s.array
   601  		if p == nil || !cgoIsGoPointer(p) {
   602  			return
   603  		}
   604  		if !top && !isPinned(p) {
   605  			panic(errorString(msg))
   606  		}
   607  		if st.Elem.PtrBytes == 0 {
   608  			return
   609  		}
   610  		for i := 0; i < s.cap; i++ {
   611  			cgoCheckArg(st.Elem, p, true, false, msg)
   612  			p = add(p, st.Elem.Size_)
   613  		}
   614  	case kindString:
   615  		ss := (*stringStruct)(p)
   616  		if !cgoIsGoPointer(ss.str) {
   617  			return
   618  		}
   619  		if !top && !isPinned(ss.str) {
   620  			panic(errorString(msg))
   621  		}
   622  	case kindStruct:
   623  		st := (*structtype)(unsafe.Pointer(t))
   624  		if !indir {
   625  			if len(st.Fields) != 1 {
   626  				throw("can't happen")
   627  			}
   628  			cgoCheckArg(st.Fields[0].Typ, p, st.Fields[0].Typ.Kind_&kindDirectIface == 0, top, msg)
   629  			return
   630  		}
   631  		for _, f := range st.Fields {
   632  			if f.Typ.PtrBytes == 0 {
   633  				continue
   634  			}
   635  			cgoCheckArg(f.Typ, add(p, f.Offset), true, top, msg)
   636  		}
   637  	case kindPtr, kindUnsafePointer:
   638  		if indir {
   639  			p = *(*unsafe.Pointer)(p)
   640  			if p == nil {
   641  				return
   642  			}
   643  		}
   644  
   645  		if !cgoIsGoPointer(p) {
   646  			return
   647  		}
   648  		if !top && !isPinned(p) {
   649  			panic(errorString(msg))
   650  		}
   651  
   652  		cgoCheckUnknownPointer(p, msg)
   653  	}
   654  }
   655  
   656  // cgoCheckUnknownPointer is called for an arbitrary pointer into Go
   657  // memory. It checks whether that Go memory contains any other
   658  // pointer into unpinned Go memory. If it does, we panic.
   659  // The return values are unused but useful to see in panic tracebacks.
   660  func cgoCheckUnknownPointer(p unsafe.Pointer, msg string) (base, i uintptr) {
   661  	if inheap(uintptr(p)) {
   662  		b, span, _ := findObject(uintptr(p), 0, 0)
   663  		base = b
   664  		if base == 0 {
   665  			return
   666  		}
   667  		if goexperiment.AllocHeaders {
   668  			tp := span.typePointersOfUnchecked(base)
   669  			for {
   670  				var addr uintptr
   671  				if tp, addr = tp.next(base + span.elemsize); addr == 0 {
   672  					break
   673  				}
   674  				pp := *(*unsafe.Pointer)(unsafe.Pointer(addr))
   675  				if cgoIsGoPointer(pp) && !isPinned(pp) {
   676  					panic(errorString(msg))
   677  				}
   678  			}
   679  		} else {
   680  			n := span.elemsize
   681  			hbits := heapBitsForAddr(base, n)
   682  			for {
   683  				var addr uintptr
   684  				if hbits, addr = hbits.next(); addr == 0 {
   685  					break
   686  				}
   687  				pp := *(*unsafe.Pointer)(unsafe.Pointer(addr))
   688  				if cgoIsGoPointer(pp) && !isPinned(pp) {
   689  					panic(errorString(msg))
   690  				}
   691  			}
   692  		}
   693  		return
   694  	}
   695  
   696  	for _, datap := range activeModules() {
   697  		if cgoInRange(p, datap.data, datap.edata) || cgoInRange(p, datap.bss, datap.ebss) {
   698  			// We have no way to know the size of the object.
   699  			// We have to assume that it might contain a pointer.
   700  			panic(errorString(msg))
   701  		}
   702  		// In the text or noptr sections, we know that the
   703  		// pointer does not point to a Go pointer.
   704  	}
   705  
   706  	return
   707  }
   708  
   709  // cgoIsGoPointer reports whether the pointer is a Go pointer--a
   710  // pointer to Go memory. We only care about Go memory that might
   711  // contain pointers.
   712  //
   713  //go:nosplit
   714  //go:nowritebarrierrec
   715  func cgoIsGoPointer(p unsafe.Pointer) bool {
   716  	if p == nil {
   717  		return false
   718  	}
   719  
   720  	if inHeapOrStack(uintptr(p)) {
   721  		return true
   722  	}
   723  
   724  	for _, datap := range activeModules() {
   725  		if cgoInRange(p, datap.data, datap.edata) || cgoInRange(p, datap.bss, datap.ebss) {
   726  			return true
   727  		}
   728  	}
   729  
   730  	return false
   731  }
   732  
   733  // cgoInRange reports whether p is between start and end.
   734  //
   735  //go:nosplit
   736  //go:nowritebarrierrec
   737  func cgoInRange(p unsafe.Pointer, start, end uintptr) bool {
   738  	return start <= uintptr(p) && uintptr(p) < end
   739  }
   740  
   741  // cgoCheckResult is called to check the result parameter of an
   742  // exported Go function. It panics if the result is or contains any
   743  // other pointer into unpinned Go memory.
   744  func cgoCheckResult(val any) {
   745  	if !goexperiment.CgoCheck2 && debug.cgocheck == 0 {
   746  		return
   747  	}
   748  
   749  	ep := efaceOf(&val)
   750  	t := ep._type
   751  	cgoCheckArg(t, ep.data, t.Kind_&kindDirectIface == 0, false, cgoResultFail)
   752  }
   753  

View as plain text