@cherrymui found the problem: a preemption can occur after the call to unlockOSThread in unwindm before g.m.incgo = true in cgocallbackg. That can cause the G to move to a new M, which breaks the assumptions of cgocallbackg.

The preemption point is the first defer in cgocallbackg1. When that deferred function runs (after the second defer, of unwindm), it can be preempted. I believe that this is due to the regabidefer experiment, which wraps most defer functions. Those wrappers are separate functions, and as such are preemption points.

Here is a test case that recreates the observed problem. In order to recreate the problem the test case uses runtime.SetCgoTraceback so that the first defer in cgocallbackg1 is executed.

foo1.go:

package main

/*
extern void cgoTraceback(void* p);
extern void cgoSymbolizer(void* p);
extern void cgoContext(void* p);
extern void GoFunction(int);

static void callGo(int i) {
	GoFunction(i);
}
*/
import "C"

import (
	"fmt"
	"runtime"
	"sync"
	"unsafe"
)

//export GoFunction
func GoFunction(i C.int) {
	fmt.Sprintf("%d\n", i)
}

func main() {
	runtime.SetCgoTraceback(0, unsafe.Pointer(C.cgoTraceback), unsafe.Pointer(C.cgoContext), unsafe.Pointer(C.cgoSymbolizer))
	const funcs = 1e3
	const calls = 1e5
	var wg sync.WaitGroup
	wg.Add(1)
	for i := 0; i < funcs; i++ {
		go func(i int) {
			defer wg.Done()
			for j := 0; j < calls; j++ {
				C.callGo(C.int(i*calls + j))
			}
		}(i)
	}
	wg.Wait()
}

foo2.c:

#include <stdio.h>
#include <signal.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>

static void crash(int signum) {
	printf("caught SIGSEGV\n");
	abort();
}

__attribute__((constructor))
void setSignalHandler() {
	struct sigaction sa;

	memset(&sa, 0, sizeof sa);
	if (sigfillset(&sa.sa_mask) != 0) {
		abort();
	}
	sa.sa_handler = crash;
	if (sigaction(SIGSEGV, &sa, NULL) != 0) {
		abort();
	}
}

void cgoTraceback(void* p) {}
void cgoSymbolizer(void* p) {}

struct contextArg {
	uintptr_t context;
};

void cgoContext(struct contextArg* p) {
	if (p->context == 0) {
		p->context = 1;
	}
}

Change https://golang.org/cl/338197 mentions this issue: runtime: avoid possible preemption when returning from Go to C

Change https://golang.org/cl/338270 mentions this issue: cmd/compile: mark defer wrapper nosplit for runtime and nosplit callee

Change https://golang.org/cl/359796 mentions this issue: runtime: add always-preempt maymorestack hook