...
Run Format

Source file src/runtime/race.go

Documentation: runtime

     1  // Copyright 2012 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // +build race
     6  
     7  package runtime
     8  
     9  import (
    10  	"unsafe"
    11  )
    12  
    13  // Public race detection API, present iff build with -race.
    14  
    15  func RaceRead(addr unsafe.Pointer)
    16  func RaceWrite(addr unsafe.Pointer)
    17  func RaceReadRange(addr unsafe.Pointer, len int)
    18  func RaceWriteRange(addr unsafe.Pointer, len int)
    19  
    20  func RaceErrors() int {
    21  	var n uint64
    22  	racecall(&__tsan_report_count, uintptr(unsafe.Pointer(&n)), 0, 0, 0)
    23  	return int(n)
    24  }
    25  
    26  //go:nosplit
    27  
    28  // RaceAcquire/RaceRelease/RaceReleaseMerge establish happens-before relations
    29  // between goroutines. These inform the race detector about actual synchronization
    30  // that it can't see for some reason (e.g. synchronization within RaceDisable/RaceEnable
    31  // sections of code).
    32  // RaceAcquire establishes a happens-before relation with the preceding
    33  // RaceReleaseMerge on addr up to and including the last RaceRelease on addr.
    34  // In terms of the C memory model (C11 §5.1.2.4, §7.17.3),
    35  // RaceAcquire is equivalent to atomic_load(memory_order_acquire).
    36  func RaceAcquire(addr unsafe.Pointer) {
    37  	raceacquire(addr)
    38  }
    39  
    40  //go:nosplit
    41  
    42  // RaceRelease performs a release operation on addr that
    43  // can synchronize with a later RaceAcquire on addr.
    44  //
    45  // In terms of the C memory model, RaceRelease is equivalent to
    46  // atomic_store(memory_order_release).
    47  func RaceRelease(addr unsafe.Pointer) {
    48  	racerelease(addr)
    49  }
    50  
    51  //go:nosplit
    52  
    53  // RaceReleaseMerge is like RaceRelease, but also establishes a happens-before
    54  // relation with the preceding RaceRelease or RaceReleaseMerge on addr.
    55  //
    56  // In terms of the C memory model, RaceReleaseMerge is equivalent to
    57  // atomic_exchange(memory_order_release).
    58  func RaceReleaseMerge(addr unsafe.Pointer) {
    59  	racereleasemerge(addr)
    60  }
    61  
    62  //go:nosplit
    63  
    64  // RaceDisable disables handling of race synchronization events in the current goroutine.
    65  // Handling is re-enabled with RaceEnable. RaceDisable/RaceEnable can be nested.
    66  // Non-synchronization events (memory accesses, function entry/exit) still affect
    67  // the race detector.
    68  func RaceDisable() {
    69  	_g_ := getg()
    70  	if _g_.raceignore == 0 {
    71  		racecall(&__tsan_go_ignore_sync_begin, _g_.racectx, 0, 0, 0)
    72  	}
    73  	_g_.raceignore++
    74  }
    75  
    76  //go:nosplit
    77  
    78  // RaceEnable re-enables handling of race events in the current goroutine.
    79  func RaceEnable() {
    80  	_g_ := getg()
    81  	_g_.raceignore--
    82  	if _g_.raceignore == 0 {
    83  		racecall(&__tsan_go_ignore_sync_end, _g_.racectx, 0, 0, 0)
    84  	}
    85  }
    86  
    87  // Private interface for the runtime.
    88  
    89  const raceenabled = true
    90  
    91  // For all functions accepting callerpc and pc,
    92  // callerpc is a return PC of the function that calls this function,
    93  // pc is start PC of the function that calls this function.
    94  func raceReadObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) {
    95  	kind := t.kind & kindMask
    96  	if kind == kindArray || kind == kindStruct {
    97  		// for composite objects we have to read every address
    98  		// because a write might happen to any subobject.
    99  		racereadrangepc(addr, t.size, callerpc, pc)
   100  	} else {
   101  		// for non-composite objects we can read just the start
   102  		// address, as any write must write the first byte.
   103  		racereadpc(addr, callerpc, pc)
   104  	}
   105  }
   106  
   107  func raceWriteObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) {
   108  	kind := t.kind & kindMask
   109  	if kind == kindArray || kind == kindStruct {
   110  		// for composite objects we have to write every address
   111  		// because a write might happen to any subobject.
   112  		racewriterangepc(addr, t.size, callerpc, pc)
   113  	} else {
   114  		// for non-composite objects we can write just the start
   115  		// address, as any write must write the first byte.
   116  		racewritepc(addr, callerpc, pc)
   117  	}
   118  }
   119  
   120  //go:noescape
   121  func racereadpc(addr unsafe.Pointer, callpc, pc uintptr)
   122  
   123  //go:noescape
   124  func racewritepc(addr unsafe.Pointer, callpc, pc uintptr)
   125  
   126  type symbolizeCodeContext struct {
   127  	pc   uintptr
   128  	fn   *byte
   129  	file *byte
   130  	line uintptr
   131  	off  uintptr
   132  	res  uintptr
   133  }
   134  
   135  var qq = [...]byte{'?', '?', 0}
   136  var dash = [...]byte{'-', 0}
   137  
   138  const (
   139  	raceGetProcCmd = iota
   140  	raceSymbolizeCodeCmd
   141  	raceSymbolizeDataCmd
   142  )
   143  
   144  // Callback from C into Go, runs on g0.
   145  func racecallback(cmd uintptr, ctx unsafe.Pointer) {
   146  	switch cmd {
   147  	case raceGetProcCmd:
   148  		throw("should have been handled by racecallbackthunk")
   149  	case raceSymbolizeCodeCmd:
   150  		raceSymbolizeCode((*symbolizeCodeContext)(ctx))
   151  	case raceSymbolizeDataCmd:
   152  		raceSymbolizeData((*symbolizeDataContext)(ctx))
   153  	default:
   154  		throw("unknown command")
   155  	}
   156  }
   157  
   158  func raceSymbolizeCode(ctx *symbolizeCodeContext) {
   159  	f := findfunc(ctx.pc)._Func()
   160  	if f != nil {
   161  		file, line := f.FileLine(ctx.pc)
   162  		if line != 0 {
   163  			ctx.fn = cfuncname(f.funcInfo())
   164  			ctx.line = uintptr(line)
   165  			ctx.file = &bytes(file)[0] // assume NUL-terminated
   166  			ctx.off = ctx.pc - f.Entry()
   167  			ctx.res = 1
   168  			return
   169  		}
   170  	}
   171  	ctx.fn = &qq[0]
   172  	ctx.file = &dash[0]
   173  	ctx.line = 0
   174  	ctx.off = ctx.pc
   175  	ctx.res = 1
   176  }
   177  
   178  type symbolizeDataContext struct {
   179  	addr  uintptr
   180  	heap  uintptr
   181  	start uintptr
   182  	size  uintptr
   183  	name  *byte
   184  	file  *byte
   185  	line  uintptr
   186  	res   uintptr
   187  }
   188  
   189  func raceSymbolizeData(ctx *symbolizeDataContext) {
   190  	if base, span, _ := findObject(ctx.addr, 0, 0); base != 0 {
   191  		ctx.heap = 1
   192  		ctx.start = base
   193  		ctx.size = span.elemsize
   194  		ctx.res = 1
   195  	}
   196  }
   197  
   198  // Race runtime functions called via runtime·racecall.
   199  //go:linkname __tsan_init __tsan_init
   200  var __tsan_init byte
   201  
   202  //go:linkname __tsan_fini __tsan_fini
   203  var __tsan_fini byte
   204  
   205  //go:linkname __tsan_proc_create __tsan_proc_create
   206  var __tsan_proc_create byte
   207  
   208  //go:linkname __tsan_proc_destroy __tsan_proc_destroy
   209  var __tsan_proc_destroy byte
   210  
   211  //go:linkname __tsan_map_shadow __tsan_map_shadow
   212  var __tsan_map_shadow byte
   213  
   214  //go:linkname __tsan_finalizer_goroutine __tsan_finalizer_goroutine
   215  var __tsan_finalizer_goroutine byte
   216  
   217  //go:linkname __tsan_go_start __tsan_go_start
   218  var __tsan_go_start byte
   219  
   220  //go:linkname __tsan_go_end __tsan_go_end
   221  var __tsan_go_end byte
   222  
   223  //go:linkname __tsan_malloc __tsan_malloc
   224  var __tsan_malloc byte
   225  
   226  //go:linkname __tsan_free __tsan_free
   227  var __tsan_free byte
   228  
   229  //go:linkname __tsan_acquire __tsan_acquire
   230  var __tsan_acquire byte
   231  
   232  //go:linkname __tsan_release __tsan_release
   233  var __tsan_release byte
   234  
   235  //go:linkname __tsan_release_merge __tsan_release_merge
   236  var __tsan_release_merge byte
   237  
   238  //go:linkname __tsan_go_ignore_sync_begin __tsan_go_ignore_sync_begin
   239  var __tsan_go_ignore_sync_begin byte
   240  
   241  //go:linkname __tsan_go_ignore_sync_end __tsan_go_ignore_sync_end
   242  var __tsan_go_ignore_sync_end byte
   243  
   244  //go:linkname __tsan_report_count __tsan_report_count
   245  var __tsan_report_count byte
   246  
   247  // Mimic what cmd/cgo would do.
   248  //go:cgo_import_static __tsan_init
   249  //go:cgo_import_static __tsan_fini
   250  //go:cgo_import_static __tsan_proc_create
   251  //go:cgo_import_static __tsan_proc_destroy
   252  //go:cgo_import_static __tsan_map_shadow
   253  //go:cgo_import_static __tsan_finalizer_goroutine
   254  //go:cgo_import_static __tsan_go_start
   255  //go:cgo_import_static __tsan_go_end
   256  //go:cgo_import_static __tsan_malloc
   257  //go:cgo_import_static __tsan_free
   258  //go:cgo_import_static __tsan_acquire
   259  //go:cgo_import_static __tsan_release
   260  //go:cgo_import_static __tsan_release_merge
   261  //go:cgo_import_static __tsan_go_ignore_sync_begin
   262  //go:cgo_import_static __tsan_go_ignore_sync_end
   263  //go:cgo_import_static __tsan_report_count
   264  
   265  // These are called from race_amd64.s.
   266  //go:cgo_import_static __tsan_read
   267  //go:cgo_import_static __tsan_read_pc
   268  //go:cgo_import_static __tsan_read_range
   269  //go:cgo_import_static __tsan_write
   270  //go:cgo_import_static __tsan_write_pc
   271  //go:cgo_import_static __tsan_write_range
   272  //go:cgo_import_static __tsan_func_enter
   273  //go:cgo_import_static __tsan_func_exit
   274  
   275  //go:cgo_import_static __tsan_go_atomic32_load
   276  //go:cgo_import_static __tsan_go_atomic64_load
   277  //go:cgo_import_static __tsan_go_atomic32_store
   278  //go:cgo_import_static __tsan_go_atomic64_store
   279  //go:cgo_import_static __tsan_go_atomic32_exchange
   280  //go:cgo_import_static __tsan_go_atomic64_exchange
   281  //go:cgo_import_static __tsan_go_atomic32_fetch_add
   282  //go:cgo_import_static __tsan_go_atomic64_fetch_add
   283  //go:cgo_import_static __tsan_go_atomic32_compare_exchange
   284  //go:cgo_import_static __tsan_go_atomic64_compare_exchange
   285  
   286  // start/end of global data (data+bss).
   287  var racedatastart uintptr
   288  var racedataend uintptr
   289  
   290  // start/end of heap for race_amd64.s
   291  var racearenastart uintptr
   292  var racearenaend uintptr
   293  
   294  func racefuncenter(uintptr)
   295  func racefuncenterfp()
   296  func racefuncexit()
   297  func raceread(uintptr)
   298  func racewrite(uintptr)
   299  func racereadrange(addr, size uintptr)
   300  func racewriterange(addr, size uintptr)
   301  func racereadrangepc1(uintptr, uintptr, uintptr)
   302  func racewriterangepc1(uintptr, uintptr, uintptr)
   303  func racecallbackthunk(uintptr)
   304  
   305  // racecall allows calling an arbitrary function f from C race runtime
   306  // with up to 4 uintptr arguments.
   307  func racecall(*byte, uintptr, uintptr, uintptr, uintptr)
   308  
   309  // checks if the address has shadow (i.e. heap or data/bss)
   310  //go:nosplit
   311  func isvalidaddr(addr unsafe.Pointer) bool {
   312  	return racearenastart <= uintptr(addr) && uintptr(addr) < racearenaend ||
   313  		racedatastart <= uintptr(addr) && uintptr(addr) < racedataend
   314  }
   315  
   316  //go:nosplit
   317  func raceinit() (gctx, pctx uintptr) {
   318  	// cgo is required to initialize libc, which is used by race runtime
   319  	if !iscgo {
   320  		throw("raceinit: race build must use cgo")
   321  	}
   322  
   323  	racecall(&__tsan_init, uintptr(unsafe.Pointer(&gctx)), uintptr(unsafe.Pointer(&pctx)), funcPC(racecallbackthunk), 0)
   324  
   325  	// Round data segment to page boundaries, because it's used in mmap().
   326  	start := ^uintptr(0)
   327  	end := uintptr(0)
   328  	if start > firstmoduledata.noptrdata {
   329  		start = firstmoduledata.noptrdata
   330  	}
   331  	if start > firstmoduledata.data {
   332  		start = firstmoduledata.data
   333  	}
   334  	if start > firstmoduledata.noptrbss {
   335  		start = firstmoduledata.noptrbss
   336  	}
   337  	if start > firstmoduledata.bss {
   338  		start = firstmoduledata.bss
   339  	}
   340  	if end < firstmoduledata.enoptrdata {
   341  		end = firstmoduledata.enoptrdata
   342  	}
   343  	if end < firstmoduledata.edata {
   344  		end = firstmoduledata.edata
   345  	}
   346  	if end < firstmoduledata.enoptrbss {
   347  		end = firstmoduledata.enoptrbss
   348  	}
   349  	if end < firstmoduledata.ebss {
   350  		end = firstmoduledata.ebss
   351  	}
   352  	size := round(end-start, _PageSize)
   353  	racecall(&__tsan_map_shadow, start, size, 0, 0)
   354  	racedatastart = start
   355  	racedataend = start + size
   356  
   357  	return
   358  }
   359  
   360  var raceFiniLock mutex
   361  
   362  //go:nosplit
   363  func racefini() {
   364  	// racefini() can only be called once to avoid races.
   365  	// This eventually (via __tsan_fini) calls C.exit which has
   366  	// undefined behavior if called more than once. If the lock is
   367  	// already held it's assumed that the first caller exits the program
   368  	// so other calls can hang forever without an issue.
   369  	lock(&raceFiniLock)
   370  	racecall(&__tsan_fini, 0, 0, 0, 0)
   371  }
   372  
   373  //go:nosplit
   374  func raceproccreate() uintptr {
   375  	var ctx uintptr
   376  	racecall(&__tsan_proc_create, uintptr(unsafe.Pointer(&ctx)), 0, 0, 0)
   377  	return ctx
   378  }
   379  
   380  //go:nosplit
   381  func raceprocdestroy(ctx uintptr) {
   382  	racecall(&__tsan_proc_destroy, ctx, 0, 0, 0)
   383  }
   384  
   385  //go:nosplit
   386  func racemapshadow(addr unsafe.Pointer, size uintptr) {
   387  	if racearenastart == 0 {
   388  		racearenastart = uintptr(addr)
   389  	}
   390  	if racearenaend < uintptr(addr)+size {
   391  		racearenaend = uintptr(addr) + size
   392  	}
   393  	racecall(&__tsan_map_shadow, uintptr(addr), size, 0, 0)
   394  }
   395  
   396  //go:nosplit
   397  func racemalloc(p unsafe.Pointer, sz uintptr) {
   398  	racecall(&__tsan_malloc, 0, 0, uintptr(p), sz)
   399  }
   400  
   401  //go:nosplit
   402  func racefree(p unsafe.Pointer, sz uintptr) {
   403  	racecall(&__tsan_free, uintptr(p), sz, 0, 0)
   404  }
   405  
   406  //go:nosplit
   407  func racegostart(pc uintptr) uintptr {
   408  	_g_ := getg()
   409  	var spawng *g
   410  	if _g_.m.curg != nil {
   411  		spawng = _g_.m.curg
   412  	} else {
   413  		spawng = _g_
   414  	}
   415  
   416  	var racectx uintptr
   417  	racecall(&__tsan_go_start, spawng.racectx, uintptr(unsafe.Pointer(&racectx)), pc, 0)
   418  	return racectx
   419  }
   420  
   421  //go:nosplit
   422  func racegoend() {
   423  	racecall(&__tsan_go_end, getg().racectx, 0, 0, 0)
   424  }
   425  
   426  //go:nosplit
   427  func racewriterangepc(addr unsafe.Pointer, sz, callpc, pc uintptr) {
   428  	_g_ := getg()
   429  	if _g_ != _g_.m.curg {
   430  		// The call is coming from manual instrumentation of Go code running on g0/gsignal.
   431  		// Not interesting.
   432  		return
   433  	}
   434  	if callpc != 0 {
   435  		racefuncenter(callpc)
   436  	}
   437  	racewriterangepc1(uintptr(addr), sz, pc)
   438  	if callpc != 0 {
   439  		racefuncexit()
   440  	}
   441  }
   442  
   443  //go:nosplit
   444  func racereadrangepc(addr unsafe.Pointer, sz, callpc, pc uintptr) {
   445  	_g_ := getg()
   446  	if _g_ != _g_.m.curg {
   447  		// The call is coming from manual instrumentation of Go code running on g0/gsignal.
   448  		// Not interesting.
   449  		return
   450  	}
   451  	if callpc != 0 {
   452  		racefuncenter(callpc)
   453  	}
   454  	racereadrangepc1(uintptr(addr), sz, pc)
   455  	if callpc != 0 {
   456  		racefuncexit()
   457  	}
   458  }
   459  
   460  //go:nosplit
   461  func raceacquire(addr unsafe.Pointer) {
   462  	raceacquireg(getg(), addr)
   463  }
   464  
   465  //go:nosplit
   466  func raceacquireg(gp *g, addr unsafe.Pointer) {
   467  	if getg().raceignore != 0 || !isvalidaddr(addr) {
   468  		return
   469  	}
   470  	racecall(&__tsan_acquire, gp.racectx, uintptr(addr), 0, 0)
   471  }
   472  
   473  //go:nosplit
   474  func racerelease(addr unsafe.Pointer) {
   475  	racereleaseg(getg(), addr)
   476  }
   477  
   478  //go:nosplit
   479  func racereleaseg(gp *g, addr unsafe.Pointer) {
   480  	if getg().raceignore != 0 || !isvalidaddr(addr) {
   481  		return
   482  	}
   483  	racecall(&__tsan_release, gp.racectx, uintptr(addr), 0, 0)
   484  }
   485  
   486  //go:nosplit
   487  func racereleasemerge(addr unsafe.Pointer) {
   488  	racereleasemergeg(getg(), addr)
   489  }
   490  
   491  //go:nosplit
   492  func racereleasemergeg(gp *g, addr unsafe.Pointer) {
   493  	if getg().raceignore != 0 || !isvalidaddr(addr) {
   494  		return
   495  	}
   496  	racecall(&__tsan_release_merge, gp.racectx, uintptr(addr), 0, 0)
   497  }
   498  
   499  //go:nosplit
   500  func racefingo() {
   501  	racecall(&__tsan_finalizer_goroutine, getg().racectx, 0, 0, 0)
   502  }
   503  
   504  // The declarations below generate ABI wrappers for functions
   505  // implemented in assembly in this package but declared in another
   506  // package.
   507  
   508  //go:linkname abigen_sync_atomic_LoadInt32 sync/atomic.LoadInt32
   509  func abigen_sync_atomic_LoadInt32(addr *int32) (val int32)
   510  
   511  //go:linkname abigen_sync_atomic_LoadInt64 sync/atomic.LoadInt64
   512  func abigen_sync_atomic_LoadInt64(addr *int64) (val int64)
   513  
   514  //go:linkname abigen_sync_atomic_LoadUint32 sync/atomic.LoadUint32
   515  func abigen_sync_atomic_LoadUint32(addr *uint32) (val uint32)
   516  
   517  //go:linkname abigen_sync_atomic_LoadUint64 sync/atomic.LoadUint64
   518  func abigen_sync_atomic_LoadUint64(addr *uint64) (val uint64)
   519  
   520  //go:linkname abigen_sync_atomic_LoadUintptr sync/atomic.LoadUintptr
   521  func abigen_sync_atomic_LoadUintptr(addr *uintptr) (val uintptr)
   522  
   523  //go:linkname abigen_sync_atomic_LoadPointer sync/atomic.LoadPointer
   524  func abigen_sync_atomic_LoadPointer(addr *unsafe.Pointer) (val unsafe.Pointer)
   525  
   526  //go:linkname abigen_sync_atomic_StoreInt32 sync/atomic.StoreInt32
   527  func abigen_sync_atomic_StoreInt32(addr *int32, val int32)
   528  
   529  //go:linkname abigen_sync_atomic_StoreInt64 sync/atomic.StoreInt64
   530  func abigen_sync_atomic_StoreInt64(addr *int64, val int64)
   531  
   532  //go:linkname abigen_sync_atomic_StoreUint32 sync/atomic.StoreUint32
   533  func abigen_sync_atomic_StoreUint32(addr *uint32, val uint32)
   534  
   535  //go:linkname abigen_sync_atomic_StoreUint64 sync/atomic.StoreUint64
   536  func abigen_sync_atomic_StoreUint64(addr *uint64, val uint64)
   537  
   538  //go:linkname abigen_sync_atomic_SwapInt32 sync/atomic.SwapInt32
   539  func abigen_sync_atomic_SwapInt32(addr *int32, new int32) (old int32)
   540  
   541  //go:linkname abigen_sync_atomic_SwapInt64 sync/atomic.SwapInt64
   542  func abigen_sync_atomic_SwapInt64(addr *int64, new int64) (old int64)
   543  
   544  //go:linkname abigen_sync_atomic_SwapUint32 sync/atomic.SwapUint32
   545  func abigen_sync_atomic_SwapUint32(addr *uint32, new uint32) (old uint32)
   546  
   547  //go:linkname abigen_sync_atomic_SwapUint64 sync/atomic.SwapUint64
   548  func abigen_sync_atomic_SwapUint64(addr *uint64, new uint64) (old uint64)
   549  
   550  //go:linkname abigen_sync_atomic_AddInt32 sync/atomic.AddInt32
   551  func abigen_sync_atomic_AddInt32(addr *int32, delta int32) (new int32)
   552  
   553  //go:linkname abigen_sync_atomic_AddUint32 sync/atomic.AddUint32
   554  func abigen_sync_atomic_AddUint32(addr *uint32, delta uint32) (new uint32)
   555  
   556  //go:linkname abigen_sync_atomic_AddInt64 sync/atomic.AddInt64
   557  func abigen_sync_atomic_AddInt64(addr *int64, delta int64) (new int64)
   558  
   559  //go:linkname abigen_sync_atomic_AddUint64 sync/atomic.AddUint64
   560  func abigen_sync_atomic_AddUint64(addr *uint64, delta uint64) (new uint64)
   561  
   562  //go:linkname abigen_sync_atomic_AddUintptr sync/atomic.AddUintptr
   563  func abigen_sync_atomic_AddUintptr(addr *uintptr, delta uintptr) (new uintptr)
   564  
   565  //go:linkname abigen_sync_atomic_CompareAndSwapInt32 sync/atomic.CompareAndSwapInt32
   566  func abigen_sync_atomic_CompareAndSwapInt32(addr *int32, old, new int32) (swapped bool)
   567  
   568  //go:linkname abigen_sync_atomic_CompareAndSwapInt64 sync/atomic.CompareAndSwapInt64
   569  func abigen_sync_atomic_CompareAndSwapInt64(addr *int64, old, new int64) (swapped bool)
   570  
   571  //go:linkname abigen_sync_atomic_CompareAndSwapUint32 sync/atomic.CompareAndSwapUint32
   572  func abigen_sync_atomic_CompareAndSwapUint32(addr *uint32, old, new uint32) (swapped bool)
   573  
   574  //go:linkname abigen_sync_atomic_CompareAndSwapUint64 sync/atomic.CompareAndSwapUint64
   575  func abigen_sync_atomic_CompareAndSwapUint64(addr *uint64, old, new uint64) (swapped bool)
   576  

View as plain text