...
Run Format

Source file src/runtime/mbarrier.go

     1	// Copyright 2015 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// Garbage collector: write barriers.
     6	//
     7	// For the concurrent garbage collector, the Go compiler implements
     8	// updates to pointer-valued fields that may be in heap objects by
     9	// emitting calls to write barriers. This file contains the actual write barrier
    10	// implementation, gcmarkwb_m, and the various wrappers called by the
    11	// compiler to implement pointer assignment, slice assignment,
    12	// typed memmove, and so on.
    13	
    14	package runtime
    15	
    16	import (
    17		"runtime/internal/sys"
    18		"unsafe"
    19	)
    20	
    21	// gcmarkwb_m is the mark-phase write barrier, the only barrier we have.
    22	// The rest of this file exists only to make calls to this function.
    23	//
    24	// This is a hybrid barrier that combines a Yuasa-style deletion
    25	// barrier—which shades the object whose reference is being
    26	// overwritten—with Dijkstra insertion barrier—which shades the object
    27	// whose reference is being written. The insertion part of the barrier
    28	// is necessary while the calling goroutine's stack is grey. In
    29	// pseudocode, the barrier is:
    30	//
    31	//     writePointer(slot, ptr):
    32	//         shade(*slot)
    33	//         if current stack is grey:
    34	//             shade(ptr)
    35	//         *slot = ptr
    36	//
    37	// slot is the destination in Go code.
    38	// ptr is the value that goes into the slot in Go code.
    39	//
    40	// Shade indicates that it has seen a white pointer by adding the referent
    41	// to wbuf as well as marking it.
    42	//
    43	// The two shades and the condition work together to prevent a mutator
    44	// from hiding an object from the garbage collector:
    45	//
    46	// 1. shade(*slot) prevents a mutator from hiding an object by moving
    47	// the sole pointer to it from the heap to its stack. If it attempts
    48	// to unlink an object from the heap, this will shade it.
    49	//
    50	// 2. shade(ptr) prevents a mutator from hiding an object by moving
    51	// the sole pointer to it from its stack into a black object in the
    52	// heap. If it attempts to install the pointer into a black object,
    53	// this will shade it.
    54	//
    55	// 3. Once a goroutine's stack is black, the shade(ptr) becomes
    56	// unnecessary. shade(ptr) prevents hiding an object by moving it from
    57	// the stack to the heap, but this requires first having a pointer
    58	// hidden on the stack. Immediately after a stack is scanned, it only
    59	// points to shaded objects, so it's not hiding anything, and the
    60	// shade(*slot) prevents it from hiding any other pointers on its
    61	// stack.
    62	//
    63	// For a detailed description of this barrier and proof of
    64	// correctness, see https://github.com/golang/proposal/blob/master/design/17503-eliminate-rescan.md
    65	//
    66	//
    67	//
    68	// Dealing with memory ordering:
    69	//
    70	// Both the Yuasa and Dijkstra barriers can be made conditional on the
    71	// color of the object containing the slot. We chose not to make these
    72	// conditional because the cost of ensuring that the object holding
    73	// the slot doesn't concurrently change color without the mutator
    74	// noticing seems prohibitive.
    75	//
    76	// Consider the following example where the mutator writes into
    77	// a slot and then loads the slot's mark bit while the GC thread
    78	// writes to the slot's mark bit and then as part of scanning reads
    79	// the slot.
    80	//
    81	// Initially both [slot] and [slotmark] are 0 (nil)
    82	// Mutator thread          GC thread
    83	// st [slot], ptr          st [slotmark], 1
    84	//
    85	// ld r1, [slotmark]       ld r2, [slot]
    86	//
    87	// Without an expensive memory barrier between the st and the ld, the final
    88	// result on most HW (including 386/amd64) can be r1==r2==0. This is a classic
    89	// example of what can happen when loads are allowed to be reordered with older
    90	// stores (avoiding such reorderings lies at the heart of the classic
    91	// Peterson/Dekker algorithms for mutual exclusion). Rather than require memory
    92	// barriers, which will slow down both the mutator and the GC, we always grey
    93	// the ptr object regardless of the slot's color.
    94	//
    95	// Another place where we intentionally omit memory barriers is when
    96	// accessing mheap_.arena_used to check if a pointer points into the
    97	// heap. On relaxed memory machines, it's possible for a mutator to
    98	// extend the size of the heap by updating arena_used, allocate an
    99	// object from this new region, and publish a pointer to that object,
   100	// but for tracing running on another processor to observe the pointer
   101	// but use the old value of arena_used. In this case, tracing will not
   102	// mark the object, even though it's reachable. However, the mutator
   103	// is guaranteed to execute a write barrier when it publishes the
   104	// pointer, so it will take care of marking the object. A general
   105	// consequence of this is that the garbage collector may cache the
   106	// value of mheap_.arena_used. (See issue #9984.)
   107	//
   108	//
   109	// Stack writes:
   110	//
   111	// The compiler omits write barriers for writes to the current frame,
   112	// but if a stack pointer has been passed down the call stack, the
   113	// compiler will generate a write barrier for writes through that
   114	// pointer (because it doesn't know it's not a heap pointer).
   115	//
   116	// One might be tempted to ignore the write barrier if slot points
   117	// into to the stack. Don't do it! Mark termination only re-scans
   118	// frames that have potentially been active since the concurrent scan,
   119	// so it depends on write barriers to track changes to pointers in
   120	// stack frames that have not been active.
   121	//
   122	//
   123	// Global writes:
   124	//
   125	// The Go garbage collector requires write barriers when heap pointers
   126	// are stored in globals. Many garbage collectors ignore writes to
   127	// globals and instead pick up global -> heap pointers during
   128	// termination. This increases pause time, so we instead rely on write
   129	// barriers for writes to globals so that we don't have to rescan
   130	// global during mark termination.
   131	//
   132	//
   133	// Publication ordering:
   134	//
   135	// The write barrier is *pre-publication*, meaning that the write
   136	// barrier happens prior to the *slot = ptr write that may make ptr
   137	// reachable by some goroutine that currently cannot reach it.
   138	//
   139	//
   140	//go:nowritebarrierrec
   141	//go:systemstack
   142	func gcmarkwb_m(slot *uintptr, ptr uintptr) {
   143		if writeBarrier.needed {
   144			// Note: This turns bad pointer writes into bad
   145			// pointer reads, which could be confusing. We avoid
   146			// reading from obviously bad pointers, which should
   147			// take care of the vast majority of these. We could
   148			// patch this up in the signal handler, or use XCHG to
   149			// combine the read and the write. Checking inheap is
   150			// insufficient since we need to track changes to
   151			// roots outside the heap.
   152			if slot1 := uintptr(unsafe.Pointer(slot)); slot1 >= minPhysPageSize {
   153				if optr := *slot; optr != 0 {
   154					shade(optr)
   155				}
   156			}
   157			// TODO: Make this conditional on the caller's stack color.
   158			if ptr != 0 && inheap(ptr) {
   159				shade(ptr)
   160			}
   161		}
   162	}
   163	
   164	// writebarrierptr_prewrite1 invokes a write barrier for *dst = src
   165	// prior to the write happening.
   166	//
   167	// Write barrier calls must not happen during critical GC and scheduler
   168	// related operations. In particular there are times when the GC assumes
   169	// that the world is stopped but scheduler related code is still being
   170	// executed, dealing with syscalls, dealing with putting gs on runnable
   171	// queues and so forth. This code cannot execute write barriers because
   172	// the GC might drop them on the floor. Stopping the world involves removing
   173	// the p associated with an m. We use the fact that m.p == nil to indicate
   174	// that we are in one these critical section and throw if the write is of
   175	// a pointer to a heap object.
   176	//go:nosplit
   177	func writebarrierptr_prewrite1(dst *uintptr, src uintptr) {
   178		mp := acquirem()
   179		if mp.inwb || mp.dying > 0 {
   180			releasem(mp)
   181			return
   182		}
   183		systemstack(func() {
   184			if mp.p == 0 && memstats.enablegc && !mp.inwb && inheap(src) {
   185				throw("writebarrierptr_prewrite1 called with mp.p == nil")
   186			}
   187			mp.inwb = true
   188			gcmarkwb_m(dst, src)
   189		})
   190		mp.inwb = false
   191		releasem(mp)
   192	}
   193	
   194	// NOTE: Really dst *unsafe.Pointer, src unsafe.Pointer,
   195	// but if we do that, Go inserts a write barrier on *dst = src.
   196	//go:nosplit
   197	func writebarrierptr(dst *uintptr, src uintptr) {
   198		if writeBarrier.cgo {
   199			cgoCheckWriteBarrier(dst, src)
   200		}
   201		if !writeBarrier.needed {
   202			*dst = src
   203			return
   204		}
   205		if src != 0 && src < minPhysPageSize {
   206			systemstack(func() {
   207				print("runtime: writebarrierptr *", dst, " = ", hex(src), "\n")
   208				throw("bad pointer in write barrier")
   209			})
   210		}
   211		writebarrierptr_prewrite1(dst, src)
   212		*dst = src
   213	}
   214	
   215	// writebarrierptr_prewrite is like writebarrierptr, but the store
   216	// will be performed by the caller after this call. The caller must
   217	// not allow preemption between this call and the write.
   218	//
   219	//go:nosplit
   220	func writebarrierptr_prewrite(dst *uintptr, src uintptr) {
   221		if writeBarrier.cgo {
   222			cgoCheckWriteBarrier(dst, src)
   223		}
   224		if !writeBarrier.needed {
   225			return
   226		}
   227		if src != 0 && src < minPhysPageSize {
   228			systemstack(func() { throw("bad pointer in write barrier") })
   229		}
   230		writebarrierptr_prewrite1(dst, src)
   231	}
   232	
   233	// typedmemmove copies a value of type t to dst from src.
   234	//go:nosplit
   235	func typedmemmove(typ *_type, dst, src unsafe.Pointer) {
   236		if typ.kind&kindNoPointers == 0 {
   237			bulkBarrierPreWrite(uintptr(dst), uintptr(src), typ.size)
   238		}
   239		// There's a race here: if some other goroutine can write to
   240		// src, it may change some pointer in src after we've
   241		// performed the write barrier but before we perform the
   242		// memory copy. This safe because the write performed by that
   243		// other goroutine must also be accompanied by a write
   244		// barrier, so at worst we've unnecessarily greyed the old
   245		// pointer that was in src.
   246		memmove(dst, src, typ.size)
   247		if writeBarrier.cgo {
   248			cgoCheckMemmove(typ, dst, src, 0, typ.size)
   249		}
   250	}
   251	
   252	//go:linkname reflect_typedmemmove reflect.typedmemmove
   253	func reflect_typedmemmove(typ *_type, dst, src unsafe.Pointer) {
   254		if raceenabled {
   255			raceWriteObjectPC(typ, dst, getcallerpc(unsafe.Pointer(&typ)), funcPC(reflect_typedmemmove))
   256			raceReadObjectPC(typ, src, getcallerpc(unsafe.Pointer(&typ)), funcPC(reflect_typedmemmove))
   257		}
   258		if msanenabled {
   259			msanwrite(dst, typ.size)
   260			msanread(src, typ.size)
   261		}
   262		typedmemmove(typ, dst, src)
   263	}
   264	
   265	// typedmemmovepartial is like typedmemmove but assumes that
   266	// dst and src point off bytes into the value and only copies size bytes.
   267	//go:linkname reflect_typedmemmovepartial reflect.typedmemmovepartial
   268	func reflect_typedmemmovepartial(typ *_type, dst, src unsafe.Pointer, off, size uintptr) {
   269		if writeBarrier.needed && typ.kind&kindNoPointers == 0 && size >= sys.PtrSize {
   270			// Pointer-align start address for bulk barrier.
   271			adst, asrc, asize := dst, src, size
   272			if frag := -off & (sys.PtrSize - 1); frag != 0 {
   273				adst = add(dst, frag)
   274				asrc = add(src, frag)
   275				asize -= frag
   276			}
   277			bulkBarrierPreWrite(uintptr(adst), uintptr(asrc), asize&^(sys.PtrSize-1))
   278		}
   279	
   280		memmove(dst, src, size)
   281		if writeBarrier.cgo {
   282			cgoCheckMemmove(typ, dst, src, off, size)
   283		}
   284	}
   285	
   286	// reflectcallmove is invoked by reflectcall to copy the return values
   287	// out of the stack and into the heap, invoking the necessary write
   288	// barriers. dst, src, and size describe the return value area to
   289	// copy. typ describes the entire frame (not just the return values).
   290	// typ may be nil, which indicates write barriers are not needed.
   291	//
   292	// It must be nosplit and must only call nosplit functions because the
   293	// stack map of reflectcall is wrong.
   294	//
   295	//go:nosplit
   296	func reflectcallmove(typ *_type, dst, src unsafe.Pointer, size uintptr) {
   297		if writeBarrier.needed && typ != nil && typ.kind&kindNoPointers == 0 && size >= sys.PtrSize {
   298			bulkBarrierPreWrite(uintptr(dst), uintptr(src), size)
   299		}
   300		memmove(dst, src, size)
   301	}
   302	
   303	//go:nosplit
   304	func typedslicecopy(typ *_type, dst, src slice) int {
   305		// TODO(rsc): If typedslicecopy becomes faster than calling
   306		// typedmemmove repeatedly, consider using during func growslice.
   307		n := dst.len
   308		if n > src.len {
   309			n = src.len
   310		}
   311		if n == 0 {
   312			return 0
   313		}
   314		dstp := dst.array
   315		srcp := src.array
   316	
   317		if raceenabled {
   318			callerpc := getcallerpc(unsafe.Pointer(&typ))
   319			pc := funcPC(slicecopy)
   320			racewriterangepc(dstp, uintptr(n)*typ.size, callerpc, pc)
   321			racereadrangepc(srcp, uintptr(n)*typ.size, callerpc, pc)
   322		}
   323		if msanenabled {
   324			msanwrite(dstp, uintptr(n)*typ.size)
   325			msanread(srcp, uintptr(n)*typ.size)
   326		}
   327	
   328		if writeBarrier.cgo {
   329			cgoCheckSliceCopy(typ, dst, src, n)
   330		}
   331	
   332		// Note: No point in checking typ.kind&kindNoPointers here:
   333		// compiler only emits calls to typedslicecopy for types with pointers,
   334		// and growslice and reflect_typedslicecopy check for pointers
   335		// before calling typedslicecopy.
   336		if !writeBarrier.needed {
   337			memmove(dstp, srcp, uintptr(n)*typ.size)
   338			return n
   339		}
   340	
   341		systemstack(func() {
   342			if uintptr(srcp) < uintptr(dstp) && uintptr(srcp)+uintptr(n)*typ.size > uintptr(dstp) {
   343				// Overlap with src before dst.
   344				// Copy backward, being careful not to move dstp/srcp
   345				// out of the array they point into.
   346				dstp = add(dstp, uintptr(n-1)*typ.size)
   347				srcp = add(srcp, uintptr(n-1)*typ.size)
   348				i := 0
   349				for {
   350					typedmemmove(typ, dstp, srcp)
   351					if i++; i >= n {
   352						break
   353					}
   354					dstp = add(dstp, -typ.size)
   355					srcp = add(srcp, -typ.size)
   356				}
   357			} else {
   358				// Copy forward, being careful not to move dstp/srcp
   359				// out of the array they point into.
   360				i := 0
   361				for {
   362					typedmemmove(typ, dstp, srcp)
   363					if i++; i >= n {
   364						break
   365					}
   366					dstp = add(dstp, typ.size)
   367					srcp = add(srcp, typ.size)
   368				}
   369			}
   370		})
   371		return n
   372	}
   373	
   374	//go:linkname reflect_typedslicecopy reflect.typedslicecopy
   375	func reflect_typedslicecopy(elemType *_type, dst, src slice) int {
   376		if elemType.kind&kindNoPointers != 0 {
   377			n := dst.len
   378			if n > src.len {
   379				n = src.len
   380			}
   381			if n == 0 {
   382				return 0
   383			}
   384	
   385			size := uintptr(n) * elemType.size
   386			if raceenabled {
   387				callerpc := getcallerpc(unsafe.Pointer(&elemType))
   388				pc := funcPC(reflect_typedslicecopy)
   389				racewriterangepc(dst.array, size, callerpc, pc)
   390				racereadrangepc(src.array, size, callerpc, pc)
   391			}
   392			if msanenabled {
   393				msanwrite(dst.array, size)
   394				msanread(src.array, size)
   395			}
   396	
   397			memmove(dst.array, src.array, size)
   398			return n
   399		}
   400		return typedslicecopy(elemType, dst, src)
   401	}
   402	
   403	// typedmemclr clears the typed memory at ptr with type typ. The
   404	// memory at ptr must already be initialized (and hence in type-safe
   405	// state). If the memory is being initialized for the first time, see
   406	// memclrNoHeapPointers.
   407	//
   408	// If the caller knows that typ has pointers, it can alternatively
   409	// call memclrHasPointers.
   410	//
   411	//go:nosplit
   412	func typedmemclr(typ *_type, ptr unsafe.Pointer) {
   413		if typ.kind&kindNoPointers == 0 {
   414			bulkBarrierPreWrite(uintptr(ptr), 0, typ.size)
   415		}
   416		memclrNoHeapPointers(ptr, typ.size)
   417	}
   418	
   419	// memclrHasPointers clears n bytes of typed memory starting at ptr.
   420	// The caller must ensure that the type of the object at ptr has
   421	// pointers, usually by checking typ.kind&kindNoPointers. However, ptr
   422	// does not have to point to the start of the allocation.
   423	//
   424	//go:nosplit
   425	func memclrHasPointers(ptr unsafe.Pointer, n uintptr) {
   426		bulkBarrierPreWrite(uintptr(ptr), 0, n)
   427		memclrNoHeapPointers(ptr, n)
   428	}
   429	

View as plain text