Source file src/runtime/mwbbuf.go

Documentation: runtime

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // This implements the write barrier buffer. The write barrier itself
     6  // is gcWriteBarrier and is implemented in assembly.
     7  //
     8  // See mbarrier.go for algorithmic details on the write barrier. This
     9  // file deals only with the buffer.
    10  //
    11  // The write barrier has a fast path and a slow path. The fast path
    12  // simply enqueues to a per-P write barrier buffer. It's written in
    13  // assembly and doesn't clobber any general purpose registers, so it
    14  // doesn't have the usual overheads of a Go call.
    15  //
    16  // When the buffer fills up, the write barrier invokes the slow path
    17  // (wbBufFlush) to flush the buffer to the GC work queues. In this
    18  // path, since the compiler didn't spill registers, we spill *all*
    19  // registers and disallow any GC safe points that could observe the
    20  // stack frame (since we don't know the types of the spilled
    21  // registers).
    22  
    23  package runtime
    24  
    25  import (
    26  	"runtime/internal/atomic"
    27  	"runtime/internal/sys"
    28  	"unsafe"
    29  )
    30  
    31  // testSmallBuf forces a small write barrier buffer to stress write
    32  // barrier flushing.
    33  const testSmallBuf = false
    34  
    35  // wbBuf is a per-P buffer of pointers queued by the write barrier.
    36  // This buffer is flushed to the GC workbufs when it fills up and on
    37  // various GC transitions.
    38  //
    39  // This is closely related to a "sequential store buffer" (SSB),
    40  // except that SSBs are usually used for maintaining remembered sets,
    41  // while this is used for marking.
    42  type wbBuf struct {
    43  	// next points to the next slot in buf. It must not be a
    44  	// pointer type because it can point past the end of buf and
    45  	// must be updated without write barriers.
    46  	//
    47  	// This is a pointer rather than an index to optimize the
    48  	// write barrier assembly.
    49  	next uintptr
    50  
    51  	// end points to just past the end of buf. It must not be a
    52  	// pointer type because it points past the end of buf and must
    53  	// be updated without write barriers.
    54  	end uintptr
    55  
    56  	// buf stores a series of pointers to execute write barriers
    57  	// on. This must be a multiple of wbBufEntryPointers because
    58  	// the write barrier only checks for overflow once per entry.
    59  	buf [wbBufEntryPointers * wbBufEntries]uintptr
    60  
    61  	// debugGen causes the write barrier buffer to flush after
    62  	// every write barrier if equal to gcWorkPauseGen. This is for
    63  	// debugging #27993. This is only set if debugCachedWork is
    64  	// set.
    65  	debugGen uint32
    66  }
    67  
    68  const (
    69  	// wbBufEntries is the number of write barriers between
    70  	// flushes of the write barrier buffer.
    71  	//
    72  	// This trades latency for throughput amortization. Higher
    73  	// values amortize flushing overhead more, but increase the
    74  	// latency of flushing. Higher values also increase the cache
    75  	// footprint of the buffer.
    76  	//
    77  	// TODO: What is the latency cost of this? Tune this value.
    78  	wbBufEntries = 256
    79  
    80  	// wbBufEntryPointers is the number of pointers added to the
    81  	// buffer by each write barrier.
    82  	wbBufEntryPointers = 2
    83  )
    84  
    85  // reset empties b by resetting its next and end pointers.
    86  func (b *wbBuf) reset() {
    87  	start := uintptr(unsafe.Pointer(&b.buf[0]))
    88  	b.next = start
    89  	if writeBarrier.cgo || (debugCachedWork && (throwOnGCWork || b.debugGen == atomic.Load(&gcWorkPauseGen))) {
    90  		// Effectively disable the buffer by forcing a flush
    91  		// on every barrier.
    92  		b.end = uintptr(unsafe.Pointer(&b.buf[wbBufEntryPointers]))
    93  	} else if testSmallBuf {
    94  		// For testing, allow two barriers in the buffer. If
    95  		// we only did one, then barriers of non-heap pointers
    96  		// would be no-ops. This lets us combine a buffered
    97  		// barrier with a flush at a later time.
    98  		b.end = uintptr(unsafe.Pointer(&b.buf[2*wbBufEntryPointers]))
    99  	} else {
   100  		b.end = start + uintptr(len(b.buf))*unsafe.Sizeof(b.buf[0])
   101  	}
   102  
   103  	if (b.end-b.next)%(wbBufEntryPointers*unsafe.Sizeof(b.buf[0])) != 0 {
   104  		throw("bad write barrier buffer bounds")
   105  	}
   106  }
   107  
   108  // discard resets b's next pointer, but not its end pointer.
   109  //
   110  // This must be nosplit because it's called by wbBufFlush.
   111  //
   112  //go:nosplit
   113  func (b *wbBuf) discard() {
   114  	b.next = uintptr(unsafe.Pointer(&b.buf[0]))
   115  }
   116  
   117  // empty reports whether b contains no pointers.
   118  func (b *wbBuf) empty() bool {
   119  	return b.next == uintptr(unsafe.Pointer(&b.buf[0]))
   120  }
   121  
   122  // putFast adds old and new to the write barrier buffer and returns
   123  // false if a flush is necessary. Callers should use this as:
   124  //
   125  //     buf := &getg().m.p.ptr().wbBuf
   126  //     if !buf.putFast(old, new) {
   127  //         wbBufFlush(...)
   128  //     }
   129  //     ... actual memory write ...
   130  //
   131  // The arguments to wbBufFlush depend on whether the caller is doing
   132  // its own cgo pointer checks. If it is, then this can be
   133  // wbBufFlush(nil, 0). Otherwise, it must pass the slot address and
   134  // new.
   135  //
   136  // The caller must ensure there are no preemption points during the
   137  // above sequence. There must be no preemption points while buf is in
   138  // use because it is a per-P resource. There must be no preemption
   139  // points between the buffer put and the write to memory because this
   140  // could allow a GC phase change, which could result in missed write
   141  // barriers.
   142  //
   143  // putFast must be nowritebarrierrec to because write barriers here would
   144  // corrupt the write barrier buffer. It (and everything it calls, if
   145  // it called anything) has to be nosplit to avoid scheduling on to a
   146  // different P and a different buffer.
   147  //
   148  //go:nowritebarrierrec
   149  //go:nosplit
   150  func (b *wbBuf) putFast(old, new uintptr) bool {
   151  	p := (*[2]uintptr)(unsafe.Pointer(b.next))
   152  	p[0] = old
   153  	p[1] = new
   154  	b.next += 2 * sys.PtrSize
   155  	return b.next != b.end
   156  }
   157  
   158  // wbBufFlush flushes the current P's write barrier buffer to the GC
   159  // workbufs. It is passed the slot and value of the write barrier that
   160  // caused the flush so that it can implement cgocheck.
   161  //
   162  // This must not have write barriers because it is part of the write
   163  // barrier implementation.
   164  //
   165  // This and everything it calls must be nosplit because 1) the stack
   166  // contains untyped slots from gcWriteBarrier and 2) there must not be
   167  // a GC safe point between the write barrier test in the caller and
   168  // flushing the buffer.
   169  //
   170  // TODO: A "go:nosplitrec" annotation would be perfect for this.
   171  //
   172  //go:nowritebarrierrec
   173  //go:nosplit
   174  func wbBufFlush(dst *uintptr, src uintptr) {
   175  	// Note: Every possible return from this function must reset
   176  	// the buffer's next pointer to prevent buffer overflow.
   177  
   178  	// This *must not* modify its arguments because this
   179  	// function's argument slots do double duty in gcWriteBarrier
   180  	// as register spill slots. Currently, not modifying the
   181  	// arguments is sufficient to keep the spill slots unmodified
   182  	// (which seems unlikely to change since it costs little and
   183  	// helps with debugging).
   184  
   185  	if getg().m.dying > 0 {
   186  		// We're going down. Not much point in write barriers
   187  		// and this way we can allow write barriers in the
   188  		// panic path.
   189  		getg().m.p.ptr().wbBuf.discard()
   190  		return
   191  	}
   192  
   193  	if writeBarrier.cgo && dst != nil {
   194  		// This must be called from the stack that did the
   195  		// write. It's nosplit all the way down.
   196  		cgoCheckWriteBarrier(dst, src)
   197  		if !writeBarrier.needed {
   198  			// We were only called for cgocheck.
   199  			getg().m.p.ptr().wbBuf.discard()
   200  			return
   201  		}
   202  	}
   203  
   204  	// Switch to the system stack so we don't have to worry about
   205  	// the untyped stack slots or safe points.
   206  	systemstack(func() {
   207  		if debugCachedWork {
   208  			// For debugging, include the old value of the
   209  			// slot and some other data in the traceback.
   210  			wbBuf := &getg().m.p.ptr().wbBuf
   211  			var old uintptr
   212  			if dst != nil {
   213  				// dst may be nil in direct calls to wbBufFlush.
   214  				old = *dst
   215  			}
   216  			wbBufFlush1Debug(old, wbBuf.buf[0], wbBuf.buf[1], &wbBuf.buf[0], wbBuf.next)
   217  		} else {
   218  			wbBufFlush1(getg().m.p.ptr())
   219  		}
   220  	})
   221  }
   222  
   223  // wbBufFlush1Debug is a temporary function for debugging issue
   224  // #27993. It exists solely to add some context to the traceback.
   225  //
   226  //go:nowritebarrierrec
   227  //go:systemstack
   228  //go:noinline
   229  func wbBufFlush1Debug(old, buf1, buf2 uintptr, start *uintptr, next uintptr) {
   230  	wbBufFlush1(getg().m.p.ptr())
   231  }
   232  
   233  // wbBufFlush1 flushes p's write barrier buffer to the GC work queue.
   234  //
   235  // This must not have write barriers because it is part of the write
   236  // barrier implementation, so this may lead to infinite loops or
   237  // buffer corruption.
   238  //
   239  // This must be non-preemptible because it uses the P's workbuf.
   240  //
   241  //go:nowritebarrierrec
   242  //go:systemstack
   243  func wbBufFlush1(_p_ *p) {
   244  	// Get the buffered pointers.
   245  	start := uintptr(unsafe.Pointer(&_p_.wbBuf.buf[0]))
   246  	n := (_p_.wbBuf.next - start) / unsafe.Sizeof(_p_.wbBuf.buf[0])
   247  	ptrs := _p_.wbBuf.buf[:n]
   248  
   249  	// Poison the buffer to make extra sure nothing is enqueued
   250  	// while we're processing the buffer.
   251  	_p_.wbBuf.next = 0
   252  
   253  	if useCheckmark {
   254  		// Slow path for checkmark mode.
   255  		for _, ptr := range ptrs {
   256  			shade(ptr)
   257  		}
   258  		_p_.wbBuf.reset()
   259  		return
   260  	}
   261  
   262  	// Mark all of the pointers in the buffer and record only the
   263  	// pointers we greyed. We use the buffer itself to temporarily
   264  	// record greyed pointers.
   265  	//
   266  	// TODO: Should scanobject/scanblock just stuff pointers into
   267  	// the wbBuf? Then this would become the sole greying path.
   268  	//
   269  	// TODO: We could avoid shading any of the "new" pointers in
   270  	// the buffer if the stack has been shaded, or even avoid
   271  	// putting them in the buffer at all (which would double its
   272  	// capacity). This is slightly complicated with the buffer; we
   273  	// could track whether any un-shaded goroutine has used the
   274  	// buffer, or just track globally whether there are any
   275  	// un-shaded stacks and flush after each stack scan.
   276  	gcw := &_p_.gcw
   277  	pos := 0
   278  	for _, ptr := range ptrs {
   279  		if ptr < minLegalPointer {
   280  			// nil pointers are very common, especially
   281  			// for the "old" values. Filter out these and
   282  			// other "obvious" non-heap pointers ASAP.
   283  			//
   284  			// TODO: Should we filter out nils in the fast
   285  			// path to reduce the rate of flushes?
   286  			continue
   287  		}
   288  		obj, span, objIndex := findObject(ptr, 0, 0)
   289  		if obj == 0 {
   290  			continue
   291  		}
   292  		// TODO: Consider making two passes where the first
   293  		// just prefetches the mark bits.
   294  		mbits := span.markBitsForIndex(objIndex)
   295  		if mbits.isMarked() {
   296  			continue
   297  		}
   298  		mbits.setMarked()
   299  		if span.spanclass.noscan() {
   300  			gcw.bytesMarked += uint64(span.elemsize)
   301  			continue
   302  		}
   303  		ptrs[pos] = obj
   304  		pos++
   305  	}
   306  
   307  	// Enqueue the greyed objects.
   308  	gcw.putBatch(ptrs[:pos])
   309  
   310  	_p_.wbBuf.reset()
   311  }
   312  

View as plain text