...
Run Format

Source file src/cmd/compile/internal/ssa/writebarrier.go

Documentation: cmd/compile/internal/ssa

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ssa
     6  
     7  import (
     8  	"cmd/compile/internal/types"
     9  	"cmd/internal/obj"
    10  	"cmd/internal/src"
    11  	"strings"
    12  )
    13  
    14  // needwb reports whether we need write barrier for store op v.
    15  // v must be Store/Move/Zero.
    16  func needwb(v *Value) bool {
    17  	t, ok := v.Aux.(*types.Type)
    18  	if !ok {
    19  		v.Fatalf("store aux is not a type: %s", v.LongString())
    20  	}
    21  	if !t.HasHeapPointer() {
    22  		return false
    23  	}
    24  	if IsStackAddr(v.Args[0]) {
    25  		return false // write on stack doesn't need write barrier
    26  	}
    27  	if v.Op == OpStore && IsGlobalAddr(v.Args[1]) && IsNewObject(v.Args[0], v.MemoryArg()) {
    28  		// Storing pointers to non-heap locations into a fresh object doesn't need a write barrier.
    29  		return false
    30  	}
    31  	if v.Op == OpMove && IsReadOnlyGlobalAddr(v.Args[1]) && IsNewObject(v.Args[0], v.MemoryArg()) {
    32  		// Copying data from readonly memory into a fresh object doesn't need a write barrier.
    33  		return false
    34  	}
    35  	return true
    36  }
    37  
    38  // writebarrier pass inserts write barriers for store ops (Store, Move, Zero)
    39  // when necessary (the condition above). It rewrites store ops to branches
    40  // and runtime calls, like
    41  //
    42  // if writeBarrier.enabled {
    43  //   gcWriteBarrier(ptr, val)	// Not a regular Go call
    44  // } else {
    45  //   *ptr = val
    46  // }
    47  //
    48  // A sequence of WB stores for many pointer fields of a single type will
    49  // be emitted together, with a single branch.
    50  func writebarrier(f *Func) {
    51  	if !f.fe.UseWriteBarrier() {
    52  		return
    53  	}
    54  
    55  	var sb, sp, wbaddr, const0 *Value
    56  	var typedmemmove, typedmemclr, gcWriteBarrier *obj.LSym
    57  	var stores, after []*Value
    58  	var sset *sparseSet
    59  	var storeNumber []int32
    60  
    61  	for _, b := range f.Blocks { // range loop is safe since the blocks we added contain no stores to expand
    62  		// first, identify all the stores that need to insert a write barrier.
    63  		// mark them with WB ops temporarily. record presence of WB ops.
    64  		nWBops := 0 // count of temporarily created WB ops remaining to be rewritten in the current block
    65  		for _, v := range b.Values {
    66  			switch v.Op {
    67  			case OpStore, OpMove, OpZero:
    68  				if needwb(v) {
    69  					switch v.Op {
    70  					case OpStore:
    71  						v.Op = OpStoreWB
    72  					case OpMove:
    73  						v.Op = OpMoveWB
    74  					case OpZero:
    75  						v.Op = OpZeroWB
    76  					}
    77  					nWBops++
    78  				}
    79  			}
    80  		}
    81  		if nWBops == 0 {
    82  			continue
    83  		}
    84  
    85  		if wbaddr == nil {
    86  			// lazily initialize global values for write barrier test and calls
    87  			// find SB and SP values in entry block
    88  			initpos := f.Entry.Pos
    89  			for _, v := range f.Entry.Values {
    90  				if v.Op == OpSB {
    91  					sb = v
    92  				}
    93  				if v.Op == OpSP {
    94  					sp = v
    95  				}
    96  				if sb != nil && sp != nil {
    97  					break
    98  				}
    99  			}
   100  			if sb == nil {
   101  				sb = f.Entry.NewValue0(initpos, OpSB, f.Config.Types.Uintptr)
   102  			}
   103  			if sp == nil {
   104  				sp = f.Entry.NewValue0(initpos, OpSP, f.Config.Types.Uintptr)
   105  			}
   106  			wbsym := f.fe.Syslook("writeBarrier")
   107  			wbaddr = f.Entry.NewValue1A(initpos, OpAddr, f.Config.Types.UInt32Ptr, wbsym, sb)
   108  			gcWriteBarrier = f.fe.Syslook("gcWriteBarrier")
   109  			typedmemmove = f.fe.Syslook("typedmemmove")
   110  			typedmemclr = f.fe.Syslook("typedmemclr")
   111  			const0 = f.ConstInt32(f.Config.Types.UInt32, 0)
   112  
   113  			// allocate auxiliary data structures for computing store order
   114  			sset = f.newSparseSet(f.NumValues())
   115  			defer f.retSparseSet(sset)
   116  			storeNumber = make([]int32, f.NumValues())
   117  		}
   118  
   119  		// order values in store order
   120  		b.Values = storeOrder(b.Values, sset, storeNumber)
   121  
   122  		firstSplit := true
   123  	again:
   124  		// find the start and end of the last contiguous WB store sequence.
   125  		// a branch will be inserted there. values after it will be moved
   126  		// to a new block.
   127  		var last *Value
   128  		var start, end int
   129  		values := b.Values
   130  	FindSeq:
   131  		for i := len(values) - 1; i >= 0; i-- {
   132  			w := values[i]
   133  			switch w.Op {
   134  			case OpStoreWB, OpMoveWB, OpZeroWB:
   135  				start = i
   136  				if last == nil {
   137  					last = w
   138  					end = i + 1
   139  				}
   140  			case OpVarDef, OpVarLive, OpVarKill:
   141  				continue
   142  			default:
   143  				if last == nil {
   144  					continue
   145  				}
   146  				break FindSeq
   147  			}
   148  		}
   149  		stores = append(stores[:0], b.Values[start:end]...) // copy to avoid aliasing
   150  		after = append(after[:0], b.Values[end:]...)
   151  		b.Values = b.Values[:start]
   152  
   153  		// find the memory before the WB stores
   154  		mem := stores[0].MemoryArg()
   155  		pos := stores[0].Pos
   156  		bThen := f.NewBlock(BlockPlain)
   157  		bElse := f.NewBlock(BlockPlain)
   158  		bEnd := f.NewBlock(b.Kind)
   159  		bThen.Pos = pos
   160  		bElse.Pos = pos
   161  		bEnd.Pos = b.Pos
   162  		b.Pos = pos
   163  
   164  		// set up control flow for end block
   165  		bEnd.SetControl(b.Control)
   166  		bEnd.Likely = b.Likely
   167  		for _, e := range b.Succs {
   168  			bEnd.Succs = append(bEnd.Succs, e)
   169  			e.b.Preds[e.i].b = bEnd
   170  		}
   171  
   172  		// set up control flow for write barrier test
   173  		// load word, test word, avoiding partial register write from load byte.
   174  		cfgtypes := &f.Config.Types
   175  		flag := b.NewValue2(pos, OpLoad, cfgtypes.UInt32, wbaddr, mem)
   176  		flag = b.NewValue2(pos, OpNeq32, cfgtypes.Bool, flag, const0)
   177  		b.Kind = BlockIf
   178  		b.SetControl(flag)
   179  		b.Likely = BranchUnlikely
   180  		b.Succs = b.Succs[:0]
   181  		b.AddEdgeTo(bThen)
   182  		b.AddEdgeTo(bElse)
   183  		// TODO: For OpStoreWB and the buffered write barrier,
   184  		// we could move the write out of the write barrier,
   185  		// which would lead to fewer branches. We could do
   186  		// something similar to OpZeroWB, since the runtime
   187  		// could provide just the barrier half and then we
   188  		// could unconditionally do an OpZero (which could
   189  		// also generate better zeroing code). OpMoveWB is
   190  		// trickier and would require changing how
   191  		// cgoCheckMemmove works.
   192  		bThen.AddEdgeTo(bEnd)
   193  		bElse.AddEdgeTo(bEnd)
   194  
   195  		// for each write barrier store, append write barrier version to bThen
   196  		// and simple store version to bElse
   197  		memThen := mem
   198  		memElse := mem
   199  
   200  		// If the source of a MoveWB is volatile (will be clobbered by a
   201  		// function call), we need to copy it to a temporary location, as
   202  		// marshaling the args of typedmemmove might clobber the value we're
   203  		// trying to move.
   204  		// Look for volatile source, copy it to temporary before we emit any
   205  		// call.
   206  		// It is unlikely to have more than one of them. Just do a linear
   207  		// search instead of using a map.
   208  		type volatileCopy struct {
   209  			src *Value // address of original volatile value
   210  			tmp *Value // address of temporary we've copied the volatile value into
   211  		}
   212  		var volatiles []volatileCopy
   213  	copyLoop:
   214  		for _, w := range stores {
   215  			if w.Op == OpMoveWB {
   216  				val := w.Args[1]
   217  				if isVolatile(val) {
   218  					for _, c := range volatiles {
   219  						if val == c.src {
   220  							continue copyLoop // already copied
   221  						}
   222  					}
   223  
   224  					t := val.Type.Elem()
   225  					tmp := f.fe.Auto(w.Pos, t)
   226  					memThen = bThen.NewValue1A(w.Pos, OpVarDef, types.TypeMem, tmp, memThen)
   227  					tmpaddr := bThen.NewValue2A(w.Pos, OpLocalAddr, t.PtrTo(), tmp, sp, memThen)
   228  					siz := t.Size()
   229  					memThen = bThen.NewValue3I(w.Pos, OpMove, types.TypeMem, siz, tmpaddr, val, memThen)
   230  					memThen.Aux = t
   231  					volatiles = append(volatiles, volatileCopy{val, tmpaddr})
   232  				}
   233  			}
   234  		}
   235  
   236  		for _, w := range stores {
   237  			ptr := w.Args[0]
   238  			pos := w.Pos
   239  
   240  			var fn *obj.LSym
   241  			var typ *obj.LSym
   242  			var val *Value
   243  			switch w.Op {
   244  			case OpStoreWB:
   245  				val = w.Args[1]
   246  				nWBops--
   247  			case OpMoveWB:
   248  				fn = typedmemmove
   249  				val = w.Args[1]
   250  				typ = w.Aux.(*types.Type).Symbol()
   251  				nWBops--
   252  			case OpZeroWB:
   253  				fn = typedmemclr
   254  				typ = w.Aux.(*types.Type).Symbol()
   255  				nWBops--
   256  			case OpVarDef, OpVarLive, OpVarKill:
   257  			}
   258  
   259  			// then block: emit write barrier call
   260  			switch w.Op {
   261  			case OpStoreWB, OpMoveWB, OpZeroWB:
   262  				if w.Op == OpStoreWB {
   263  					memThen = bThen.NewValue3A(pos, OpWB, types.TypeMem, gcWriteBarrier, ptr, val, memThen)
   264  				} else {
   265  					srcval := val
   266  					if w.Op == OpMoveWB && isVolatile(srcval) {
   267  						for _, c := range volatiles {
   268  							if srcval == c.src {
   269  								srcval = c.tmp
   270  								break
   271  							}
   272  						}
   273  					}
   274  					memThen = wbcall(pos, bThen, fn, typ, ptr, srcval, memThen, sp, sb)
   275  				}
   276  				// Note that we set up a writebarrier function call.
   277  				f.fe.SetWBPos(pos)
   278  			case OpVarDef, OpVarLive, OpVarKill:
   279  				memThen = bThen.NewValue1A(pos, w.Op, types.TypeMem, w.Aux, memThen)
   280  			}
   281  
   282  			// else block: normal store
   283  			switch w.Op {
   284  			case OpStoreWB:
   285  				memElse = bElse.NewValue3A(pos, OpStore, types.TypeMem, w.Aux, ptr, val, memElse)
   286  			case OpMoveWB:
   287  				memElse = bElse.NewValue3I(pos, OpMove, types.TypeMem, w.AuxInt, ptr, val, memElse)
   288  				memElse.Aux = w.Aux
   289  			case OpZeroWB:
   290  				memElse = bElse.NewValue2I(pos, OpZero, types.TypeMem, w.AuxInt, ptr, memElse)
   291  				memElse.Aux = w.Aux
   292  			case OpVarDef, OpVarLive, OpVarKill:
   293  				memElse = bElse.NewValue1A(pos, w.Op, types.TypeMem, w.Aux, memElse)
   294  			}
   295  		}
   296  
   297  		// mark volatile temps dead
   298  		for _, c := range volatiles {
   299  			tmpNode := c.tmp.Aux
   300  			memThen = bThen.NewValue1A(memThen.Pos, OpVarKill, types.TypeMem, tmpNode, memThen)
   301  		}
   302  
   303  		// merge memory
   304  		// Splice memory Phi into the last memory of the original sequence,
   305  		// which may be used in subsequent blocks. Other memories in the
   306  		// sequence must be dead after this block since there can be only
   307  		// one memory live.
   308  		bEnd.Values = append(bEnd.Values, last)
   309  		last.Block = bEnd
   310  		last.reset(OpPhi)
   311  		last.Type = types.TypeMem
   312  		last.AddArg(memThen)
   313  		last.AddArg(memElse)
   314  		for _, w := range stores {
   315  			if w != last {
   316  				w.resetArgs()
   317  			}
   318  		}
   319  		for _, w := range stores {
   320  			if w != last {
   321  				f.freeValue(w)
   322  			}
   323  		}
   324  
   325  		// put values after the store sequence into the end block
   326  		bEnd.Values = append(bEnd.Values, after...)
   327  		for _, w := range after {
   328  			w.Block = bEnd
   329  		}
   330  
   331  		// Preemption is unsafe between loading the write
   332  		// barrier-enabled flag and performing the write
   333  		// because that would allow a GC phase transition,
   334  		// which would invalidate the flag. Remember the
   335  		// conditional block so liveness analysis can disable
   336  		// safe-points. This is somewhat subtle because we're
   337  		// splitting b bottom-up.
   338  		if firstSplit {
   339  			// Add b itself.
   340  			b.Func.WBLoads = append(b.Func.WBLoads, b)
   341  			firstSplit = false
   342  		} else {
   343  			// We've already split b, so we just pushed a
   344  			// write barrier test into bEnd.
   345  			b.Func.WBLoads = append(b.Func.WBLoads, bEnd)
   346  		}
   347  
   348  		// if we have more stores in this block, do this block again
   349  		if nWBops > 0 {
   350  			goto again
   351  		}
   352  	}
   353  }
   354  
   355  // wbcall emits write barrier runtime call in b, returns memory.
   356  func wbcall(pos src.XPos, b *Block, fn, typ *obj.LSym, ptr, val, mem, sp, sb *Value) *Value {
   357  	config := b.Func.Config
   358  
   359  	// put arguments on stack
   360  	off := config.ctxt.FixedFrameSize()
   361  
   362  	if typ != nil { // for typedmemmove
   363  		taddr := b.NewValue1A(pos, OpAddr, b.Func.Config.Types.Uintptr, typ, sb)
   364  		off = round(off, taddr.Type.Alignment())
   365  		arg := b.NewValue1I(pos, OpOffPtr, taddr.Type.PtrTo(), off, sp)
   366  		mem = b.NewValue3A(pos, OpStore, types.TypeMem, ptr.Type, arg, taddr, mem)
   367  		off += taddr.Type.Size()
   368  	}
   369  
   370  	off = round(off, ptr.Type.Alignment())
   371  	arg := b.NewValue1I(pos, OpOffPtr, ptr.Type.PtrTo(), off, sp)
   372  	mem = b.NewValue3A(pos, OpStore, types.TypeMem, ptr.Type, arg, ptr, mem)
   373  	off += ptr.Type.Size()
   374  
   375  	if val != nil {
   376  		off = round(off, val.Type.Alignment())
   377  		arg = b.NewValue1I(pos, OpOffPtr, val.Type.PtrTo(), off, sp)
   378  		mem = b.NewValue3A(pos, OpStore, types.TypeMem, val.Type, arg, val, mem)
   379  		off += val.Type.Size()
   380  	}
   381  	off = round(off, config.PtrSize)
   382  
   383  	// issue call
   384  	mem = b.NewValue1A(pos, OpStaticCall, types.TypeMem, fn, mem)
   385  	mem.AuxInt = off - config.ctxt.FixedFrameSize()
   386  	return mem
   387  }
   388  
   389  // round to a multiple of r, r is a power of 2
   390  func round(o int64, r int64) int64 {
   391  	return (o + r - 1) &^ (r - 1)
   392  }
   393  
   394  // IsStackAddr reports whether v is known to be an address of a stack slot.
   395  func IsStackAddr(v *Value) bool {
   396  	for v.Op == OpOffPtr || v.Op == OpAddPtr || v.Op == OpPtrIndex || v.Op == OpCopy {
   397  		v = v.Args[0]
   398  	}
   399  	switch v.Op {
   400  	case OpSP, OpLocalAddr:
   401  		return true
   402  	}
   403  	return false
   404  }
   405  
   406  // IsGlobalAddr reports whether v is known to be an address of a global.
   407  func IsGlobalAddr(v *Value) bool {
   408  	return v.Op == OpAddr && v.Args[0].Op == OpSB
   409  }
   410  
   411  // IsReadOnlyGlobalAddr reports whether v is known to be an address of a read-only global.
   412  func IsReadOnlyGlobalAddr(v *Value) bool {
   413  	if !IsGlobalAddr(v) {
   414  		return false
   415  	}
   416  	// See TODO in OpAddr case in IsSanitizerSafeAddr below.
   417  	return strings.HasPrefix(v.Aux.(*obj.LSym).Name, `"".statictmp_`)
   418  }
   419  
   420  // IsNewObject reports whether v is a pointer to a freshly allocated & zeroed object at memory state mem.
   421  // TODO: Be more precise. We really want "IsNilPointer" for the particular field in question.
   422  // Right now, we can only detect a new object before any writes have been done to it.
   423  // We could ignore non-pointer writes, writes to offsets which
   424  // are known not to overlap the write in question, etc.
   425  func IsNewObject(v *Value, mem *Value) bool {
   426  	if v.Op != OpLoad {
   427  		return false
   428  	}
   429  	if v.MemoryArg() != mem {
   430  		return false
   431  	}
   432  	if mem.Op != OpStaticCall {
   433  		return false
   434  	}
   435  	if !isSameSym(mem.Aux, "runtime.newobject") {
   436  		return false
   437  	}
   438  	if v.Args[0].Op != OpOffPtr {
   439  		return false
   440  	}
   441  	if v.Args[0].Args[0].Op != OpSP {
   442  		return false
   443  	}
   444  	c := v.Block.Func.Config
   445  	if v.Args[0].AuxInt != c.ctxt.FixedFrameSize()+c.RegSize { // offset of return value
   446  		return false
   447  	}
   448  	return true
   449  }
   450  
   451  // IsSanitizerSafeAddr reports whether v is known to be an address
   452  // that doesn't need instrumentation.
   453  func IsSanitizerSafeAddr(v *Value) bool {
   454  	for v.Op == OpOffPtr || v.Op == OpAddPtr || v.Op == OpPtrIndex || v.Op == OpCopy {
   455  		v = v.Args[0]
   456  	}
   457  	switch v.Op {
   458  	case OpSP, OpLocalAddr:
   459  		// Stack addresses are always safe.
   460  		return true
   461  	case OpITab, OpStringPtr, OpGetClosurePtr:
   462  		// Itabs, string data, and closure fields are
   463  		// read-only once initialized.
   464  		return true
   465  	case OpAddr:
   466  		sym := v.Aux.(*obj.LSym)
   467  		// TODO(mdempsky): Find a cleaner way to
   468  		// detect this. It would be nice if we could
   469  		// test sym.Type==objabi.SRODATA, but we don't
   470  		// initialize sym.Type until after function
   471  		// compilation.
   472  		if strings.HasPrefix(sym.Name, `"".statictmp_`) {
   473  			return true
   474  		}
   475  	}
   476  	return false
   477  }
   478  
   479  // isVolatile reports whether v is a pointer to argument region on stack which
   480  // will be clobbered by a function call.
   481  func isVolatile(v *Value) bool {
   482  	for v.Op == OpOffPtr || v.Op == OpAddPtr || v.Op == OpPtrIndex || v.Op == OpCopy {
   483  		v = v.Args[0]
   484  	}
   485  	return v.Op == OpSP
   486  }
   487  

View as plain text