Black Lives Matter. Support the Equal Justice Initiative.

Source file src/cmd/compile/internal/arm64/ssa.go

Documentation: cmd/compile/internal/arm64

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package arm64
     6  
     7  import (
     8  	"math"
     9  
    10  	"cmd/compile/internal/gc"
    11  	"cmd/compile/internal/logopt"
    12  	"cmd/compile/internal/ssa"
    13  	"cmd/compile/internal/types"
    14  	"cmd/internal/obj"
    15  	"cmd/internal/obj/arm64"
    16  )
    17  
    18  // loadByType returns the load instruction of the given type.
    19  func loadByType(t *types.Type) obj.As {
    20  	if t.IsFloat() {
    21  		switch t.Size() {
    22  		case 4:
    23  			return arm64.AFMOVS
    24  		case 8:
    25  			return arm64.AFMOVD
    26  		}
    27  	} else {
    28  		switch t.Size() {
    29  		case 1:
    30  			if t.IsSigned() {
    31  				return arm64.AMOVB
    32  			} else {
    33  				return arm64.AMOVBU
    34  			}
    35  		case 2:
    36  			if t.IsSigned() {
    37  				return arm64.AMOVH
    38  			} else {
    39  				return arm64.AMOVHU
    40  			}
    41  		case 4:
    42  			if t.IsSigned() {
    43  				return arm64.AMOVW
    44  			} else {
    45  				return arm64.AMOVWU
    46  			}
    47  		case 8:
    48  			return arm64.AMOVD
    49  		}
    50  	}
    51  	panic("bad load type")
    52  }
    53  
    54  // storeByType returns the store instruction of the given type.
    55  func storeByType(t *types.Type) obj.As {
    56  	if t.IsFloat() {
    57  		switch t.Size() {
    58  		case 4:
    59  			return arm64.AFMOVS
    60  		case 8:
    61  			return arm64.AFMOVD
    62  		}
    63  	} else {
    64  		switch t.Size() {
    65  		case 1:
    66  			return arm64.AMOVB
    67  		case 2:
    68  			return arm64.AMOVH
    69  		case 4:
    70  			return arm64.AMOVW
    71  		case 8:
    72  			return arm64.AMOVD
    73  		}
    74  	}
    75  	panic("bad store type")
    76  }
    77  
    78  // makeshift encodes a register shifted by a constant, used as an Offset in Prog
    79  func makeshift(reg int16, typ int64, s int64) int64 {
    80  	return int64(reg&31)<<16 | typ | (s&63)<<10
    81  }
    82  
    83  // genshift generates a Prog for r = r0 op (r1 shifted by n)
    84  func genshift(s *gc.SSAGenState, as obj.As, r0, r1, r int16, typ int64, n int64) *obj.Prog {
    85  	p := s.Prog(as)
    86  	p.From.Type = obj.TYPE_SHIFT
    87  	p.From.Offset = makeshift(r1, typ, n)
    88  	p.Reg = r0
    89  	if r != 0 {
    90  		p.To.Type = obj.TYPE_REG
    91  		p.To.Reg = r
    92  	}
    93  	return p
    94  }
    95  
    96  // generate the memory operand for the indexed load/store instructions
    97  func genIndexedOperand(v *ssa.Value) obj.Addr {
    98  	// Reg: base register, Index: (shifted) index register
    99  	mop := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[0].Reg()}
   100  	switch v.Op {
   101  	case ssa.OpARM64MOVDloadidx8, ssa.OpARM64MOVDstoreidx8, ssa.OpARM64MOVDstorezeroidx8:
   102  		mop.Index = arm64.REG_LSL | 3<<5 | v.Args[1].Reg()&31
   103  	case ssa.OpARM64MOVWloadidx4, ssa.OpARM64MOVWUloadidx4, ssa.OpARM64MOVWstoreidx4, ssa.OpARM64MOVWstorezeroidx4:
   104  		mop.Index = arm64.REG_LSL | 2<<5 | v.Args[1].Reg()&31
   105  	case ssa.OpARM64MOVHloadidx2, ssa.OpARM64MOVHUloadidx2, ssa.OpARM64MOVHstoreidx2, ssa.OpARM64MOVHstorezeroidx2:
   106  		mop.Index = arm64.REG_LSL | 1<<5 | v.Args[1].Reg()&31
   107  	default: // not shifted
   108  		mop.Index = v.Args[1].Reg()
   109  	}
   110  	return mop
   111  }
   112  
   113  func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) {
   114  	switch v.Op {
   115  	case ssa.OpCopy, ssa.OpARM64MOVDreg:
   116  		if v.Type.IsMemory() {
   117  			return
   118  		}
   119  		x := v.Args[0].Reg()
   120  		y := v.Reg()
   121  		if x == y {
   122  			return
   123  		}
   124  		as := arm64.AMOVD
   125  		if v.Type.IsFloat() {
   126  			switch v.Type.Size() {
   127  			case 4:
   128  				as = arm64.AFMOVS
   129  			case 8:
   130  				as = arm64.AFMOVD
   131  			default:
   132  				panic("bad float size")
   133  			}
   134  		}
   135  		p := s.Prog(as)
   136  		p.From.Type = obj.TYPE_REG
   137  		p.From.Reg = x
   138  		p.To.Type = obj.TYPE_REG
   139  		p.To.Reg = y
   140  	case ssa.OpARM64MOVDnop:
   141  		if v.Reg() != v.Args[0].Reg() {
   142  			v.Fatalf("input[0] and output not in same register %s", v.LongString())
   143  		}
   144  		// nothing to do
   145  	case ssa.OpLoadReg:
   146  		if v.Type.IsFlags() {
   147  			v.Fatalf("load flags not implemented: %v", v.LongString())
   148  			return
   149  		}
   150  		p := s.Prog(loadByType(v.Type))
   151  		gc.AddrAuto(&p.From, v.Args[0])
   152  		p.To.Type = obj.TYPE_REG
   153  		p.To.Reg = v.Reg()
   154  	case ssa.OpStoreReg:
   155  		if v.Type.IsFlags() {
   156  			v.Fatalf("store flags not implemented: %v", v.LongString())
   157  			return
   158  		}
   159  		p := s.Prog(storeByType(v.Type))
   160  		p.From.Type = obj.TYPE_REG
   161  		p.From.Reg = v.Args[0].Reg()
   162  		gc.AddrAuto(&p.To, v)
   163  	case ssa.OpARM64ADD,
   164  		ssa.OpARM64SUB,
   165  		ssa.OpARM64AND,
   166  		ssa.OpARM64OR,
   167  		ssa.OpARM64XOR,
   168  		ssa.OpARM64BIC,
   169  		ssa.OpARM64EON,
   170  		ssa.OpARM64ORN,
   171  		ssa.OpARM64MUL,
   172  		ssa.OpARM64MULW,
   173  		ssa.OpARM64MNEG,
   174  		ssa.OpARM64MNEGW,
   175  		ssa.OpARM64MULH,
   176  		ssa.OpARM64UMULH,
   177  		ssa.OpARM64MULL,
   178  		ssa.OpARM64UMULL,
   179  		ssa.OpARM64DIV,
   180  		ssa.OpARM64UDIV,
   181  		ssa.OpARM64DIVW,
   182  		ssa.OpARM64UDIVW,
   183  		ssa.OpARM64MOD,
   184  		ssa.OpARM64UMOD,
   185  		ssa.OpARM64MODW,
   186  		ssa.OpARM64UMODW,
   187  		ssa.OpARM64SLL,
   188  		ssa.OpARM64SRL,
   189  		ssa.OpARM64SRA,
   190  		ssa.OpARM64FADDS,
   191  		ssa.OpARM64FADDD,
   192  		ssa.OpARM64FSUBS,
   193  		ssa.OpARM64FSUBD,
   194  		ssa.OpARM64FMULS,
   195  		ssa.OpARM64FMULD,
   196  		ssa.OpARM64FNMULS,
   197  		ssa.OpARM64FNMULD,
   198  		ssa.OpARM64FDIVS,
   199  		ssa.OpARM64FDIVD,
   200  		ssa.OpARM64ROR,
   201  		ssa.OpARM64RORW:
   202  		r := v.Reg()
   203  		r1 := v.Args[0].Reg()
   204  		r2 := v.Args[1].Reg()
   205  		p := s.Prog(v.Op.Asm())
   206  		p.From.Type = obj.TYPE_REG
   207  		p.From.Reg = r2
   208  		p.Reg = r1
   209  		p.To.Type = obj.TYPE_REG
   210  		p.To.Reg = r
   211  	case ssa.OpARM64FMADDS,
   212  		ssa.OpARM64FMADDD,
   213  		ssa.OpARM64FNMADDS,
   214  		ssa.OpARM64FNMADDD,
   215  		ssa.OpARM64FMSUBS,
   216  		ssa.OpARM64FMSUBD,
   217  		ssa.OpARM64FNMSUBS,
   218  		ssa.OpARM64FNMSUBD,
   219  		ssa.OpARM64MADD,
   220  		ssa.OpARM64MADDW,
   221  		ssa.OpARM64MSUB,
   222  		ssa.OpARM64MSUBW:
   223  		rt := v.Reg()
   224  		ra := v.Args[0].Reg()
   225  		rm := v.Args[1].Reg()
   226  		rn := v.Args[2].Reg()
   227  		p := s.Prog(v.Op.Asm())
   228  		p.Reg = ra
   229  		p.From.Type = obj.TYPE_REG
   230  		p.From.Reg = rm
   231  		p.SetFrom3(obj.Addr{Type: obj.TYPE_REG, Reg: rn})
   232  		p.To.Type = obj.TYPE_REG
   233  		p.To.Reg = rt
   234  	case ssa.OpARM64ADDconst,
   235  		ssa.OpARM64SUBconst,
   236  		ssa.OpARM64ANDconst,
   237  		ssa.OpARM64ORconst,
   238  		ssa.OpARM64XORconst,
   239  		ssa.OpARM64SLLconst,
   240  		ssa.OpARM64SRLconst,
   241  		ssa.OpARM64SRAconst,
   242  		ssa.OpARM64RORconst,
   243  		ssa.OpARM64RORWconst:
   244  		p := s.Prog(v.Op.Asm())
   245  		p.From.Type = obj.TYPE_CONST
   246  		p.From.Offset = v.AuxInt
   247  		p.Reg = v.Args[0].Reg()
   248  		p.To.Type = obj.TYPE_REG
   249  		p.To.Reg = v.Reg()
   250  	case ssa.OpARM64ADDSconstflags:
   251  		p := s.Prog(v.Op.Asm())
   252  		p.From.Type = obj.TYPE_CONST
   253  		p.From.Offset = v.AuxInt
   254  		p.Reg = v.Args[0].Reg()
   255  		p.To.Type = obj.TYPE_REG
   256  		p.To.Reg = v.Reg0()
   257  	case ssa.OpARM64ADCzerocarry:
   258  		p := s.Prog(v.Op.Asm())
   259  		p.From.Type = obj.TYPE_REG
   260  		p.From.Reg = arm64.REGZERO
   261  		p.Reg = arm64.REGZERO
   262  		p.To.Type = obj.TYPE_REG
   263  		p.To.Reg = v.Reg()
   264  	case ssa.OpARM64ADCSflags,
   265  		ssa.OpARM64ADDSflags,
   266  		ssa.OpARM64SBCSflags,
   267  		ssa.OpARM64SUBSflags:
   268  		r := v.Reg0()
   269  		r1 := v.Args[0].Reg()
   270  		r2 := v.Args[1].Reg()
   271  		p := s.Prog(v.Op.Asm())
   272  		p.From.Type = obj.TYPE_REG
   273  		p.From.Reg = r2
   274  		p.Reg = r1
   275  		p.To.Type = obj.TYPE_REG
   276  		p.To.Reg = r
   277  	case ssa.OpARM64NEGSflags:
   278  		p := s.Prog(v.Op.Asm())
   279  		p.From.Type = obj.TYPE_REG
   280  		p.From.Reg = v.Args[0].Reg()
   281  		p.To.Type = obj.TYPE_REG
   282  		p.To.Reg = v.Reg0()
   283  	case ssa.OpARM64NGCzerocarry:
   284  		p := s.Prog(v.Op.Asm())
   285  		p.From.Type = obj.TYPE_REG
   286  		p.From.Reg = arm64.REGZERO
   287  		p.To.Type = obj.TYPE_REG
   288  		p.To.Reg = v.Reg()
   289  	case ssa.OpARM64EXTRconst,
   290  		ssa.OpARM64EXTRWconst:
   291  		p := s.Prog(v.Op.Asm())
   292  		p.From.Type = obj.TYPE_CONST
   293  		p.From.Offset = v.AuxInt
   294  		p.SetFrom3(obj.Addr{Type: obj.TYPE_REG, Reg: v.Args[0].Reg()})
   295  		p.Reg = v.Args[1].Reg()
   296  		p.To.Type = obj.TYPE_REG
   297  		p.To.Reg = v.Reg()
   298  	case ssa.OpARM64MVNshiftLL, ssa.OpARM64NEGshiftLL:
   299  		genshift(s, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_LL, v.AuxInt)
   300  	case ssa.OpARM64MVNshiftRL, ssa.OpARM64NEGshiftRL:
   301  		genshift(s, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_LR, v.AuxInt)
   302  	case ssa.OpARM64MVNshiftRA, ssa.OpARM64NEGshiftRA:
   303  		genshift(s, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_AR, v.AuxInt)
   304  	case ssa.OpARM64ADDshiftLL,
   305  		ssa.OpARM64SUBshiftLL,
   306  		ssa.OpARM64ANDshiftLL,
   307  		ssa.OpARM64ORshiftLL,
   308  		ssa.OpARM64XORshiftLL,
   309  		ssa.OpARM64EONshiftLL,
   310  		ssa.OpARM64ORNshiftLL,
   311  		ssa.OpARM64BICshiftLL:
   312  		genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_LL, v.AuxInt)
   313  	case ssa.OpARM64ADDshiftRL,
   314  		ssa.OpARM64SUBshiftRL,
   315  		ssa.OpARM64ANDshiftRL,
   316  		ssa.OpARM64ORshiftRL,
   317  		ssa.OpARM64XORshiftRL,
   318  		ssa.OpARM64EONshiftRL,
   319  		ssa.OpARM64ORNshiftRL,
   320  		ssa.OpARM64BICshiftRL:
   321  		genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_LR, v.AuxInt)
   322  	case ssa.OpARM64ADDshiftRA,
   323  		ssa.OpARM64SUBshiftRA,
   324  		ssa.OpARM64ANDshiftRA,
   325  		ssa.OpARM64ORshiftRA,
   326  		ssa.OpARM64XORshiftRA,
   327  		ssa.OpARM64EONshiftRA,
   328  		ssa.OpARM64ORNshiftRA,
   329  		ssa.OpARM64BICshiftRA:
   330  		genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_AR, v.AuxInt)
   331  	case ssa.OpARM64MOVDconst:
   332  		p := s.Prog(v.Op.Asm())
   333  		p.From.Type = obj.TYPE_CONST
   334  		p.From.Offset = v.AuxInt
   335  		p.To.Type = obj.TYPE_REG
   336  		p.To.Reg = v.Reg()
   337  	case ssa.OpARM64FMOVSconst,
   338  		ssa.OpARM64FMOVDconst:
   339  		p := s.Prog(v.Op.Asm())
   340  		p.From.Type = obj.TYPE_FCONST
   341  		p.From.Val = math.Float64frombits(uint64(v.AuxInt))
   342  		p.To.Type = obj.TYPE_REG
   343  		p.To.Reg = v.Reg()
   344  	case ssa.OpARM64FCMPS0,
   345  		ssa.OpARM64FCMPD0:
   346  		p := s.Prog(v.Op.Asm())
   347  		p.From.Type = obj.TYPE_FCONST
   348  		p.From.Val = math.Float64frombits(0)
   349  		p.Reg = v.Args[0].Reg()
   350  	case ssa.OpARM64CMP,
   351  		ssa.OpARM64CMPW,
   352  		ssa.OpARM64CMN,
   353  		ssa.OpARM64CMNW,
   354  		ssa.OpARM64TST,
   355  		ssa.OpARM64TSTW,
   356  		ssa.OpARM64FCMPS,
   357  		ssa.OpARM64FCMPD:
   358  		p := s.Prog(v.Op.Asm())
   359  		p.From.Type = obj.TYPE_REG
   360  		p.From.Reg = v.Args[1].Reg()
   361  		p.Reg = v.Args[0].Reg()
   362  	case ssa.OpARM64CMPconst,
   363  		ssa.OpARM64CMPWconst,
   364  		ssa.OpARM64CMNconst,
   365  		ssa.OpARM64CMNWconst,
   366  		ssa.OpARM64TSTconst,
   367  		ssa.OpARM64TSTWconst:
   368  		p := s.Prog(v.Op.Asm())
   369  		p.From.Type = obj.TYPE_CONST
   370  		p.From.Offset = v.AuxInt
   371  		p.Reg = v.Args[0].Reg()
   372  	case ssa.OpARM64CMPshiftLL, ssa.OpARM64CMNshiftLL, ssa.OpARM64TSTshiftLL:
   373  		genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_LL, v.AuxInt)
   374  	case ssa.OpARM64CMPshiftRL, ssa.OpARM64CMNshiftRL, ssa.OpARM64TSTshiftRL:
   375  		genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_LR, v.AuxInt)
   376  	case ssa.OpARM64CMPshiftRA, ssa.OpARM64CMNshiftRA, ssa.OpARM64TSTshiftRA:
   377  		genshift(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_AR, v.AuxInt)
   378  	case ssa.OpARM64MOVDaddr:
   379  		p := s.Prog(arm64.AMOVD)
   380  		p.From.Type = obj.TYPE_ADDR
   381  		p.From.Reg = v.Args[0].Reg()
   382  		p.To.Type = obj.TYPE_REG
   383  		p.To.Reg = v.Reg()
   384  
   385  		var wantreg string
   386  		// MOVD $sym+off(base), R
   387  		// the assembler expands it as the following:
   388  		// - base is SP: add constant offset to SP (R13)
   389  		//               when constant is large, tmp register (R11) may be used
   390  		// - base is SB: load external address from constant pool (use relocation)
   391  		switch v.Aux.(type) {
   392  		default:
   393  			v.Fatalf("aux is of unknown type %T", v.Aux)
   394  		case *obj.LSym:
   395  			wantreg = "SB"
   396  			gc.AddAux(&p.From, v)
   397  		case *gc.Node:
   398  			wantreg = "SP"
   399  			gc.AddAux(&p.From, v)
   400  		case nil:
   401  			// No sym, just MOVD $off(SP), R
   402  			wantreg = "SP"
   403  			p.From.Offset = v.AuxInt
   404  		}
   405  		if reg := v.Args[0].RegName(); reg != wantreg {
   406  			v.Fatalf("bad reg %s for symbol type %T, want %s", reg, v.Aux, wantreg)
   407  		}
   408  	case ssa.OpARM64MOVBload,
   409  		ssa.OpARM64MOVBUload,
   410  		ssa.OpARM64MOVHload,
   411  		ssa.OpARM64MOVHUload,
   412  		ssa.OpARM64MOVWload,
   413  		ssa.OpARM64MOVWUload,
   414  		ssa.OpARM64MOVDload,
   415  		ssa.OpARM64FMOVSload,
   416  		ssa.OpARM64FMOVDload:
   417  		p := s.Prog(v.Op.Asm())
   418  		p.From.Type = obj.TYPE_MEM
   419  		p.From.Reg = v.Args[0].Reg()
   420  		gc.AddAux(&p.From, v)
   421  		p.To.Type = obj.TYPE_REG
   422  		p.To.Reg = v.Reg()
   423  	case ssa.OpARM64MOVBloadidx,
   424  		ssa.OpARM64MOVBUloadidx,
   425  		ssa.OpARM64MOVHloadidx,
   426  		ssa.OpARM64MOVHUloadidx,
   427  		ssa.OpARM64MOVWloadidx,
   428  		ssa.OpARM64MOVWUloadidx,
   429  		ssa.OpARM64MOVDloadidx,
   430  		ssa.OpARM64FMOVSloadidx,
   431  		ssa.OpARM64FMOVDloadidx,
   432  		ssa.OpARM64MOVHloadidx2,
   433  		ssa.OpARM64MOVHUloadidx2,
   434  		ssa.OpARM64MOVWloadidx4,
   435  		ssa.OpARM64MOVWUloadidx4,
   436  		ssa.OpARM64MOVDloadidx8:
   437  		p := s.Prog(v.Op.Asm())
   438  		p.From = genIndexedOperand(v)
   439  		p.To.Type = obj.TYPE_REG
   440  		p.To.Reg = v.Reg()
   441  	case ssa.OpARM64LDAR,
   442  		ssa.OpARM64LDARB,
   443  		ssa.OpARM64LDARW:
   444  		p := s.Prog(v.Op.Asm())
   445  		p.From.Type = obj.TYPE_MEM
   446  		p.From.Reg = v.Args[0].Reg()
   447  		gc.AddAux(&p.From, v)
   448  		p.To.Type = obj.TYPE_REG
   449  		p.To.Reg = v.Reg0()
   450  	case ssa.OpARM64MOVBstore,
   451  		ssa.OpARM64MOVHstore,
   452  		ssa.OpARM64MOVWstore,
   453  		ssa.OpARM64MOVDstore,
   454  		ssa.OpARM64FMOVSstore,
   455  		ssa.OpARM64FMOVDstore,
   456  		ssa.OpARM64STLRB,
   457  		ssa.OpARM64STLR,
   458  		ssa.OpARM64STLRW:
   459  		p := s.Prog(v.Op.Asm())
   460  		p.From.Type = obj.TYPE_REG
   461  		p.From.Reg = v.Args[1].Reg()
   462  		p.To.Type = obj.TYPE_MEM
   463  		p.To.Reg = v.Args[0].Reg()
   464  		gc.AddAux(&p.To, v)
   465  	case ssa.OpARM64MOVBstoreidx,
   466  		ssa.OpARM64MOVHstoreidx,
   467  		ssa.OpARM64MOVWstoreidx,
   468  		ssa.OpARM64MOVDstoreidx,
   469  		ssa.OpARM64FMOVSstoreidx,
   470  		ssa.OpARM64FMOVDstoreidx,
   471  		ssa.OpARM64MOVHstoreidx2,
   472  		ssa.OpARM64MOVWstoreidx4,
   473  		ssa.OpARM64MOVDstoreidx8:
   474  		p := s.Prog(v.Op.Asm())
   475  		p.To = genIndexedOperand(v)
   476  		p.From.Type = obj.TYPE_REG
   477  		p.From.Reg = v.Args[2].Reg()
   478  	case ssa.OpARM64STP:
   479  		p := s.Prog(v.Op.Asm())
   480  		p.From.Type = obj.TYPE_REGREG
   481  		p.From.Reg = v.Args[1].Reg()
   482  		p.From.Offset = int64(v.Args[2].Reg())
   483  		p.To.Type = obj.TYPE_MEM
   484  		p.To.Reg = v.Args[0].Reg()
   485  		gc.AddAux(&p.To, v)
   486  	case ssa.OpARM64MOVBstorezero,
   487  		ssa.OpARM64MOVHstorezero,
   488  		ssa.OpARM64MOVWstorezero,
   489  		ssa.OpARM64MOVDstorezero:
   490  		p := s.Prog(v.Op.Asm())
   491  		p.From.Type = obj.TYPE_REG
   492  		p.From.Reg = arm64.REGZERO
   493  		p.To.Type = obj.TYPE_MEM
   494  		p.To.Reg = v.Args[0].Reg()
   495  		gc.AddAux(&p.To, v)
   496  	case ssa.OpARM64MOVBstorezeroidx,
   497  		ssa.OpARM64MOVHstorezeroidx,
   498  		ssa.OpARM64MOVWstorezeroidx,
   499  		ssa.OpARM64MOVDstorezeroidx,
   500  		ssa.OpARM64MOVHstorezeroidx2,
   501  		ssa.OpARM64MOVWstorezeroidx4,
   502  		ssa.OpARM64MOVDstorezeroidx8:
   503  		p := s.Prog(v.Op.Asm())
   504  		p.To = genIndexedOperand(v)
   505  		p.From.Type = obj.TYPE_REG
   506  		p.From.Reg = arm64.REGZERO
   507  	case ssa.OpARM64MOVQstorezero:
   508  		p := s.Prog(v.Op.Asm())
   509  		p.From.Type = obj.TYPE_REGREG
   510  		p.From.Reg = arm64.REGZERO
   511  		p.From.Offset = int64(arm64.REGZERO)
   512  		p.To.Type = obj.TYPE_MEM
   513  		p.To.Reg = v.Args[0].Reg()
   514  		gc.AddAux(&p.To, v)
   515  	case ssa.OpARM64BFI,
   516  		ssa.OpARM64BFXIL:
   517  		r := v.Reg()
   518  		if r != v.Args[0].Reg() {
   519  			v.Fatalf("input[0] and output not in same register %s", v.LongString())
   520  		}
   521  		p := s.Prog(v.Op.Asm())
   522  		p.From.Type = obj.TYPE_CONST
   523  		p.From.Offset = v.AuxInt >> 8
   524  		p.SetFrom3(obj.Addr{Type: obj.TYPE_CONST, Offset: v.AuxInt & 0xff})
   525  		p.Reg = v.Args[1].Reg()
   526  		p.To.Type = obj.TYPE_REG
   527  		p.To.Reg = r
   528  	case ssa.OpARM64SBFIZ,
   529  		ssa.OpARM64SBFX,
   530  		ssa.OpARM64UBFIZ,
   531  		ssa.OpARM64UBFX:
   532  		p := s.Prog(v.Op.Asm())
   533  		p.From.Type = obj.TYPE_CONST
   534  		p.From.Offset = v.AuxInt >> 8
   535  		p.SetFrom3(obj.Addr{Type: obj.TYPE_CONST, Offset: v.AuxInt & 0xff})
   536  		p.Reg = v.Args[0].Reg()
   537  		p.To.Type = obj.TYPE_REG
   538  		p.To.Reg = v.Reg()
   539  	case ssa.OpARM64LoweredMuluhilo:
   540  		r0 := v.Args[0].Reg()
   541  		r1 := v.Args[1].Reg()
   542  		p := s.Prog(arm64.AUMULH)
   543  		p.From.Type = obj.TYPE_REG
   544  		p.From.Reg = r1
   545  		p.Reg = r0
   546  		p.To.Type = obj.TYPE_REG
   547  		p.To.Reg = v.Reg0()
   548  		p1 := s.Prog(arm64.AMUL)
   549  		p1.From.Type = obj.TYPE_REG
   550  		p1.From.Reg = r1
   551  		p1.Reg = r0
   552  		p1.To.Type = obj.TYPE_REG
   553  		p1.To.Reg = v.Reg1()
   554  	case ssa.OpARM64LoweredAtomicExchange64,
   555  		ssa.OpARM64LoweredAtomicExchange32:
   556  		// LDAXR	(Rarg0), Rout
   557  		// STLXR	Rarg1, (Rarg0), Rtmp
   558  		// CBNZ		Rtmp, -2(PC)
   559  		ld := arm64.ALDAXR
   560  		st := arm64.ASTLXR
   561  		if v.Op == ssa.OpARM64LoweredAtomicExchange32 {
   562  			ld = arm64.ALDAXRW
   563  			st = arm64.ASTLXRW
   564  		}
   565  		r0 := v.Args[0].Reg()
   566  		r1 := v.Args[1].Reg()
   567  		out := v.Reg0()
   568  		p := s.Prog(ld)
   569  		p.From.Type = obj.TYPE_MEM
   570  		p.From.Reg = r0
   571  		p.To.Type = obj.TYPE_REG
   572  		p.To.Reg = out
   573  		p1 := s.Prog(st)
   574  		p1.From.Type = obj.TYPE_REG
   575  		p1.From.Reg = r1
   576  		p1.To.Type = obj.TYPE_MEM
   577  		p1.To.Reg = r0
   578  		p1.RegTo2 = arm64.REGTMP
   579  		p2 := s.Prog(arm64.ACBNZ)
   580  		p2.From.Type = obj.TYPE_REG
   581  		p2.From.Reg = arm64.REGTMP
   582  		p2.To.Type = obj.TYPE_BRANCH
   583  		gc.Patch(p2, p)
   584  	case ssa.OpARM64LoweredAtomicAdd64,
   585  		ssa.OpARM64LoweredAtomicAdd32:
   586  		// LDAXR	(Rarg0), Rout
   587  		// ADD		Rarg1, Rout
   588  		// STLXR	Rout, (Rarg0), Rtmp
   589  		// CBNZ		Rtmp, -3(PC)
   590  		ld := arm64.ALDAXR
   591  		st := arm64.ASTLXR
   592  		if v.Op == ssa.OpARM64LoweredAtomicAdd32 {
   593  			ld = arm64.ALDAXRW
   594  			st = arm64.ASTLXRW
   595  		}
   596  		r0 := v.Args[0].Reg()
   597  		r1 := v.Args[1].Reg()
   598  		out := v.Reg0()
   599  		p := s.Prog(ld)
   600  		p.From.Type = obj.TYPE_MEM
   601  		p.From.Reg = r0
   602  		p.To.Type = obj.TYPE_REG
   603  		p.To.Reg = out
   604  		p1 := s.Prog(arm64.AADD)
   605  		p1.From.Type = obj.TYPE_REG
   606  		p1.From.Reg = r1
   607  		p1.To.Type = obj.TYPE_REG
   608  		p1.To.Reg = out
   609  		p2 := s.Prog(st)
   610  		p2.From.Type = obj.TYPE_REG
   611  		p2.From.Reg = out
   612  		p2.To.Type = obj.TYPE_MEM
   613  		p2.To.Reg = r0
   614  		p2.RegTo2 = arm64.REGTMP
   615  		p3 := s.Prog(arm64.ACBNZ)
   616  		p3.From.Type = obj.TYPE_REG
   617  		p3.From.Reg = arm64.REGTMP
   618  		p3.To.Type = obj.TYPE_BRANCH
   619  		gc.Patch(p3, p)
   620  	case ssa.OpARM64LoweredAtomicAdd64Variant,
   621  		ssa.OpARM64LoweredAtomicAdd32Variant:
   622  		// LDADDAL	Rarg1, (Rarg0), Rout
   623  		// ADD		Rarg1, Rout
   624  		op := arm64.ALDADDALD
   625  		if v.Op == ssa.OpARM64LoweredAtomicAdd32Variant {
   626  			op = arm64.ALDADDALW
   627  		}
   628  		r0 := v.Args[0].Reg()
   629  		r1 := v.Args[1].Reg()
   630  		out := v.Reg0()
   631  		p := s.Prog(op)
   632  		p.From.Type = obj.TYPE_REG
   633  		p.From.Reg = r1
   634  		p.To.Type = obj.TYPE_MEM
   635  		p.To.Reg = r0
   636  		p.RegTo2 = out
   637  		p1 := s.Prog(arm64.AADD)
   638  		p1.From.Type = obj.TYPE_REG
   639  		p1.From.Reg = r1
   640  		p1.To.Type = obj.TYPE_REG
   641  		p1.To.Reg = out
   642  	case ssa.OpARM64LoweredAtomicCas64,
   643  		ssa.OpARM64LoweredAtomicCas32:
   644  		// LDAXR	(Rarg0), Rtmp
   645  		// CMP		Rarg1, Rtmp
   646  		// BNE		3(PC)
   647  		// STLXR	Rarg2, (Rarg0), Rtmp
   648  		// CBNZ		Rtmp, -4(PC)
   649  		// CSET		EQ, Rout
   650  		ld := arm64.ALDAXR
   651  		st := arm64.ASTLXR
   652  		cmp := arm64.ACMP
   653  		if v.Op == ssa.OpARM64LoweredAtomicCas32 {
   654  			ld = arm64.ALDAXRW
   655  			st = arm64.ASTLXRW
   656  			cmp = arm64.ACMPW
   657  		}
   658  		r0 := v.Args[0].Reg()
   659  		r1 := v.Args[1].Reg()
   660  		r2 := v.Args[2].Reg()
   661  		out := v.Reg0()
   662  		p := s.Prog(ld)
   663  		p.From.Type = obj.TYPE_MEM
   664  		p.From.Reg = r0
   665  		p.To.Type = obj.TYPE_REG
   666  		p.To.Reg = arm64.REGTMP
   667  		p1 := s.Prog(cmp)
   668  		p1.From.Type = obj.TYPE_REG
   669  		p1.From.Reg = r1
   670  		p1.Reg = arm64.REGTMP
   671  		p2 := s.Prog(arm64.ABNE)
   672  		p2.To.Type = obj.TYPE_BRANCH
   673  		p3 := s.Prog(st)
   674  		p3.From.Type = obj.TYPE_REG
   675  		p3.From.Reg = r2
   676  		p3.To.Type = obj.TYPE_MEM
   677  		p3.To.Reg = r0
   678  		p3.RegTo2 = arm64.REGTMP
   679  		p4 := s.Prog(arm64.ACBNZ)
   680  		p4.From.Type = obj.TYPE_REG
   681  		p4.From.Reg = arm64.REGTMP
   682  		p4.To.Type = obj.TYPE_BRANCH
   683  		gc.Patch(p4, p)
   684  		p5 := s.Prog(arm64.ACSET)
   685  		p5.From.Type = obj.TYPE_REG // assembler encodes conditional bits in Reg
   686  		p5.From.Reg = arm64.COND_EQ
   687  		p5.To.Type = obj.TYPE_REG
   688  		p5.To.Reg = out
   689  		gc.Patch(p2, p5)
   690  	case ssa.OpARM64LoweredAtomicAnd8,
   691  		ssa.OpARM64LoweredAtomicOr8:
   692  		// LDAXRB	(Rarg0), Rout
   693  		// AND/OR	Rarg1, Rout
   694  		// STLXRB	Rout, (Rarg0), Rtmp
   695  		// CBNZ		Rtmp, -3(PC)
   696  		r0 := v.Args[0].Reg()
   697  		r1 := v.Args[1].Reg()
   698  		out := v.Reg0()
   699  		p := s.Prog(arm64.ALDAXRB)
   700  		p.From.Type = obj.TYPE_MEM
   701  		p.From.Reg = r0
   702  		p.To.Type = obj.TYPE_REG
   703  		p.To.Reg = out
   704  		p1 := s.Prog(v.Op.Asm())
   705  		p1.From.Type = obj.TYPE_REG
   706  		p1.From.Reg = r1
   707  		p1.To.Type = obj.TYPE_REG
   708  		p1.To.Reg = out
   709  		p2 := s.Prog(arm64.ASTLXRB)
   710  		p2.From.Type = obj.TYPE_REG
   711  		p2.From.Reg = out
   712  		p2.To.Type = obj.TYPE_MEM
   713  		p2.To.Reg = r0
   714  		p2.RegTo2 = arm64.REGTMP
   715  		p3 := s.Prog(arm64.ACBNZ)
   716  		p3.From.Type = obj.TYPE_REG
   717  		p3.From.Reg = arm64.REGTMP
   718  		p3.To.Type = obj.TYPE_BRANCH
   719  		gc.Patch(p3, p)
   720  	case ssa.OpARM64MOVBreg,
   721  		ssa.OpARM64MOVBUreg,
   722  		ssa.OpARM64MOVHreg,
   723  		ssa.OpARM64MOVHUreg,
   724  		ssa.OpARM64MOVWreg,
   725  		ssa.OpARM64MOVWUreg:
   726  		a := v.Args[0]
   727  		for a.Op == ssa.OpCopy || a.Op == ssa.OpARM64MOVDreg {
   728  			a = a.Args[0]
   729  		}
   730  		if a.Op == ssa.OpLoadReg {
   731  			t := a.Type
   732  			switch {
   733  			case v.Op == ssa.OpARM64MOVBreg && t.Size() == 1 && t.IsSigned(),
   734  				v.Op == ssa.OpARM64MOVBUreg && t.Size() == 1 && !t.IsSigned(),
   735  				v.Op == ssa.OpARM64MOVHreg && t.Size() == 2 && t.IsSigned(),
   736  				v.Op == ssa.OpARM64MOVHUreg && t.Size() == 2 && !t.IsSigned(),
   737  				v.Op == ssa.OpARM64MOVWreg && t.Size() == 4 && t.IsSigned(),
   738  				v.Op == ssa.OpARM64MOVWUreg && t.Size() == 4 && !t.IsSigned():
   739  				// arg is a proper-typed load, already zero/sign-extended, don't extend again
   740  				if v.Reg() == v.Args[0].Reg() {
   741  					return
   742  				}
   743  				p := s.Prog(arm64.AMOVD)
   744  				p.From.Type = obj.TYPE_REG
   745  				p.From.Reg = v.Args[0].Reg()
   746  				p.To.Type = obj.TYPE_REG
   747  				p.To.Reg = v.Reg()
   748  				return
   749  			default:
   750  			}
   751  		}
   752  		fallthrough
   753  	case ssa.OpARM64MVN,
   754  		ssa.OpARM64NEG,
   755  		ssa.OpARM64FABSD,
   756  		ssa.OpARM64FMOVDfpgp,
   757  		ssa.OpARM64FMOVDgpfp,
   758  		ssa.OpARM64FMOVSfpgp,
   759  		ssa.OpARM64FMOVSgpfp,
   760  		ssa.OpARM64FNEGS,
   761  		ssa.OpARM64FNEGD,
   762  		ssa.OpARM64FSQRTD,
   763  		ssa.OpARM64FCVTZSSW,
   764  		ssa.OpARM64FCVTZSDW,
   765  		ssa.OpARM64FCVTZUSW,
   766  		ssa.OpARM64FCVTZUDW,
   767  		ssa.OpARM64FCVTZSS,
   768  		ssa.OpARM64FCVTZSD,
   769  		ssa.OpARM64FCVTZUS,
   770  		ssa.OpARM64FCVTZUD,
   771  		ssa.OpARM64SCVTFWS,
   772  		ssa.OpARM64SCVTFWD,
   773  		ssa.OpARM64SCVTFS,
   774  		ssa.OpARM64SCVTFD,
   775  		ssa.OpARM64UCVTFWS,
   776  		ssa.OpARM64UCVTFWD,
   777  		ssa.OpARM64UCVTFS,
   778  		ssa.OpARM64UCVTFD,
   779  		ssa.OpARM64FCVTSD,
   780  		ssa.OpARM64FCVTDS,
   781  		ssa.OpARM64REV,
   782  		ssa.OpARM64REVW,
   783  		ssa.OpARM64REV16W,
   784  		ssa.OpARM64RBIT,
   785  		ssa.OpARM64RBITW,
   786  		ssa.OpARM64CLZ,
   787  		ssa.OpARM64CLZW,
   788  		ssa.OpARM64FRINTAD,
   789  		ssa.OpARM64FRINTMD,
   790  		ssa.OpARM64FRINTND,
   791  		ssa.OpARM64FRINTPD,
   792  		ssa.OpARM64FRINTZD:
   793  		p := s.Prog(v.Op.Asm())
   794  		p.From.Type = obj.TYPE_REG
   795  		p.From.Reg = v.Args[0].Reg()
   796  		p.To.Type = obj.TYPE_REG
   797  		p.To.Reg = v.Reg()
   798  	case ssa.OpARM64LoweredRound32F, ssa.OpARM64LoweredRound64F:
   799  		// input is already rounded
   800  	case ssa.OpARM64VCNT:
   801  		p := s.Prog(v.Op.Asm())
   802  		p.From.Type = obj.TYPE_REG
   803  		p.From.Reg = (v.Args[0].Reg()-arm64.REG_F0)&31 + arm64.REG_ARNG + ((arm64.ARNG_8B & 15) << 5)
   804  		p.To.Type = obj.TYPE_REG
   805  		p.To.Reg = (v.Reg()-arm64.REG_F0)&31 + arm64.REG_ARNG + ((arm64.ARNG_8B & 15) << 5)
   806  	case ssa.OpARM64VUADDLV:
   807  		p := s.Prog(v.Op.Asm())
   808  		p.From.Type = obj.TYPE_REG
   809  		p.From.Reg = (v.Args[0].Reg()-arm64.REG_F0)&31 + arm64.REG_ARNG + ((arm64.ARNG_8B & 15) << 5)
   810  		p.To.Type = obj.TYPE_REG
   811  		p.To.Reg = v.Reg() - arm64.REG_F0 + arm64.REG_V0
   812  	case ssa.OpARM64CSEL, ssa.OpARM64CSEL0:
   813  		r1 := int16(arm64.REGZERO)
   814  		if v.Op != ssa.OpARM64CSEL0 {
   815  			r1 = v.Args[1].Reg()
   816  		}
   817  		p := s.Prog(v.Op.Asm())
   818  		p.From.Type = obj.TYPE_REG // assembler encodes conditional bits in Reg
   819  		p.From.Reg = condBits[v.Aux.(ssa.Op)]
   820  		p.Reg = v.Args[0].Reg()
   821  		p.SetFrom3(obj.Addr{Type: obj.TYPE_REG, Reg: r1})
   822  		p.To.Type = obj.TYPE_REG
   823  		p.To.Reg = v.Reg()
   824  	case ssa.OpARM64DUFFZERO:
   825  		// runtime.duffzero expects start address in R20
   826  		p := s.Prog(obj.ADUFFZERO)
   827  		p.To.Type = obj.TYPE_MEM
   828  		p.To.Name = obj.NAME_EXTERN
   829  		p.To.Sym = gc.Duffzero
   830  		p.To.Offset = v.AuxInt
   831  	case ssa.OpARM64LoweredZero:
   832  		// STP.P	(ZR,ZR), 16(R16)
   833  		// CMP	Rarg1, R16
   834  		// BLE	-2(PC)
   835  		// arg1 is the address of the last 16-byte unit to zero
   836  		p := s.Prog(arm64.ASTP)
   837  		p.Scond = arm64.C_XPOST
   838  		p.From.Type = obj.TYPE_REGREG
   839  		p.From.Reg = arm64.REGZERO
   840  		p.From.Offset = int64(arm64.REGZERO)
   841  		p.To.Type = obj.TYPE_MEM
   842  		p.To.Reg = arm64.REG_R16
   843  		p.To.Offset = 16
   844  		p2 := s.Prog(arm64.ACMP)
   845  		p2.From.Type = obj.TYPE_REG
   846  		p2.From.Reg = v.Args[1].Reg()
   847  		p2.Reg = arm64.REG_R16
   848  		p3 := s.Prog(arm64.ABLE)
   849  		p3.To.Type = obj.TYPE_BRANCH
   850  		gc.Patch(p3, p)
   851  	case ssa.OpARM64DUFFCOPY:
   852  		p := s.Prog(obj.ADUFFCOPY)
   853  		p.To.Type = obj.TYPE_MEM
   854  		p.To.Name = obj.NAME_EXTERN
   855  		p.To.Sym = gc.Duffcopy
   856  		p.To.Offset = v.AuxInt
   857  	case ssa.OpARM64LoweredMove:
   858  		// MOVD.P	8(R16), Rtmp
   859  		// MOVD.P	Rtmp, 8(R17)
   860  		// CMP	Rarg2, R16
   861  		// BLE	-3(PC)
   862  		// arg2 is the address of the last element of src
   863  		p := s.Prog(arm64.AMOVD)
   864  		p.Scond = arm64.C_XPOST
   865  		p.From.Type = obj.TYPE_MEM
   866  		p.From.Reg = arm64.REG_R16
   867  		p.From.Offset = 8
   868  		p.To.Type = obj.TYPE_REG
   869  		p.To.Reg = arm64.REGTMP
   870  		p2 := s.Prog(arm64.AMOVD)
   871  		p2.Scond = arm64.C_XPOST
   872  		p2.From.Type = obj.TYPE_REG
   873  		p2.From.Reg = arm64.REGTMP
   874  		p2.To.Type = obj.TYPE_MEM
   875  		p2.To.Reg = arm64.REG_R17
   876  		p2.To.Offset = 8
   877  		p3 := s.Prog(arm64.ACMP)
   878  		p3.From.Type = obj.TYPE_REG
   879  		p3.From.Reg = v.Args[2].Reg()
   880  		p3.Reg = arm64.REG_R16
   881  		p4 := s.Prog(arm64.ABLE)
   882  		p4.To.Type = obj.TYPE_BRANCH
   883  		gc.Patch(p4, p)
   884  	case ssa.OpARM64CALLstatic, ssa.OpARM64CALLclosure, ssa.OpARM64CALLinter:
   885  		s.Call(v)
   886  	case ssa.OpARM64LoweredWB:
   887  		p := s.Prog(obj.ACALL)
   888  		p.To.Type = obj.TYPE_MEM
   889  		p.To.Name = obj.NAME_EXTERN
   890  		p.To.Sym = v.Aux.(*obj.LSym)
   891  	case ssa.OpARM64LoweredPanicBoundsA, ssa.OpARM64LoweredPanicBoundsB, ssa.OpARM64LoweredPanicBoundsC:
   892  		p := s.Prog(obj.ACALL)
   893  		p.To.Type = obj.TYPE_MEM
   894  		p.To.Name = obj.NAME_EXTERN
   895  		p.To.Sym = gc.BoundsCheckFunc[v.AuxInt]
   896  		s.UseArgs(16) // space used in callee args area by assembly stubs
   897  	case ssa.OpARM64LoweredNilCheck:
   898  		// Issue a load which will fault if arg is nil.
   899  		p := s.Prog(arm64.AMOVB)
   900  		p.From.Type = obj.TYPE_MEM
   901  		p.From.Reg = v.Args[0].Reg()
   902  		gc.AddAux(&p.From, v)
   903  		p.To.Type = obj.TYPE_REG
   904  		p.To.Reg = arm64.REGTMP
   905  		if logopt.Enabled() {
   906  			logopt.LogOpt(v.Pos, "nilcheck", "genssa", v.Block.Func.Name)
   907  		}
   908  		if gc.Debug_checknil != 0 && v.Pos.Line() > 1 { // v.Line==1 in generated wrappers
   909  			gc.Warnl(v.Pos, "generated nil check")
   910  		}
   911  	case ssa.OpARM64Equal,
   912  		ssa.OpARM64NotEqual,
   913  		ssa.OpARM64LessThan,
   914  		ssa.OpARM64LessEqual,
   915  		ssa.OpARM64GreaterThan,
   916  		ssa.OpARM64GreaterEqual,
   917  		ssa.OpARM64LessThanU,
   918  		ssa.OpARM64LessEqualU,
   919  		ssa.OpARM64GreaterThanU,
   920  		ssa.OpARM64GreaterEqualU,
   921  		ssa.OpARM64LessThanF,
   922  		ssa.OpARM64LessEqualF,
   923  		ssa.OpARM64GreaterThanF,
   924  		ssa.OpARM64GreaterEqualF:
   925  		// generate boolean values using CSET
   926  		p := s.Prog(arm64.ACSET)
   927  		p.From.Type = obj.TYPE_REG // assembler encodes conditional bits in Reg
   928  		p.From.Reg = condBits[v.Op]
   929  		p.To.Type = obj.TYPE_REG
   930  		p.To.Reg = v.Reg()
   931  	case ssa.OpARM64LoweredGetClosurePtr:
   932  		// Closure pointer is R26 (arm64.REGCTXT).
   933  		gc.CheckLoweredGetClosurePtr(v)
   934  	case ssa.OpARM64LoweredGetCallerSP:
   935  		// caller's SP is FixedFrameSize below the address of the first arg
   936  		p := s.Prog(arm64.AMOVD)
   937  		p.From.Type = obj.TYPE_ADDR
   938  		p.From.Offset = -gc.Ctxt.FixedFrameSize()
   939  		p.From.Name = obj.NAME_PARAM
   940  		p.To.Type = obj.TYPE_REG
   941  		p.To.Reg = v.Reg()
   942  	case ssa.OpARM64LoweredGetCallerPC:
   943  		p := s.Prog(obj.AGETCALLERPC)
   944  		p.To.Type = obj.TYPE_REG
   945  		p.To.Reg = v.Reg()
   946  	case ssa.OpARM64FlagConstant:
   947  		v.Fatalf("FlagConstant op should never make it to codegen %v", v.LongString())
   948  	case ssa.OpARM64InvertFlags:
   949  		v.Fatalf("InvertFlags should never make it to codegen %v", v.LongString())
   950  	case ssa.OpClobber:
   951  		// TODO: implement for clobberdead experiment. Nop is ok for now.
   952  	default:
   953  		v.Fatalf("genValue not implemented: %s", v.LongString())
   954  	}
   955  }
   956  
   957  var condBits = map[ssa.Op]int16{
   958  	ssa.OpARM64Equal:         arm64.COND_EQ,
   959  	ssa.OpARM64NotEqual:      arm64.COND_NE,
   960  	ssa.OpARM64LessThan:      arm64.COND_LT,
   961  	ssa.OpARM64LessThanU:     arm64.COND_LO,
   962  	ssa.OpARM64LessEqual:     arm64.COND_LE,
   963  	ssa.OpARM64LessEqualU:    arm64.COND_LS,
   964  	ssa.OpARM64GreaterThan:   arm64.COND_GT,
   965  	ssa.OpARM64GreaterThanU:  arm64.COND_HI,
   966  	ssa.OpARM64GreaterEqual:  arm64.COND_GE,
   967  	ssa.OpARM64GreaterEqualU: arm64.COND_HS,
   968  	ssa.OpARM64LessThanF:     arm64.COND_MI,
   969  	ssa.OpARM64LessEqualF:    arm64.COND_LS,
   970  	ssa.OpARM64GreaterThanF:  arm64.COND_GT,
   971  	ssa.OpARM64GreaterEqualF: arm64.COND_GE,
   972  }
   973  
   974  var blockJump = map[ssa.BlockKind]struct {
   975  	asm, invasm obj.As
   976  }{
   977  	ssa.BlockARM64EQ:     {arm64.ABEQ, arm64.ABNE},
   978  	ssa.BlockARM64NE:     {arm64.ABNE, arm64.ABEQ},
   979  	ssa.BlockARM64LT:     {arm64.ABLT, arm64.ABGE},
   980  	ssa.BlockARM64GE:     {arm64.ABGE, arm64.ABLT},
   981  	ssa.BlockARM64LE:     {arm64.ABLE, arm64.ABGT},
   982  	ssa.BlockARM64GT:     {arm64.ABGT, arm64.ABLE},
   983  	ssa.BlockARM64ULT:    {arm64.ABLO, arm64.ABHS},
   984  	ssa.BlockARM64UGE:    {arm64.ABHS, arm64.ABLO},
   985  	ssa.BlockARM64UGT:    {arm64.ABHI, arm64.ABLS},
   986  	ssa.BlockARM64ULE:    {arm64.ABLS, arm64.ABHI},
   987  	ssa.BlockARM64Z:      {arm64.ACBZ, arm64.ACBNZ},
   988  	ssa.BlockARM64NZ:     {arm64.ACBNZ, arm64.ACBZ},
   989  	ssa.BlockARM64ZW:     {arm64.ACBZW, arm64.ACBNZW},
   990  	ssa.BlockARM64NZW:    {arm64.ACBNZW, arm64.ACBZW},
   991  	ssa.BlockARM64TBZ:    {arm64.ATBZ, arm64.ATBNZ},
   992  	ssa.BlockARM64TBNZ:   {arm64.ATBNZ, arm64.ATBZ},
   993  	ssa.BlockARM64FLT:    {arm64.ABMI, arm64.ABPL},
   994  	ssa.BlockARM64FGE:    {arm64.ABGE, arm64.ABLT},
   995  	ssa.BlockARM64FLE:    {arm64.ABLS, arm64.ABHI},
   996  	ssa.BlockARM64FGT:    {arm64.ABGT, arm64.ABLE},
   997  	ssa.BlockARM64LTnoov: {arm64.ABMI, arm64.ABPL},
   998  	ssa.BlockARM64GEnoov: {arm64.ABPL, arm64.ABMI},
   999  }
  1000  
  1001  // To model a 'LEnoov' ('<=' without overflow checking) branching
  1002  var leJumps = [2][2]gc.IndexJump{
  1003  	{{Jump: arm64.ABEQ, Index: 0}, {Jump: arm64.ABPL, Index: 1}}, // next == b.Succs[0]
  1004  	{{Jump: arm64.ABMI, Index: 0}, {Jump: arm64.ABEQ, Index: 0}}, // next == b.Succs[1]
  1005  }
  1006  
  1007  // To model a 'GTnoov' ('>' without overflow checking) branching
  1008  var gtJumps = [2][2]gc.IndexJump{
  1009  	{{Jump: arm64.ABMI, Index: 1}, {Jump: arm64.ABEQ, Index: 1}}, // next == b.Succs[0]
  1010  	{{Jump: arm64.ABEQ, Index: 1}, {Jump: arm64.ABPL, Index: 0}}, // next == b.Succs[1]
  1011  }
  1012  
  1013  func ssaGenBlock(s *gc.SSAGenState, b, next *ssa.Block) {
  1014  	switch b.Kind {
  1015  	case ssa.BlockPlain:
  1016  		if b.Succs[0].Block() != next {
  1017  			p := s.Prog(obj.AJMP)
  1018  			p.To.Type = obj.TYPE_BRANCH
  1019  			s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
  1020  		}
  1021  
  1022  	case ssa.BlockDefer:
  1023  		// defer returns in R0:
  1024  		// 0 if we should continue executing
  1025  		// 1 if we should jump to deferreturn call
  1026  		p := s.Prog(arm64.ACMP)
  1027  		p.From.Type = obj.TYPE_CONST
  1028  		p.From.Offset = 0
  1029  		p.Reg = arm64.REG_R0
  1030  		p = s.Prog(arm64.ABNE)
  1031  		p.To.Type = obj.TYPE_BRANCH
  1032  		s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[1].Block()})
  1033  		if b.Succs[0].Block() != next {
  1034  			p := s.Prog(obj.AJMP)
  1035  			p.To.Type = obj.TYPE_BRANCH
  1036  			s.Branches = append(s.Branches, gc.Branch{P: p, B: b.Succs[0].Block()})
  1037  		}
  1038  
  1039  	case ssa.BlockExit:
  1040  
  1041  	case ssa.BlockRet:
  1042  		s.Prog(obj.ARET)
  1043  
  1044  	case ssa.BlockRetJmp:
  1045  		p := s.Prog(obj.ARET)
  1046  		p.To.Type = obj.TYPE_MEM
  1047  		p.To.Name = obj.NAME_EXTERN
  1048  		p.To.Sym = b.Aux.(*obj.LSym)
  1049  
  1050  	case ssa.BlockARM64EQ, ssa.BlockARM64NE,
  1051  		ssa.BlockARM64LT, ssa.BlockARM64GE,
  1052  		ssa.BlockARM64LE, ssa.BlockARM64GT,
  1053  		ssa.BlockARM64ULT, ssa.BlockARM64UGT,
  1054  		ssa.BlockARM64ULE, ssa.BlockARM64UGE,
  1055  		ssa.BlockARM64Z, ssa.BlockARM64NZ,
  1056  		ssa.BlockARM64ZW, ssa.BlockARM64NZW,
  1057  		ssa.BlockARM64FLT, ssa.BlockARM64FGE,
  1058  		ssa.BlockARM64FLE, ssa.BlockARM64FGT,
  1059  		ssa.BlockARM64LTnoov, ssa.BlockARM64GEnoov:
  1060  		jmp := blockJump[b.Kind]
  1061  		var p *obj.Prog
  1062  		switch next {
  1063  		case b.Succs[0].Block():
  1064  			p = s.Br(jmp.invasm, b.Succs[1].Block())
  1065  		case b.Succs[1].Block():
  1066  			p = s.Br(jmp.asm, b.Succs[0].Block())
  1067  		default:
  1068  			if b.Likely != ssa.BranchUnlikely {
  1069  				p = s.Br(jmp.asm, b.Succs[0].Block())
  1070  				s.Br(obj.AJMP, b.Succs[1].Block())
  1071  			} else {
  1072  				p = s.Br(jmp.invasm, b.Succs[1].Block())
  1073  				s.Br(obj.AJMP, b.Succs[0].Block())
  1074  			}
  1075  		}
  1076  		if !b.Controls[0].Type.IsFlags() {
  1077  			p.From.Type = obj.TYPE_REG
  1078  			p.From.Reg = b.Controls[0].Reg()
  1079  		}
  1080  	case ssa.BlockARM64TBZ, ssa.BlockARM64TBNZ:
  1081  		jmp := blockJump[b.Kind]
  1082  		var p *obj.Prog
  1083  		switch next {
  1084  		case b.Succs[0].Block():
  1085  			p = s.Br(jmp.invasm, b.Succs[1].Block())
  1086  		case b.Succs[1].Block():
  1087  			p = s.Br(jmp.asm, b.Succs[0].Block())
  1088  		default:
  1089  			if b.Likely != ssa.BranchUnlikely {
  1090  				p = s.Br(jmp.asm, b.Succs[0].Block())
  1091  				s.Br(obj.AJMP, b.Succs[1].Block())
  1092  			} else {
  1093  				p = s.Br(jmp.invasm, b.Succs[1].Block())
  1094  				s.Br(obj.AJMP, b.Succs[0].Block())
  1095  			}
  1096  		}
  1097  		p.From.Offset = b.AuxInt
  1098  		p.From.Type = obj.TYPE_CONST
  1099  		p.Reg = b.Controls[0].Reg()
  1100  
  1101  	case ssa.BlockARM64LEnoov:
  1102  		s.CombJump(b, next, &leJumps)
  1103  	case ssa.BlockARM64GTnoov:
  1104  		s.CombJump(b, next, &gtJumps)
  1105  	default:
  1106  		b.Fatalf("branch not implemented: %s", b.LongString())
  1107  	}
  1108  }
  1109  

View as plain text