Source file src/cmd/compile/internal/riscv64/ssa.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package riscv64
     6  
     7  import (
     8  	"cmd/compile/internal/base"
     9  	"cmd/compile/internal/ir"
    10  	"cmd/compile/internal/logopt"
    11  	"cmd/compile/internal/objw"
    12  	"cmd/compile/internal/ssa"
    13  	"cmd/compile/internal/ssa/block"
    14  	"cmd/compile/internal/ssa/ssaop"
    15  	"cmd/compile/internal/ssagen"
    16  	"cmd/compile/internal/types"
    17  	"cmd/internal/obj"
    18  	"cmd/internal/obj/riscv"
    19  	"internal/abi"
    20  	"internal/buildcfg"
    21  )
    22  
    23  // ssaRegToReg maps ssa register numbers to obj register numbers.
    24  var ssaRegToReg = []int16{
    25  	riscv.REG_X0,
    26  	// X1 (LR): unused
    27  	riscv.REG_X2,
    28  	riscv.REG_X3,
    29  	riscv.REG_X4,
    30  	riscv.REG_X5,
    31  	riscv.REG_X6,
    32  	riscv.REG_X7,
    33  	riscv.REG_X8,
    34  	riscv.REG_X9,
    35  	riscv.REG_X10,
    36  	riscv.REG_X11,
    37  	riscv.REG_X12,
    38  	riscv.REG_X13,
    39  	riscv.REG_X14,
    40  	riscv.REG_X15,
    41  	riscv.REG_X16,
    42  	riscv.REG_X17,
    43  	riscv.REG_X18,
    44  	riscv.REG_X19,
    45  	riscv.REG_X20,
    46  	riscv.REG_X21,
    47  	riscv.REG_X22,
    48  	riscv.REG_X23,
    49  	riscv.REG_X24,
    50  	riscv.REG_X25,
    51  	riscv.REG_X26,
    52  	riscv.REG_X27,
    53  	riscv.REG_X28,
    54  	riscv.REG_X29,
    55  	riscv.REG_X30,
    56  	riscv.REG_X31,
    57  	riscv.REG_F0,
    58  	riscv.REG_F1,
    59  	riscv.REG_F2,
    60  	riscv.REG_F3,
    61  	riscv.REG_F4,
    62  	riscv.REG_F5,
    63  	riscv.REG_F6,
    64  	riscv.REG_F7,
    65  	riscv.REG_F8,
    66  	riscv.REG_F9,
    67  	riscv.REG_F10,
    68  	riscv.REG_F11,
    69  	riscv.REG_F12,
    70  	riscv.REG_F13,
    71  	riscv.REG_F14,
    72  	riscv.REG_F15,
    73  	riscv.REG_F16,
    74  	riscv.REG_F17,
    75  	riscv.REG_F18,
    76  	riscv.REG_F19,
    77  	riscv.REG_F20,
    78  	riscv.REG_F21,
    79  	riscv.REG_F22,
    80  	riscv.REG_F23,
    81  	riscv.REG_F24,
    82  	riscv.REG_F25,
    83  	riscv.REG_F26,
    84  	riscv.REG_F27,
    85  	riscv.REG_F28,
    86  	riscv.REG_F29,
    87  	riscv.REG_F30,
    88  	riscv.REG_F31,
    89  	0, // SB isn't a real register.  We fill an Addr.Reg field with 0 in this case.
    90  }
    91  
    92  func loadByType(t *types.Type) obj.As {
    93  	width := t.Size()
    94  
    95  	if t.IsFloat() {
    96  		switch width {
    97  		case 4:
    98  			return riscv.AMOVF
    99  		case 8:
   100  			return riscv.AMOVD
   101  		default:
   102  			base.Fatalf("unknown float width for load %d in type %v", width, t)
   103  			return 0
   104  		}
   105  	}
   106  
   107  	switch width {
   108  	case 1:
   109  		if t.IsSigned() {
   110  			return riscv.AMOVB
   111  		} else {
   112  			return riscv.AMOVBU
   113  		}
   114  	case 2:
   115  		if t.IsSigned() {
   116  			return riscv.AMOVH
   117  		} else {
   118  			return riscv.AMOVHU
   119  		}
   120  	case 4:
   121  		if t.IsSigned() {
   122  			return riscv.AMOVW
   123  		} else {
   124  			return riscv.AMOVWU
   125  		}
   126  	case 8:
   127  		return riscv.AMOV
   128  	default:
   129  		base.Fatalf("unknown width for load %d in type %v", width, t)
   130  		return 0
   131  	}
   132  }
   133  
   134  // storeByType returns the store instruction of the given type.
   135  func storeByType(t *types.Type) obj.As {
   136  	width := t.Size()
   137  
   138  	if t.IsFloat() {
   139  		switch width {
   140  		case 4:
   141  			return riscv.AMOVF
   142  		case 8:
   143  			return riscv.AMOVD
   144  		default:
   145  			base.Fatalf("unknown float width for store %d in type %v", width, t)
   146  			return 0
   147  		}
   148  	}
   149  
   150  	switch width {
   151  	case 1:
   152  		return riscv.AMOVB
   153  	case 2:
   154  		return riscv.AMOVH
   155  	case 4:
   156  		return riscv.AMOVW
   157  	case 8:
   158  		return riscv.AMOV
   159  	default:
   160  		base.Fatalf("unknown width for store %d in type %v", width, t)
   161  		return 0
   162  	}
   163  }
   164  
   165  // largestMove returns the largest move instruction possible and its size,
   166  // given the alignment of the total size of the move.
   167  //
   168  // e.g., a 16-byte move may use MOV, but an 11-byte move must use MOVB.
   169  //
   170  // Note that the moves may not be on naturally aligned addresses depending on
   171  // the source and destination.
   172  //
   173  // This matches the calculation in ssa.moveSize.
   174  func largestMove(alignment int64) (obj.As, int64) {
   175  	switch {
   176  	case alignment%8 == 0:
   177  		return riscv.AMOV, 8
   178  	case alignment%4 == 0:
   179  		return riscv.AMOVW, 4
   180  	case alignment%2 == 0:
   181  		return riscv.AMOVH, 2
   182  	default:
   183  		return riscv.AMOVB, 1
   184  	}
   185  }
   186  
   187  var fracMovOps = []obj.As{riscv.AMOVB, riscv.AMOVH, riscv.AMOVW, riscv.AMOV}
   188  
   189  // ssaMarkMoves marks any MOVXconst ops that need to avoid clobbering flags.
   190  // RISC-V has no flags, so this is a no-op.
   191  func ssaMarkMoves(s *ssagen.State, b *ssa.Block) {}
   192  
   193  func ssaGenValue(s *ssagen.State, v *ssa.Value) {
   194  	switch v.Op {
   195  	case ssaop.OpInitMem:
   196  		// memory arg needs no code
   197  	case ssaop.OpArg:
   198  		// input args need no code
   199  	case ssaop.OpPhi:
   200  		ssagen.CheckLoweredPhi(v)
   201  	case ssaop.OpCopy, ssaop.OpRISCV64MOVDreg:
   202  		if v.Type.IsMemory() {
   203  			return
   204  		}
   205  		rs := v.Args[0].Reg()
   206  		rd := v.Reg()
   207  		if rs == rd {
   208  			return
   209  		}
   210  		as := riscv.AMOV
   211  		if v.Type.IsFloat() {
   212  			as = riscv.AMOVD
   213  		}
   214  		p := s.Prog(as)
   215  		p.From.Type = obj.TYPE_REG
   216  		p.From.Reg = rs
   217  		p.To.Type = obj.TYPE_REG
   218  		p.To.Reg = rd
   219  	case ssaop.OpLoadReg:
   220  		if v.Type.IsFlags() {
   221  			v.Fatalf("load flags not implemented: %v", v.LongString())
   222  			return
   223  		}
   224  		p := s.Prog(loadByType(v.Type))
   225  		ssagen.AddrAuto(&p.From, v.Args[0])
   226  		p.To.Type = obj.TYPE_REG
   227  		p.To.Reg = v.Reg()
   228  	case ssaop.OpStoreReg:
   229  		if v.Type.IsFlags() {
   230  			v.Fatalf("store flags not implemented: %v", v.LongString())
   231  			return
   232  		}
   233  		p := s.Prog(storeByType(v.Type))
   234  		p.From.Type = obj.TYPE_REG
   235  		p.From.Reg = v.Args[0].Reg()
   236  		ssagen.AddrAuto(&p.To, v)
   237  	case ssaop.OpArgIntReg, ssaop.OpArgFloatReg:
   238  		// The assembler needs to wrap the entry safepoint/stack growth code with spill/unspill
   239  		// The loop only runs once.
   240  		for _, a := range v.Block.Func.RegArgs {
   241  			// Pass the spill/unspill information along to the assembler, offset by size of
   242  			// the saved LR slot.
   243  			addr := ssagen.SpillSlotAddr(a, riscv.REG_SP, base.Ctxt.Arch.FixedFrameSize)
   244  			s.FuncInfo().AddSpill(
   245  				obj.RegSpill{Reg: a.Reg, Addr: addr, Unspill: loadByType(a.Type), Spill: storeByType(a.Type)})
   246  		}
   247  		v.Block.Func.RegArgs = nil
   248  
   249  		ssagen.CheckArgReg(v)
   250  	case ssaop.OpSP, ssaop.OpSB, ssaop.OpGetG:
   251  		// nothing to do
   252  	case ssaop.OpRISCV64MOVBreg, ssaop.OpRISCV64MOVHreg, ssaop.OpRISCV64MOVWreg,
   253  		ssaop.OpRISCV64MOVBUreg, ssaop.OpRISCV64MOVHUreg, ssaop.OpRISCV64MOVWUreg:
   254  		a := v.Args[0]
   255  		for a.Op == ssaop.OpCopy || a.Op == ssaop.OpRISCV64MOVDreg {
   256  			a = a.Args[0]
   257  		}
   258  		as := v.Op.Asm()
   259  		rs := v.Args[0].Reg()
   260  		rd := v.Reg()
   261  		if a.Op == ssaop.OpLoadReg {
   262  			t := a.Type
   263  			switch {
   264  			case v.Op == ssaop.OpRISCV64MOVBreg && t.Size() == 1 && t.IsSigned(),
   265  				v.Op == ssaop.OpRISCV64MOVHreg && t.Size() == 2 && t.IsSigned(),
   266  				v.Op == ssaop.OpRISCV64MOVWreg && t.Size() == 4 && t.IsSigned(),
   267  				v.Op == ssaop.OpRISCV64MOVBUreg && t.Size() == 1 && !t.IsSigned(),
   268  				v.Op == ssaop.OpRISCV64MOVHUreg && t.Size() == 2 && !t.IsSigned(),
   269  				v.Op == ssaop.OpRISCV64MOVWUreg && t.Size() == 4 && !t.IsSigned():
   270  				// arg is a proper-typed load and already sign/zero-extended
   271  				if rs == rd {
   272  					return
   273  				}
   274  				as = riscv.AMOV
   275  			default:
   276  			}
   277  		}
   278  		p := s.Prog(as)
   279  		p.From.Type = obj.TYPE_REG
   280  		p.From.Reg = rs
   281  		p.To.Type = obj.TYPE_REG
   282  		p.To.Reg = rd
   283  	case ssaop.OpRISCV64ADD, ssaop.OpRISCV64SUB, ssaop.OpRISCV64SUBW, ssaop.OpRISCV64XNOR, ssaop.OpRISCV64XOR,
   284  		ssaop.OpRISCV64OR, ssaop.OpRISCV64ORN, ssaop.OpRISCV64AND, ssaop.OpRISCV64ANDN,
   285  		ssaop.OpRISCV64SLL, ssaop.OpRISCV64SLLW, ssaop.OpRISCV64SRA, ssaop.OpRISCV64SRAW, ssaop.OpRISCV64SRL, ssaop.OpRISCV64SRLW,
   286  		ssaop.OpRISCV64SLT, ssaop.OpRISCV64SLTU, ssaop.OpRISCV64MUL, ssaop.OpRISCV64MULW, ssaop.OpRISCV64MULH,
   287  		ssaop.OpRISCV64MULHU, ssaop.OpRISCV64DIV, ssaop.OpRISCV64DIVU, ssaop.OpRISCV64DIVW,
   288  		ssaop.OpRISCV64DIVUW, ssaop.OpRISCV64REM, ssaop.OpRISCV64REMU, ssaop.OpRISCV64REMW,
   289  		ssaop.OpRISCV64REMUW,
   290  		ssaop.OpRISCV64BEXT, ssaop.OpRISCV64BINV, ssaop.OpRISCV64BSET,
   291  		ssaop.OpRISCV64ROL, ssaop.OpRISCV64ROLW, ssaop.OpRISCV64ROR, ssaop.OpRISCV64RORW,
   292  		ssaop.OpRISCV64FADDS, ssaop.OpRISCV64FSUBS, ssaop.OpRISCV64FMULS, ssaop.OpRISCV64FDIVS,
   293  		ssaop.OpRISCV64FEQS, ssaop.OpRISCV64FNES, ssaop.OpRISCV64FLTS, ssaop.OpRISCV64FLES,
   294  		ssaop.OpRISCV64FADDD, ssaop.OpRISCV64FSUBD, ssaop.OpRISCV64FMULD, ssaop.OpRISCV64FDIVD,
   295  		ssaop.OpRISCV64FEQD, ssaop.OpRISCV64FNED, ssaop.OpRISCV64FLTD, ssaop.OpRISCV64FLED, ssaop.OpRISCV64FSGNJD,
   296  		ssaop.OpRISCV64MIN, ssaop.OpRISCV64MAX, ssaop.OpRISCV64MINU, ssaop.OpRISCV64MAXU,
   297  		ssaop.OpRISCV64SH1ADD, ssaop.OpRISCV64SH2ADD, ssaop.OpRISCV64SH3ADD,
   298  		ssaop.OpRISCV64CZEROEQZ, ssaop.OpRISCV64CZERONEZ:
   299  		r := v.Reg()
   300  		r1 := v.Args[0].Reg()
   301  		r2 := v.Args[1].Reg()
   302  		p := s.Prog(v.Op.Asm())
   303  		p.From.Type = obj.TYPE_REG
   304  		p.From.Reg = r2
   305  		p.Reg = r1
   306  		p.To.Type = obj.TYPE_REG
   307  		p.To.Reg = r
   308  
   309  	case ssaop.OpRISCV64LoweredFMAXD, ssaop.OpRISCV64LoweredFMIND, ssaop.OpRISCV64LoweredFMAXS, ssaop.OpRISCV64LoweredFMINS:
   310  		// Most of FMIN/FMAX result match Go's required behaviour, unless one of the
   311  		// inputs is a NaN. As such, we need to explicitly test for NaN
   312  		// before using FMIN/FMAX.
   313  
   314  		// FADD Rarg0, Rarg1, Rout // FADD is used to propagate a NaN to the result in these cases.
   315  		// FEQ  Rarg0, Rarg0, Rtmp
   316  		// BEQZ Rtmp, end
   317  		// FEQ  Rarg1, Rarg1, Rtmp
   318  		// BEQZ Rtmp, end
   319  		// F(MIN | MAX)
   320  
   321  		r0 := v.Args[0].Reg()
   322  		r1 := v.Args[1].Reg()
   323  		out := v.Reg()
   324  		add, feq := riscv.AFADDD, riscv.AFEQD
   325  		if v.Op == ssaop.OpRISCV64LoweredFMAXS || v.Op == ssaop.OpRISCV64LoweredFMINS {
   326  			add = riscv.AFADDS
   327  			feq = riscv.AFEQS
   328  		}
   329  
   330  		p1 := s.Prog(add)
   331  		p1.From.Type = obj.TYPE_REG
   332  		p1.From.Reg = r0
   333  		p1.Reg = r1
   334  		p1.To.Type = obj.TYPE_REG
   335  		p1.To.Reg = out
   336  
   337  		p2 := s.Prog(feq)
   338  		p2.From.Type = obj.TYPE_REG
   339  		p2.From.Reg = r0
   340  		p2.Reg = r0
   341  		p2.To.Type = obj.TYPE_REG
   342  		p2.To.Reg = riscv.REG_TMP
   343  
   344  		p3 := s.Prog(riscv.ABEQ)
   345  		p3.From.Type = obj.TYPE_REG
   346  		p3.From.Reg = riscv.REG_ZERO
   347  		p3.Reg = riscv.REG_TMP
   348  		p3.To.Type = obj.TYPE_BRANCH
   349  
   350  		p4 := s.Prog(feq)
   351  		p4.From.Type = obj.TYPE_REG
   352  		p4.From.Reg = r1
   353  		p4.Reg = r1
   354  		p4.To.Type = obj.TYPE_REG
   355  		p4.To.Reg = riscv.REG_TMP
   356  
   357  		p5 := s.Prog(riscv.ABEQ)
   358  		p5.From.Type = obj.TYPE_REG
   359  		p5.From.Reg = riscv.REG_ZERO
   360  		p5.Reg = riscv.REG_TMP
   361  		p5.To.Type = obj.TYPE_BRANCH
   362  
   363  		p6 := s.Prog(v.Op.Asm())
   364  		p6.From.Type = obj.TYPE_REG
   365  		p6.From.Reg = r1
   366  		p6.Reg = r0
   367  		p6.To.Type = obj.TYPE_REG
   368  		p6.To.Reg = out
   369  
   370  		nop := s.Prog(obj.ANOP)
   371  		p3.To.SetTarget(nop)
   372  		p5.To.SetTarget(nop)
   373  
   374  	case ssaop.OpRISCV64LoweredRoundToEvenD, ssaop.OpRISCV64LoweredRoundD, ssaop.OpRISCV64LoweredFloorD, ssaop.OpRISCV64LoweredCeilD, ssaop.OpRISCV64LoweredTruncD:
   375  		rm := riscv.RM_RNE
   376  		switch v.Op {
   377  		case ssaop.OpRISCV64LoweredRoundD:
   378  			rm = riscv.RM_RMM
   379  		case ssaop.OpRISCV64LoweredFloorD:
   380  			rm = riscv.RM_RDN
   381  		case ssaop.OpRISCV64LoweredCeilD:
   382  			rm = riscv.RM_RUP
   383  		case ssaop.OpRISCV64LoweredTruncD:
   384  			rm = riscv.RM_RTZ
   385  		}
   386  		arg := v.Args[0].Reg()
   387  		out := v.Reg()
   388  
   389  		p := s.Prog(riscv.AFCVTLD)
   390  		p.Scond = riscv.RoundingModeSuffix(rm)
   391  		p.From.Type = obj.TYPE_REG
   392  		p.From.Reg = arg
   393  		p.To.Type = obj.TYPE_REG
   394  		p.To.Reg = riscv.REG_TMP
   395  
   396  		p = s.Prog(riscv.AFCVTDL)
   397  		p.From.Type = obj.TYPE_REG
   398  		p.From.Reg = riscv.REG_TMP
   399  		p.To.Type = obj.TYPE_REG
   400  		p.To.Reg = out
   401  
   402  		p = s.Prog(riscv.AFSGNJD)
   403  		p.From.Type = obj.TYPE_REG
   404  		p.From.Reg = arg
   405  		p.Reg = out
   406  		p.To.Type = obj.TYPE_REG
   407  		p.To.Reg = out
   408  
   409  	case ssaop.OpRISCV64LoweredMuluhilo:
   410  		r0 := v.Args[0].Reg()
   411  		r1 := v.Args[1].Reg()
   412  		p := s.Prog(riscv.AMULHU)
   413  		p.From.Type = obj.TYPE_REG
   414  		p.From.Reg = r1
   415  		p.Reg = r0
   416  		p.To.Type = obj.TYPE_REG
   417  		p.To.Reg = v.Reg0()
   418  		p1 := s.Prog(riscv.AMUL)
   419  		p1.From.Type = obj.TYPE_REG
   420  		p1.From.Reg = r1
   421  		p1.Reg = r0
   422  		p1.To.Type = obj.TYPE_REG
   423  		p1.To.Reg = v.Reg1()
   424  	case ssaop.OpRISCV64LoweredMuluover:
   425  		r0 := v.Args[0].Reg()
   426  		r1 := v.Args[1].Reg()
   427  		p := s.Prog(riscv.AMULHU)
   428  		p.From.Type = obj.TYPE_REG
   429  		p.From.Reg = r1
   430  		p.Reg = r0
   431  		p.To.Type = obj.TYPE_REG
   432  		p.To.Reg = v.Reg1()
   433  		p1 := s.Prog(riscv.AMUL)
   434  		p1.From.Type = obj.TYPE_REG
   435  		p1.From.Reg = r1
   436  		p1.Reg = r0
   437  		p1.To.Type = obj.TYPE_REG
   438  		p1.To.Reg = v.Reg0()
   439  		p2 := s.Prog(riscv.ASNEZ)
   440  		p2.From.Type = obj.TYPE_REG
   441  		p2.From.Reg = v.Reg1()
   442  		p2.To.Type = obj.TYPE_REG
   443  		p2.To.Reg = v.Reg1()
   444  	case ssaop.OpRISCV64FMADDD, ssaop.OpRISCV64FMSUBD, ssaop.OpRISCV64FNMADDD, ssaop.OpRISCV64FNMSUBD,
   445  		ssaop.OpRISCV64FMADDS, ssaop.OpRISCV64FMSUBS, ssaop.OpRISCV64FNMADDS, ssaop.OpRISCV64FNMSUBS:
   446  		r := v.Reg()
   447  		r1 := v.Args[0].Reg()
   448  		r2 := v.Args[1].Reg()
   449  		r3 := v.Args[2].Reg()
   450  		p := s.Prog(v.Op.Asm())
   451  		p.From.Type = obj.TYPE_REG
   452  		p.From.Reg = r2
   453  		p.Reg = r1
   454  		p.AddRestSource(obj.Addr{Type: obj.TYPE_REG, Reg: r3})
   455  		p.To.Type = obj.TYPE_REG
   456  		p.To.Reg = r
   457  	case ssaop.OpRISCV64FSQRTS, ssaop.OpRISCV64FSQRTD,
   458  		ssaop.OpRISCV64FNEGS, ssaop.OpRISCV64FNEGD,
   459  		ssaop.OpRISCV64FABSS, ssaop.OpRISCV64FABSD,
   460  		ssaop.OpRISCV64FMVSX, ssaop.OpRISCV64FMVXS, ssaop.OpRISCV64FMVDX, ssaop.OpRISCV64FMVXD,
   461  		ssaop.OpRISCV64FCVTSW, ssaop.OpRISCV64FCVTSL, ssaop.OpRISCV64FCVTWS, ssaop.OpRISCV64FCVTLS,
   462  		ssaop.OpRISCV64FCVTDW, ssaop.OpRISCV64FCVTDL, ssaop.OpRISCV64FCVTWD, ssaop.OpRISCV64FCVTLD, ssaop.OpRISCV64FCVTDS, ssaop.OpRISCV64FCVTSD,
   463  		ssaop.OpRISCV64FCLASSS, ssaop.OpRISCV64FCLASSD,
   464  		ssaop.OpRISCV64NOT, ssaop.OpRISCV64NEG, ssaop.OpRISCV64NEGW, ssaop.OpRISCV64CLZ, ssaop.OpRISCV64CLZW, ssaop.OpRISCV64CTZ, ssaop.OpRISCV64CTZW,
   465  		ssaop.OpRISCV64REV8, ssaop.OpRISCV64CPOP, ssaop.OpRISCV64CPOPW:
   466  		p := s.Prog(v.Op.Asm())
   467  		p.From.Type = obj.TYPE_REG
   468  		p.From.Reg = v.Args[0].Reg()
   469  		p.To.Type = obj.TYPE_REG
   470  		p.To.Reg = v.Reg()
   471  	case ssaop.OpRISCV64ADDI, ssaop.OpRISCV64ADDIW, ssaop.OpRISCV64XORI, ssaop.OpRISCV64ORI, ssaop.OpRISCV64ANDI,
   472  		ssaop.OpRISCV64SLLI, ssaop.OpRISCV64SLLIW, ssaop.OpRISCV64SRAI, ssaop.OpRISCV64SRAIW,
   473  		ssaop.OpRISCV64SRLI, ssaop.OpRISCV64SRLIW, ssaop.OpRISCV64SLTI, ssaop.OpRISCV64SLTIU,
   474  		ssaop.OpRISCV64BEXTI, ssaop.OpRISCV64BINVI, ssaop.OpRISCV64BSETI,
   475  		ssaop.OpRISCV64RORI, ssaop.OpRISCV64RORIW:
   476  		p := s.Prog(v.Op.Asm())
   477  		p.From.Type = obj.TYPE_CONST
   478  		p.From.Offset = v.AuxInt
   479  		p.Reg = v.Args[0].Reg()
   480  		p.To.Type = obj.TYPE_REG
   481  		p.To.Reg = v.Reg()
   482  	case ssaop.OpRISCV64MOVDconst:
   483  		p := s.Prog(v.Op.Asm())
   484  		p.From.Type = obj.TYPE_CONST
   485  		p.From.Offset = v.AuxInt
   486  		p.To.Type = obj.TYPE_REG
   487  		p.To.Reg = v.Reg()
   488  	case ssaop.OpRISCV64FMOVDconst, ssaop.OpRISCV64FMOVFconst:
   489  		p := s.Prog(v.Op.Asm())
   490  		p.From.Type = obj.TYPE_FCONST
   491  		p.From.Val = v.AuxFloat()
   492  		p.From.Name = obj.NAME_NONE
   493  		p.From.Reg = obj.REG_NONE
   494  		p.To.Type = obj.TYPE_REG
   495  		p.To.Reg = v.Reg()
   496  	case ssaop.OpRISCV64MOVaddr:
   497  		p := s.Prog(v.Op.Asm())
   498  		p.From.Type = obj.TYPE_ADDR
   499  		p.To.Type = obj.TYPE_REG
   500  		p.To.Reg = v.Reg()
   501  
   502  		var wantreg string
   503  		// MOVW $sym+off(base), R
   504  		switch v.Aux.(type) {
   505  		default:
   506  			v.Fatalf("aux is of unknown type %T", v.Aux)
   507  		case *obj.LSym:
   508  			wantreg = "SB"
   509  			ssagen.AddAux(&p.From, v)
   510  		case *ir.Name:
   511  			wantreg = "SP"
   512  			ssagen.AddAux(&p.From, v)
   513  		case nil:
   514  			// No sym, just MOVW $off(SP), R
   515  			wantreg = "SP"
   516  			p.From.Reg = riscv.REG_SP
   517  			p.From.Offset = v.AuxInt
   518  		}
   519  		if reg := v.Args[0].RegName(); reg != wantreg {
   520  			v.Fatalf("bad reg %s for symbol type %T, want %s", reg, v.Aux, wantreg)
   521  		}
   522  	case ssaop.OpRISCV64MOVBload, ssaop.OpRISCV64MOVHload, ssaop.OpRISCV64MOVWload, ssaop.OpRISCV64MOVDload,
   523  		ssaop.OpRISCV64MOVBUload, ssaop.OpRISCV64MOVHUload, ssaop.OpRISCV64MOVWUload,
   524  		ssaop.OpRISCV64FMOVWload, ssaop.OpRISCV64FMOVDload:
   525  		p := s.Prog(v.Op.Asm())
   526  		p.From.Type = obj.TYPE_MEM
   527  		p.From.Reg = v.Args[0].Reg()
   528  		ssagen.AddAux(&p.From, v)
   529  		p.To.Type = obj.TYPE_REG
   530  		p.To.Reg = v.Reg()
   531  	case ssaop.OpRISCV64MOVBstore, ssaop.OpRISCV64MOVHstore, ssaop.OpRISCV64MOVWstore, ssaop.OpRISCV64MOVDstore,
   532  		ssaop.OpRISCV64FMOVWstore, ssaop.OpRISCV64FMOVDstore:
   533  		p := s.Prog(v.Op.Asm())
   534  		p.From.Type = obj.TYPE_REG
   535  		p.From.Reg = v.Args[1].Reg()
   536  		p.To.Type = obj.TYPE_MEM
   537  		p.To.Reg = v.Args[0].Reg()
   538  		ssagen.AddAux(&p.To, v)
   539  	case ssaop.OpRISCV64MOVBstorezero, ssaop.OpRISCV64MOVHstorezero, ssaop.OpRISCV64MOVWstorezero, ssaop.OpRISCV64MOVDstorezero:
   540  		p := s.Prog(v.Op.Asm())
   541  		p.From.Type = obj.TYPE_REG
   542  		p.From.Reg = riscv.REG_ZERO
   543  		p.To.Type = obj.TYPE_MEM
   544  		p.To.Reg = v.Args[0].Reg()
   545  		ssagen.AddAux(&p.To, v)
   546  	case ssaop.OpRISCV64SEQZ, ssaop.OpRISCV64SNEZ:
   547  		p := s.Prog(v.Op.Asm())
   548  		p.From.Type = obj.TYPE_REG
   549  		p.From.Reg = v.Args[0].Reg()
   550  		p.To.Type = obj.TYPE_REG
   551  		p.To.Reg = v.Reg()
   552  	case ssaop.OpRISCV64CALLstatic, ssaop.OpRISCV64CALLclosure, ssaop.OpRISCV64CALLinter:
   553  		s.Call(v)
   554  	case ssaop.OpRISCV64CALLtail, ssaop.OpRISCV64CALLtailinter:
   555  		s.TailCall(v)
   556  	case ssaop.OpRISCV64LoweredWB:
   557  		p := s.Prog(obj.ACALL)
   558  		p.To.Type = obj.TYPE_MEM
   559  		p.To.Name = obj.NAME_EXTERN
   560  		// AuxInt encodes how many buffer entries we need.
   561  		p.To.Sym = ir.Syms.GCWriteBarrier[v.AuxInt-1]
   562  
   563  	case ssaop.OpRISCV64LoweredPanicBoundsRR, ssaop.OpRISCV64LoweredPanicBoundsRC, ssaop.OpRISCV64LoweredPanicBoundsCR, ssaop.OpRISCV64LoweredPanicBoundsCC:
   564  		// Compute the constant we put in the PCData entry for this call.
   565  		code, signed := ssa.BoundsKind(v.AuxInt).Code()
   566  		xIsReg := false
   567  		yIsReg := false
   568  		xVal := 0
   569  		yVal := 0
   570  		switch v.Op {
   571  		case ssaop.OpRISCV64LoweredPanicBoundsRR:
   572  			xIsReg = true
   573  			xVal = int(v.Args[0].Reg() - riscv.REG_X5)
   574  			yIsReg = true
   575  			yVal = int(v.Args[1].Reg() - riscv.REG_X5)
   576  		case ssaop.OpRISCV64LoweredPanicBoundsRC:
   577  			xIsReg = true
   578  			xVal = int(v.Args[0].Reg() - riscv.REG_X5)
   579  			c := v.Aux.(ssa.PanicBoundsC).C
   580  			if c >= 0 && c <= abi.BoundsMaxConst {
   581  				yVal = int(c)
   582  			} else {
   583  				// Move constant to a register
   584  				yIsReg = true
   585  				if yVal == xVal {
   586  					yVal = 1
   587  				}
   588  				p := s.Prog(riscv.AMOV)
   589  				p.From.Type = obj.TYPE_CONST
   590  				p.From.Offset = c
   591  				p.To.Type = obj.TYPE_REG
   592  				p.To.Reg = riscv.REG_X5 + int16(yVal)
   593  			}
   594  		case ssaop.OpRISCV64LoweredPanicBoundsCR:
   595  			yIsReg = true
   596  			yVal = int(v.Args[0].Reg() - riscv.REG_X5)
   597  			c := v.Aux.(ssa.PanicBoundsC).C
   598  			if c >= 0 && c <= abi.BoundsMaxConst {
   599  				xVal = int(c)
   600  			} else {
   601  				// Move constant to a register
   602  				if xVal == yVal {
   603  					xVal = 1
   604  				}
   605  				p := s.Prog(riscv.AMOV)
   606  				p.From.Type = obj.TYPE_CONST
   607  				p.From.Offset = c
   608  				p.To.Type = obj.TYPE_REG
   609  				p.To.Reg = riscv.REG_X5 + int16(xVal)
   610  			}
   611  		case ssaop.OpRISCV64LoweredPanicBoundsCC:
   612  			c := v.Aux.(ssa.PanicBoundsCC).Cx
   613  			if c >= 0 && c <= abi.BoundsMaxConst {
   614  				xVal = int(c)
   615  			} else {
   616  				// Move constant to a register
   617  				xIsReg = true
   618  				p := s.Prog(riscv.AMOV)
   619  				p.From.Type = obj.TYPE_CONST
   620  				p.From.Offset = c
   621  				p.To.Type = obj.TYPE_REG
   622  				p.To.Reg = riscv.REG_X5 + int16(xVal)
   623  			}
   624  			c = v.Aux.(ssa.PanicBoundsCC).Cy
   625  			if c >= 0 && c <= abi.BoundsMaxConst {
   626  				yVal = int(c)
   627  			} else {
   628  				// Move constant to a register
   629  				yIsReg = true
   630  				yVal = 1
   631  				p := s.Prog(riscv.AMOV)
   632  				p.From.Type = obj.TYPE_CONST
   633  				p.From.Offset = c
   634  				p.To.Type = obj.TYPE_REG
   635  				p.To.Reg = riscv.REG_X5 + int16(yVal)
   636  			}
   637  		}
   638  		c := abi.BoundsEncode(code, signed, xIsReg, yIsReg, xVal, yVal)
   639  
   640  		p := s.Prog(obj.APCDATA)
   641  		p.From.SetConst(abi.PCDATA_PanicBounds)
   642  		p.To.SetConst(int64(c))
   643  		p = s.Prog(obj.ACALL)
   644  		p.To.Type = obj.TYPE_MEM
   645  		p.To.Name = obj.NAME_EXTERN
   646  		p.To.Sym = ir.Syms.PanicBounds
   647  
   648  	case ssaop.OpRISCV64LoweredAtomicLoad8:
   649  		p1 := s.Prog(riscv.AFENCE)
   650  		p1.From.Type = obj.TYPE_SPECIAL
   651  		p1.From.Offset = int64(riscv.SPOP_FENCE_RW)
   652  		p1.To.Type = obj.TYPE_SPECIAL
   653  		p1.To.Offset = int64(riscv.SPOP_FENCE_RW)
   654  
   655  		p := s.Prog(riscv.AMOVBU)
   656  		p.From.Type = obj.TYPE_MEM
   657  		p.From.Reg = v.Args[0].Reg()
   658  		p.To.Type = obj.TYPE_REG
   659  		p.To.Reg = v.Reg0()
   660  
   661  		p2 := s.Prog(riscv.AFENCE)
   662  		p2.From.Type = obj.TYPE_SPECIAL
   663  		p2.From.Offset = int64(riscv.SPOP_FENCE_R)
   664  		p2.To.Type = obj.TYPE_SPECIAL
   665  		p2.To.Offset = int64(riscv.SPOP_FENCE_RW)
   666  
   667  	case ssaop.OpRISCV64LoweredAtomicLoad32, ssaop.OpRISCV64LoweredAtomicLoad64:
   668  		as := riscv.ALRW
   669  		if v.Op == ssaop.OpRISCV64LoweredAtomicLoad64 {
   670  			as = riscv.ALRD
   671  		}
   672  		p := s.Prog(as)
   673  		p.From.Type = obj.TYPE_MEM
   674  		p.From.Reg = v.Args[0].Reg()
   675  		p.To.Type = obj.TYPE_REG
   676  		p.To.Reg = v.Reg0()
   677  
   678  	case ssaop.OpRISCV64LoweredAtomicStore8:
   679  		p1 := s.Prog(riscv.AFENCE)
   680  		p1.From.Type = obj.TYPE_SPECIAL
   681  		p1.From.Offset = int64(riscv.SPOP_FENCE_RW)
   682  		p1.To.Type = obj.TYPE_SPECIAL
   683  		p1.To.Offset = int64(riscv.SPOP_FENCE_W)
   684  
   685  		p := s.Prog(riscv.AMOVB)
   686  		p.From.Type = obj.TYPE_REG
   687  		p.From.Reg = v.Args[1].Reg()
   688  		p.To.Type = obj.TYPE_MEM
   689  		p.To.Reg = v.Args[0].Reg()
   690  
   691  		p2 := s.Prog(riscv.AFENCE)
   692  		p2.From.Type = obj.TYPE_SPECIAL
   693  		p2.From.Offset = int64(riscv.SPOP_FENCE_RW)
   694  		p2.To.Type = obj.TYPE_SPECIAL
   695  		p2.To.Offset = int64(riscv.SPOP_FENCE_RW)
   696  
   697  	case ssaop.OpRISCV64LoweredAtomicStore32, ssaop.OpRISCV64LoweredAtomicStore64:
   698  		as := riscv.AAMOSWAPW
   699  		if v.Op == ssaop.OpRISCV64LoweredAtomicStore64 {
   700  			as = riscv.AAMOSWAPD
   701  		}
   702  		p := s.Prog(as)
   703  		p.From.Type = obj.TYPE_REG
   704  		p.From.Reg = v.Args[1].Reg()
   705  		p.To.Type = obj.TYPE_MEM
   706  		p.To.Reg = v.Args[0].Reg()
   707  		p.RegTo2 = riscv.REG_ZERO
   708  
   709  	case ssaop.OpRISCV64LoweredAtomicAdd32, ssaop.OpRISCV64LoweredAtomicAdd64:
   710  		as := riscv.AAMOADDW
   711  		if v.Op == ssaop.OpRISCV64LoweredAtomicAdd64 {
   712  			as = riscv.AAMOADDD
   713  		}
   714  		p := s.Prog(as)
   715  		p.From.Type = obj.TYPE_REG
   716  		p.From.Reg = v.Args[1].Reg()
   717  		p.To.Type = obj.TYPE_MEM
   718  		p.To.Reg = v.Args[0].Reg()
   719  		p.RegTo2 = riscv.REG_TMP
   720  
   721  		p2 := s.Prog(riscv.AADD)
   722  		p2.From.Type = obj.TYPE_REG
   723  		p2.From.Reg = riscv.REG_TMP
   724  		p2.Reg = v.Args[1].Reg()
   725  		p2.To.Type = obj.TYPE_REG
   726  		p2.To.Reg = v.Reg0()
   727  
   728  	case ssaop.OpRISCV64LoweredAtomicExchange32, ssaop.OpRISCV64LoweredAtomicExchange64:
   729  		as := riscv.AAMOSWAPW
   730  		if v.Op == ssaop.OpRISCV64LoweredAtomicExchange64 {
   731  			as = riscv.AAMOSWAPD
   732  		}
   733  		p := s.Prog(as)
   734  		p.From.Type = obj.TYPE_REG
   735  		p.From.Reg = v.Args[1].Reg()
   736  		p.To.Type = obj.TYPE_MEM
   737  		p.To.Reg = v.Args[0].Reg()
   738  		p.RegTo2 = v.Reg0()
   739  
   740  	case ssaop.OpRISCV64LoweredAtomicCas32, ssaop.OpRISCV64LoweredAtomicCas64:
   741  		// MOV  ZERO, Rout
   742  		// LR	(Rarg0), Rtmp
   743  		// BNE	Rtmp, Rarg1, 3(PC)
   744  		// SC	Rarg2, (Rarg0), Rtmp
   745  		// BNE	Rtmp, ZERO, -3(PC)
   746  		// MOV	$1, Rout
   747  
   748  		lr := riscv.ALRW
   749  		sc := riscv.ASCW
   750  		if v.Op == ssaop.OpRISCV64LoweredAtomicCas64 {
   751  			lr = riscv.ALRD
   752  			sc = riscv.ASCD
   753  		}
   754  
   755  		r0 := v.Args[0].Reg()
   756  		r1 := v.Args[1].Reg()
   757  		r2 := v.Args[2].Reg()
   758  		out := v.Reg0()
   759  
   760  		p := s.Prog(riscv.AMOV)
   761  		p.From.Type = obj.TYPE_REG
   762  		p.From.Reg = riscv.REG_ZERO
   763  		p.To.Type = obj.TYPE_REG
   764  		p.To.Reg = out
   765  
   766  		p1 := s.Prog(lr)
   767  		p1.From.Type = obj.TYPE_MEM
   768  		p1.From.Reg = r0
   769  		p1.To.Type = obj.TYPE_REG
   770  		p1.To.Reg = riscv.REG_TMP
   771  
   772  		p2 := s.Prog(riscv.ABNE)
   773  		p2.From.Type = obj.TYPE_REG
   774  		p2.From.Reg = r1
   775  		p2.Reg = riscv.REG_TMP
   776  		p2.To.Type = obj.TYPE_BRANCH
   777  
   778  		p3 := s.Prog(sc)
   779  		p3.From.Type = obj.TYPE_REG
   780  		p3.From.Reg = r2
   781  		p3.To.Type = obj.TYPE_MEM
   782  		p3.To.Reg = r0
   783  		p3.RegTo2 = riscv.REG_TMP
   784  
   785  		p4 := s.Prog(riscv.ABNE)
   786  		p4.From.Type = obj.TYPE_REG
   787  		p4.From.Reg = riscv.REG_TMP
   788  		p4.Reg = riscv.REG_ZERO
   789  		p4.To.Type = obj.TYPE_BRANCH
   790  		p4.To.SetTarget(p1)
   791  
   792  		p5 := s.Prog(riscv.AMOV)
   793  		p5.From.Type = obj.TYPE_CONST
   794  		p5.From.Offset = 1
   795  		p5.To.Type = obj.TYPE_REG
   796  		p5.To.Reg = out
   797  
   798  		p6 := s.Prog(obj.ANOP)
   799  		p2.To.SetTarget(p6)
   800  
   801  	case ssaop.OpRISCV64LoweredAtomicAnd32, ssaop.OpRISCV64LoweredAtomicOr32:
   802  		p := s.Prog(v.Op.Asm())
   803  		p.From.Type = obj.TYPE_REG
   804  		p.From.Reg = v.Args[1].Reg()
   805  		p.To.Type = obj.TYPE_MEM
   806  		p.To.Reg = v.Args[0].Reg()
   807  		p.RegTo2 = riscv.REG_ZERO
   808  
   809  	case ssaop.OpRISCV64LoweredAtomicAnd32value, ssaop.OpRISCV64LoweredAtomicAnd64value,
   810  		ssaop.OpRISCV64LoweredAtomicOr32value, ssaop.OpRISCV64LoweredAtomicOr64value:
   811  		p := s.Prog(v.Op.Asm())
   812  		p.From.Type = obj.TYPE_REG
   813  		p.From.Reg = v.Args[1].Reg()
   814  		p.To.Type = obj.TYPE_MEM
   815  		p.To.Reg = v.Args[0].Reg()
   816  		p.RegTo2 = v.Reg0()
   817  
   818  	case ssaop.OpRISCV64LoweredZero:
   819  		ptr := v.Args[0].Reg()
   820  		n, align := v.AuxSizeAndAlign()
   821  		mov, sz := largestMove(align)
   822  
   823  		// mov	ZERO, (offset)(Rarg0)
   824  		var off int64
   825  		for n >= sz {
   826  			zeroOp(s, mov, ptr, off)
   827  			off += sz
   828  			n -= sz
   829  		}
   830  
   831  		for i := len(fracMovOps) - 1; i >= 0; i-- {
   832  			tsz := int64(1 << i)
   833  			if n < tsz {
   834  				continue
   835  			}
   836  			zeroOp(s, fracMovOps[i], ptr, off)
   837  			off += tsz
   838  			n -= tsz
   839  		}
   840  
   841  	case ssaop.OpRISCV64LoweredZeroLoop:
   842  		ptr := v.Args[0].Reg()
   843  		n, align := v.AuxSizeAndAlign()
   844  		mov, sz := largestMove(align)
   845  		chunk := 8 * sz
   846  
   847  		if n <= 3*chunk {
   848  			v.Fatalf("ZeroLoop too small:%d, expect:%d", n, 3*chunk)
   849  		}
   850  
   851  		tmp := v.RegTmp()
   852  
   853  		if n >= 1<<31 {
   854  			p := s.Prog(riscv.AMOV)
   855  			p.From.Type = obj.TYPE_CONST
   856  			p.From.Offset = n - n%chunk
   857  			p.To.Type = obj.TYPE_REG
   858  			p.To.Reg = tmp
   859  		}
   860  		p := s.Prog(riscv.AADD)
   861  		if n >= 1<<31 {
   862  			p.From.Type = obj.TYPE_REG
   863  			p.From.Reg = tmp
   864  		} else {
   865  			p.From.Type = obj.TYPE_CONST
   866  			p.From.Offset = n - n%chunk
   867  		}
   868  		p.Reg = ptr
   869  		p.To.Type = obj.TYPE_REG
   870  		p.To.Reg = tmp
   871  
   872  		for i := int64(0); i < 8; i++ {
   873  			zeroOp(s, mov, ptr, sz*i)
   874  		}
   875  
   876  		p2 := s.Prog(riscv.AADD)
   877  		p2.From.Type = obj.TYPE_CONST
   878  		p2.From.Offset = chunk
   879  		p2.To.Type = obj.TYPE_REG
   880  		p2.To.Reg = ptr
   881  
   882  		p3 := s.Prog(riscv.ABNE)
   883  		p3.From.Reg = tmp
   884  		p3.From.Type = obj.TYPE_REG
   885  		p3.Reg = ptr
   886  		p3.To.Type = obj.TYPE_BRANCH
   887  		p3.To.SetTarget(p.Link)
   888  
   889  		n %= chunk
   890  
   891  		// mov	ZERO, (offset)(Rarg0)
   892  		var off int64
   893  		for n >= sz {
   894  			zeroOp(s, mov, ptr, off)
   895  			off += sz
   896  			n -= sz
   897  		}
   898  
   899  		for i := len(fracMovOps) - 1; i >= 0; i-- {
   900  			tsz := int64(1 << i)
   901  			if n < tsz {
   902  				continue
   903  			}
   904  			zeroOp(s, fracMovOps[i], ptr, off)
   905  			off += tsz
   906  			n -= tsz
   907  		}
   908  
   909  	case ssaop.OpRISCV64LoweredMove:
   910  		dst := v.Args[0].Reg()
   911  		src := v.Args[1].Reg()
   912  		if dst == src {
   913  			break
   914  		}
   915  
   916  		n, align := v.AuxSizeAndAlign()
   917  		mov, sz := largestMove(align)
   918  
   919  		var off int64
   920  		tmp := int16(riscv.REG_X5)
   921  		for n >= sz {
   922  			moveOp(s, mov, dst, src, tmp, off)
   923  			off += sz
   924  			n -= sz
   925  		}
   926  
   927  		for i := len(fracMovOps) - 1; i >= 0; i-- {
   928  			tsz := int64(1 << i)
   929  			if n < tsz {
   930  				continue
   931  			}
   932  			moveOp(s, fracMovOps[i], dst, src, tmp, off)
   933  			off += tsz
   934  			n -= tsz
   935  		}
   936  
   937  	case ssaop.OpRISCV64LoweredMoveLoop:
   938  		dst := v.Args[0].Reg()
   939  		src := v.Args[1].Reg()
   940  		if dst == src {
   941  			break
   942  		}
   943  
   944  		n, align := v.AuxSizeAndAlign()
   945  		mov, sz := largestMove(align)
   946  		chunk := 8 * sz
   947  
   948  		if n <= 3*chunk {
   949  			v.Fatalf("MoveLoop too small:%d, expect:%d", n, 3*chunk)
   950  		}
   951  		tmp := int16(riscv.REG_X5)
   952  
   953  		if n >= 1<<31 {
   954  			p := s.Prog(riscv.AMOV)
   955  			p.From.Type = obj.TYPE_CONST
   956  			p.From.Offset = n - n%chunk
   957  			p.To.Type = obj.TYPE_REG
   958  			p.To.Reg = riscv.REG_X6
   959  		}
   960  		p := s.Prog(riscv.AADD)
   961  		if n >= 1<<31 {
   962  			p.From.Type = obj.TYPE_REG
   963  			p.From.Reg = riscv.REG_X6
   964  		} else {
   965  			p.From.Type = obj.TYPE_CONST
   966  			p.From.Offset = n - n%chunk
   967  		}
   968  		p.Reg = src
   969  		p.To.Type = obj.TYPE_REG
   970  		p.To.Reg = riscv.REG_X6
   971  
   972  		for i := int64(0); i < 8; i++ {
   973  			moveOp(s, mov, dst, src, tmp, sz*i)
   974  		}
   975  
   976  		p1 := s.Prog(riscv.AADD)
   977  		p1.From.Type = obj.TYPE_CONST
   978  		p1.From.Offset = chunk
   979  		p1.To.Type = obj.TYPE_REG
   980  		p1.To.Reg = src
   981  
   982  		p2 := s.Prog(riscv.AADD)
   983  		p2.From.Type = obj.TYPE_CONST
   984  		p2.From.Offset = chunk
   985  		p2.To.Type = obj.TYPE_REG
   986  		p2.To.Reg = dst
   987  
   988  		p3 := s.Prog(riscv.ABNE)
   989  		p3.From.Reg = riscv.REG_X6
   990  		p3.From.Type = obj.TYPE_REG
   991  		p3.Reg = src
   992  		p3.To.Type = obj.TYPE_BRANCH
   993  		p3.To.SetTarget(p.Link)
   994  
   995  		n %= chunk
   996  
   997  		var off int64
   998  		for n >= sz {
   999  			moveOp(s, mov, dst, src, tmp, off)
  1000  			off += sz
  1001  			n -= sz
  1002  		}
  1003  
  1004  		for i := len(fracMovOps) - 1; i >= 0; i-- {
  1005  			tsz := int64(1 << i)
  1006  			if n < tsz {
  1007  				continue
  1008  			}
  1009  			moveOp(s, fracMovOps[i], dst, src, tmp, off)
  1010  			off += tsz
  1011  			n -= tsz
  1012  		}
  1013  
  1014  	case ssaop.OpRISCV64LoweredNilCheck:
  1015  		// Issue a load which will fault if arg is nil.
  1016  		p := s.Prog(riscv.AMOVB)
  1017  		p.From.Type = obj.TYPE_MEM
  1018  		p.From.Reg = v.Args[0].Reg()
  1019  		ssagen.AddAux(&p.From, v)
  1020  		p.To.Type = obj.TYPE_REG
  1021  		p.To.Reg = riscv.REG_ZERO
  1022  		if logopt.Enabled() {
  1023  			logopt.LogOpt(v.Pos, "nilcheck", "genssa", v.Block.Func.Name)
  1024  		}
  1025  		if base.Debug.Nil != 0 && v.Pos.Line() > 1 { // v.Pos == 1 in generated wrappers
  1026  			base.WarnfAt(v.Pos, "generated nil check")
  1027  		}
  1028  
  1029  	case ssaop.OpRISCV64LoweredGetClosurePtr:
  1030  		// Closure pointer is S10 (riscv.REG_CTXT).
  1031  		ssagen.CheckLoweredGetClosurePtr(v)
  1032  
  1033  	case ssaop.OpRISCV64LoweredGetCallerSP:
  1034  		// caller's SP is FixedFrameSize below the address of the first arg
  1035  		p := s.Prog(riscv.AMOV)
  1036  		p.From.Type = obj.TYPE_ADDR
  1037  		p.From.Offset = -base.Ctxt.Arch.FixedFrameSize
  1038  		p.From.Name = obj.NAME_PARAM
  1039  		p.To.Type = obj.TYPE_REG
  1040  		p.To.Reg = v.Reg()
  1041  
  1042  	case ssaop.OpRISCV64LoweredGetCallerPC:
  1043  		p := s.Prog(obj.AGETCALLERPC)
  1044  		p.To.Type = obj.TYPE_REG
  1045  		p.To.Reg = v.Reg()
  1046  
  1047  	case ssaop.OpRISCV64LoweredPubBarrier:
  1048  		// FENCE W, W
  1049  		p := s.Prog(v.Op.Asm())
  1050  		p.From.Type = obj.TYPE_SPECIAL
  1051  		p.From.Offset = int64(riscv.SPOP_FENCE_W)
  1052  		p.To.Type = obj.TYPE_SPECIAL
  1053  		p.To.Offset = int64(riscv.SPOP_FENCE_W)
  1054  
  1055  	case ssaop.OpRISCV64LoweredRound32F, ssaop.OpRISCV64LoweredRound64F:
  1056  		// input is already rounded
  1057  
  1058  	case ssaop.OpClobber:
  1059  		// MOV $0xdeaddead, REGTMP
  1060  		// MOVW REGTMP, (slot)
  1061  		// MOVW REGTMP, 4(slot)
  1062  		p := s.Prog(riscv.AMOV)
  1063  		p.From.Type = obj.TYPE_CONST
  1064  		p.From.Offset = 0xdeaddead
  1065  		p.To.Type = obj.TYPE_REG
  1066  		p.To.Reg = riscv.REG_TMP
  1067  		p = s.Prog(riscv.AMOVW)
  1068  		p.From.Type = obj.TYPE_REG
  1069  		p.From.Reg = riscv.REG_TMP
  1070  		p.To.Type = obj.TYPE_MEM
  1071  		p.To.Reg = riscv.REG_SP
  1072  		ssagen.AddAux(&p.To, v)
  1073  		p = s.Prog(riscv.AMOVW)
  1074  		p.From.Type = obj.TYPE_REG
  1075  		p.From.Reg = riscv.REG_TMP
  1076  		p.To.Type = obj.TYPE_MEM
  1077  		p.To.Reg = riscv.REG_SP
  1078  		ssagen.AddAux2(&p.To, v, v.AuxInt+4)
  1079  	case ssaop.OpClobberReg:
  1080  		x := uint64(0xdeaddeaddeaddead)
  1081  		p := s.Prog(riscv.AMOV)
  1082  		p.From.Type = obj.TYPE_CONST
  1083  		p.From.Offset = int64(x)
  1084  		p.To.Type = obj.TYPE_REG
  1085  		p.To.Reg = v.Reg()
  1086  
  1087  	default:
  1088  		v.Fatalf("Unhandled op %v", v.Op)
  1089  	}
  1090  }
  1091  
  1092  var blockBranch = [...]obj.As{
  1093  	block.BlockRISCV64BEQ:  riscv.ABEQ,
  1094  	block.BlockRISCV64BEQZ: riscv.ABEQZ,
  1095  	block.BlockRISCV64BGE:  riscv.ABGE,
  1096  	block.BlockRISCV64BGEU: riscv.ABGEU,
  1097  	block.BlockRISCV64BGEZ: riscv.ABGEZ,
  1098  	block.BlockRISCV64BGTZ: riscv.ABGTZ,
  1099  	block.BlockRISCV64BLEZ: riscv.ABLEZ,
  1100  	block.BlockRISCV64BLT:  riscv.ABLT,
  1101  	block.BlockRISCV64BLTU: riscv.ABLTU,
  1102  	block.BlockRISCV64BLTZ: riscv.ABLTZ,
  1103  	block.BlockRISCV64BNE:  riscv.ABNE,
  1104  	block.BlockRISCV64BNEZ: riscv.ABNEZ,
  1105  }
  1106  
  1107  func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
  1108  
  1109  	switch b.Kind {
  1110  	case block.BlockPlain, block.BlockDefer:
  1111  		if b.Succs[0].Block() != next {
  1112  			p := s.Prog(obj.AJMP)
  1113  			p.To.Type = obj.TYPE_BRANCH
  1114  			s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
  1115  		}
  1116  	case block.BlockExit, block.BlockRetJmp:
  1117  	case block.BlockRet:
  1118  		s.Prog(obj.ARET)
  1119  	case block.BlockRISCV64BEQ, block.BlockRISCV64BEQZ, block.BlockRISCV64BNE, block.BlockRISCV64BNEZ,
  1120  		block.BlockRISCV64BLT, block.BlockRISCV64BLEZ, block.BlockRISCV64BGE, block.BlockRISCV64BGEZ,
  1121  		block.BlockRISCV64BLTZ, block.BlockRISCV64BGTZ, block.BlockRISCV64BLTU, block.BlockRISCV64BGEU:
  1122  
  1123  		as := blockBranch[b.Kind]
  1124  		invAs := riscv.InvertBranch(as)
  1125  
  1126  		var p *obj.Prog
  1127  		switch next {
  1128  		case b.Succs[0].Block():
  1129  			p = s.Br(invAs, b.Succs[1].Block())
  1130  		case b.Succs[1].Block():
  1131  			p = s.Br(as, b.Succs[0].Block())
  1132  		default:
  1133  			if b.Likely != ssa.BranchUnlikely {
  1134  				p = s.Br(as, b.Succs[0].Block())
  1135  				s.Br(obj.AJMP, b.Succs[1].Block())
  1136  			} else {
  1137  				p = s.Br(invAs, b.Succs[1].Block())
  1138  				s.Br(obj.AJMP, b.Succs[0].Block())
  1139  			}
  1140  		}
  1141  
  1142  		p.From.Type = obj.TYPE_REG
  1143  		switch b.Kind {
  1144  		case block.BlockRISCV64BEQ, block.BlockRISCV64BNE, block.BlockRISCV64BLT, block.BlockRISCV64BGE, block.BlockRISCV64BLTU, block.BlockRISCV64BGEU:
  1145  			if b.NumControls() != 2 {
  1146  				b.Fatalf("Unexpected number of controls (%d != 2): %s", b.NumControls(), b.LongString())
  1147  			}
  1148  			p.From.Reg = b.Controls[0].Reg()
  1149  			p.Reg = b.Controls[1].Reg()
  1150  
  1151  		case block.BlockRISCV64BEQZ, block.BlockRISCV64BNEZ, block.BlockRISCV64BGEZ, block.BlockRISCV64BLEZ, block.BlockRISCV64BLTZ, block.BlockRISCV64BGTZ:
  1152  			if b.NumControls() != 1 {
  1153  				b.Fatalf("Unexpected number of controls (%d != 1): %s", b.NumControls(), b.LongString())
  1154  			}
  1155  			p.From.Reg = b.Controls[0].Reg()
  1156  		}
  1157  
  1158  	case block.BlockRISCV64JUMPTABLE:
  1159  		// Jump table:
  1160  		// TMP = base + index*8 (SH3ADD if Zba else SLLI+ADD).
  1161  		// Load slot into TMP, then indirect JMP through TMP.
  1162  		var p *obj.Prog
  1163  		if buildcfg.GORISCV64 >= 22 {
  1164  			p = s.Prog(riscv.ASH3ADD)
  1165  			p.From.Type = obj.TYPE_REG
  1166  			p.From.Reg = b.Controls[1].Reg()
  1167  			p.Reg = b.Controls[0].Reg()
  1168  			p.To.Type = obj.TYPE_REG
  1169  			p.To.Reg = riscv.REG_TMP
  1170  		} else {
  1171  			p = s.Prog(riscv.ASLLI)
  1172  			p.From.Type = obj.TYPE_CONST
  1173  			p.From.Offset = 3
  1174  			p.Reg = b.Controls[0].Reg()
  1175  			p.To.Type = obj.TYPE_REG
  1176  			p.To.Reg = riscv.REG_TMP
  1177  
  1178  			p = s.Prog(riscv.AADD)
  1179  			p.From.Type = obj.TYPE_REG
  1180  			p.From.Reg = riscv.REG_TMP
  1181  			p.Reg = b.Controls[1].Reg()
  1182  			p.To.Type = obj.TYPE_REG
  1183  			p.To.Reg = riscv.REG_TMP
  1184  		}
  1185  
  1186  		p = s.Prog(riscv.AMOV)
  1187  		p.From.Type = obj.TYPE_MEM
  1188  		p.From.Reg = riscv.REG_TMP
  1189  		p.To.Type = obj.TYPE_REG
  1190  		p.To.Reg = riscv.REG_TMP
  1191  
  1192  		p = s.Prog(obj.AJMP)
  1193  		p.To.Type = obj.TYPE_MEM
  1194  		p.To.Reg = riscv.REG_TMP
  1195  		// Save jump tables for later resolution of the target blocks.
  1196  		s.JumpTables = append(s.JumpTables, b)
  1197  
  1198  	default:
  1199  		b.Fatalf("Unhandled block: %s", b.LongString())
  1200  	}
  1201  }
  1202  
  1203  func loadRegResult(s *ssagen.State, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  1204  	p := s.Prog(loadByType(t))
  1205  	p.From.Type = obj.TYPE_MEM
  1206  	p.From.Name = obj.NAME_AUTO
  1207  	p.From.Sym = n.Linksym()
  1208  	p.From.Offset = n.FrameOffset() + off
  1209  	p.To.Type = obj.TYPE_REG
  1210  	p.To.Reg = reg
  1211  	return p
  1212  }
  1213  
  1214  func spillArgReg(pp *objw.Progs, p *obj.Prog, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  1215  	p = pp.Append(p, storeByType(t), obj.TYPE_REG, reg, 0, obj.TYPE_MEM, 0, n.FrameOffset()+off)
  1216  	p.To.Name = obj.NAME_PARAM
  1217  	p.To.Sym = n.Linksym()
  1218  	p.Pos = p.Pos.WithNotStmt()
  1219  	return p
  1220  }
  1221  
  1222  func zeroOp(s *ssagen.State, mov obj.As, reg int16, off int64) {
  1223  	p := s.Prog(mov)
  1224  	p.From.Type = obj.TYPE_REG
  1225  	p.From.Reg = riscv.REG_ZERO
  1226  	p.To.Type = obj.TYPE_MEM
  1227  	p.To.Reg = reg
  1228  	p.To.Offset = off
  1229  	return
  1230  }
  1231  
  1232  func moveOp(s *ssagen.State, mov obj.As, dst int16, src int16, tmp int16, off int64) {
  1233  	p := s.Prog(mov)
  1234  	p.From.Type = obj.TYPE_MEM
  1235  	p.From.Reg = src
  1236  	p.From.Offset = off
  1237  	p.To.Type = obj.TYPE_REG
  1238  	p.To.Reg = tmp
  1239  
  1240  	p1 := s.Prog(mov)
  1241  	p1.From.Type = obj.TYPE_REG
  1242  	p1.From.Reg = tmp
  1243  	p1.To.Type = obj.TYPE_MEM
  1244  	p1.To.Reg = dst
  1245  	p1.To.Offset = off
  1246  
  1247  	return
  1248  }
  1249  

View as plain text