Source file src/cmd/compile/internal/riscv64/ssa.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package riscv64
     6  
     7  import (
     8  	"cmd/compile/internal/base"
     9  	"cmd/compile/internal/ir"
    10  	"cmd/compile/internal/logopt"
    11  	"cmd/compile/internal/objw"
    12  	"cmd/compile/internal/ssa"
    13  	"cmd/compile/internal/ssa/block"
    14  	"cmd/compile/internal/ssa/ssaop"
    15  	"cmd/compile/internal/ssagen"
    16  	"cmd/compile/internal/types"
    17  	"cmd/internal/obj"
    18  	"cmd/internal/obj/riscv"
    19  	"internal/abi"
    20  	"internal/buildcfg"
    21  )
    22  
    23  // ssaRegToReg maps ssa register numbers to obj register numbers.
    24  var ssaRegToReg = []int16{
    25  	riscv.REG_X0,
    26  	// X1 (LR): unused
    27  	riscv.REG_X2,
    28  	riscv.REG_X3,
    29  	riscv.REG_X4,
    30  	riscv.REG_X5,
    31  	riscv.REG_X6,
    32  	riscv.REG_X7,
    33  	riscv.REG_X8,
    34  	riscv.REG_X9,
    35  	riscv.REG_X10,
    36  	riscv.REG_X11,
    37  	riscv.REG_X12,
    38  	riscv.REG_X13,
    39  	riscv.REG_X14,
    40  	riscv.REG_X15,
    41  	riscv.REG_X16,
    42  	riscv.REG_X17,
    43  	riscv.REG_X18,
    44  	riscv.REG_X19,
    45  	riscv.REG_X20,
    46  	riscv.REG_X21,
    47  	riscv.REG_X22,
    48  	riscv.REG_X23,
    49  	riscv.REG_X24,
    50  	riscv.REG_X25,
    51  	riscv.REG_X26,
    52  	riscv.REG_X27,
    53  	riscv.REG_X28,
    54  	riscv.REG_X29,
    55  	riscv.REG_X30,
    56  	riscv.REG_X31,
    57  	riscv.REG_F0,
    58  	riscv.REG_F1,
    59  	riscv.REG_F2,
    60  	riscv.REG_F3,
    61  	riscv.REG_F4,
    62  	riscv.REG_F5,
    63  	riscv.REG_F6,
    64  	riscv.REG_F7,
    65  	riscv.REG_F8,
    66  	riscv.REG_F9,
    67  	riscv.REG_F10,
    68  	riscv.REG_F11,
    69  	riscv.REG_F12,
    70  	riscv.REG_F13,
    71  	riscv.REG_F14,
    72  	riscv.REG_F15,
    73  	riscv.REG_F16,
    74  	riscv.REG_F17,
    75  	riscv.REG_F18,
    76  	riscv.REG_F19,
    77  	riscv.REG_F20,
    78  	riscv.REG_F21,
    79  	riscv.REG_F22,
    80  	riscv.REG_F23,
    81  	riscv.REG_F24,
    82  	riscv.REG_F25,
    83  	riscv.REG_F26,
    84  	riscv.REG_F27,
    85  	riscv.REG_F28,
    86  	riscv.REG_F29,
    87  	riscv.REG_F30,
    88  	riscv.REG_F31,
    89  	0, // SB isn't a real register.  We fill an Addr.Reg field with 0 in this case.
    90  }
    91  
    92  func loadByType(t *types.Type) obj.As {
    93  	width := t.Size()
    94  
    95  	if t.IsFloat() {
    96  		switch width {
    97  		case 4:
    98  			return riscv.AMOVF
    99  		case 8:
   100  			return riscv.AMOVD
   101  		default:
   102  			base.Fatalf("unknown float width for load %d in type %v", width, t)
   103  			return 0
   104  		}
   105  	}
   106  
   107  	switch width {
   108  	case 1:
   109  		if t.IsSigned() {
   110  			return riscv.AMOVB
   111  		} else {
   112  			return riscv.AMOVBU
   113  		}
   114  	case 2:
   115  		if t.IsSigned() {
   116  			return riscv.AMOVH
   117  		} else {
   118  			return riscv.AMOVHU
   119  		}
   120  	case 4:
   121  		if t.IsSigned() {
   122  			return riscv.AMOVW
   123  		} else {
   124  			return riscv.AMOVWU
   125  		}
   126  	case 8:
   127  		return riscv.AMOV
   128  	default:
   129  		base.Fatalf("unknown width for load %d in type %v", width, t)
   130  		return 0
   131  	}
   132  }
   133  
   134  // storeByType returns the store instruction of the given type.
   135  func storeByType(t *types.Type) obj.As {
   136  	width := t.Size()
   137  
   138  	if t.IsFloat() {
   139  		switch width {
   140  		case 4:
   141  			return riscv.AMOVF
   142  		case 8:
   143  			return riscv.AMOVD
   144  		default:
   145  			base.Fatalf("unknown float width for store %d in type %v", width, t)
   146  			return 0
   147  		}
   148  	}
   149  
   150  	switch width {
   151  	case 1:
   152  		return riscv.AMOVB
   153  	case 2:
   154  		return riscv.AMOVH
   155  	case 4:
   156  		return riscv.AMOVW
   157  	case 8:
   158  		return riscv.AMOV
   159  	default:
   160  		base.Fatalf("unknown width for store %d in type %v", width, t)
   161  		return 0
   162  	}
   163  }
   164  
   165  // largestMove returns the largest move instruction possible and its size,
   166  // given the alignment of the total size of the move.
   167  //
   168  // e.g., a 16-byte move may use MOV, but an 11-byte move must use MOVB.
   169  //
   170  // Note that the moves may not be on naturally aligned addresses depending on
   171  // the source and destination.
   172  //
   173  // This matches the calculation in ssa.moveSize.
   174  func largestMove(alignment int64) (obj.As, int64) {
   175  	switch {
   176  	case alignment%8 == 0:
   177  		return riscv.AMOV, 8
   178  	case alignment%4 == 0:
   179  		return riscv.AMOVW, 4
   180  	case alignment%2 == 0:
   181  		return riscv.AMOVH, 2
   182  	default:
   183  		return riscv.AMOVB, 1
   184  	}
   185  }
   186  
   187  var fracMovOps = []obj.As{riscv.AMOVB, riscv.AMOVH, riscv.AMOVW, riscv.AMOV}
   188  
   189  // ssaMarkMoves marks any MOVXconst ops that need to avoid clobbering flags.
   190  // RISC-V has no flags, so this is a no-op.
   191  func ssaMarkMoves(s *ssagen.State, b *ssa.Block) {}
   192  
   193  func ssaGenValue(s *ssagen.State, v *ssa.Value) {
   194  	s.SetPos(v.Pos)
   195  
   196  	switch v.Op {
   197  	case ssaop.OpInitMem:
   198  		// memory arg needs no code
   199  	case ssaop.OpArg:
   200  		// input args need no code
   201  	case ssaop.OpPhi:
   202  		ssagen.CheckLoweredPhi(v)
   203  	case ssaop.OpCopy, ssaop.OpRISCV64MOVDreg:
   204  		if v.Type.IsMemory() {
   205  			return
   206  		}
   207  		rs := v.Args[0].Reg()
   208  		rd := v.Reg()
   209  		if rs == rd {
   210  			return
   211  		}
   212  		as := riscv.AMOV
   213  		if v.Type.IsFloat() {
   214  			as = riscv.AMOVD
   215  		}
   216  		p := s.Prog(as)
   217  		p.From.Type = obj.TYPE_REG
   218  		p.From.Reg = rs
   219  		p.To.Type = obj.TYPE_REG
   220  		p.To.Reg = rd
   221  	case ssaop.OpRISCV64MOVDnop:
   222  		// nothing to do
   223  	case ssaop.OpLoadReg:
   224  		if v.Type.IsFlags() {
   225  			v.Fatalf("load flags not implemented: %v", v.LongString())
   226  			return
   227  		}
   228  		p := s.Prog(loadByType(v.Type))
   229  		ssagen.AddrAuto(&p.From, v.Args[0])
   230  		p.To.Type = obj.TYPE_REG
   231  		p.To.Reg = v.Reg()
   232  	case ssaop.OpStoreReg:
   233  		if v.Type.IsFlags() {
   234  			v.Fatalf("store flags not implemented: %v", v.LongString())
   235  			return
   236  		}
   237  		p := s.Prog(storeByType(v.Type))
   238  		p.From.Type = obj.TYPE_REG
   239  		p.From.Reg = v.Args[0].Reg()
   240  		ssagen.AddrAuto(&p.To, v)
   241  	case ssaop.OpArgIntReg, ssaop.OpArgFloatReg:
   242  		// The assembler needs to wrap the entry safepoint/stack growth code with spill/unspill
   243  		// The loop only runs once.
   244  		for _, a := range v.Block.Func.RegArgs {
   245  			// Pass the spill/unspill information along to the assembler, offset by size of
   246  			// the saved LR slot.
   247  			addr := ssagen.SpillSlotAddr(a, riscv.REG_SP, base.Ctxt.Arch.FixedFrameSize)
   248  			s.FuncInfo().AddSpill(
   249  				obj.RegSpill{Reg: a.Reg, Addr: addr, Unspill: loadByType(a.Type), Spill: storeByType(a.Type)})
   250  		}
   251  		v.Block.Func.RegArgs = nil
   252  
   253  		ssagen.CheckArgReg(v)
   254  	case ssaop.OpSP, ssaop.OpSB, ssaop.OpGetG:
   255  		// nothing to do
   256  	case ssaop.OpRISCV64MOVBreg, ssaop.OpRISCV64MOVHreg, ssaop.OpRISCV64MOVWreg,
   257  		ssaop.OpRISCV64MOVBUreg, ssaop.OpRISCV64MOVHUreg, ssaop.OpRISCV64MOVWUreg:
   258  		a := v.Args[0]
   259  		for a.Op == ssaop.OpCopy || a.Op == ssaop.OpRISCV64MOVDreg {
   260  			a = a.Args[0]
   261  		}
   262  		as := v.Op.Asm()
   263  		rs := v.Args[0].Reg()
   264  		rd := v.Reg()
   265  		if a.Op == ssaop.OpLoadReg {
   266  			t := a.Type
   267  			switch {
   268  			case v.Op == ssaop.OpRISCV64MOVBreg && t.Size() == 1 && t.IsSigned(),
   269  				v.Op == ssaop.OpRISCV64MOVHreg && t.Size() == 2 && t.IsSigned(),
   270  				v.Op == ssaop.OpRISCV64MOVWreg && t.Size() == 4 && t.IsSigned(),
   271  				v.Op == ssaop.OpRISCV64MOVBUreg && t.Size() == 1 && !t.IsSigned(),
   272  				v.Op == ssaop.OpRISCV64MOVHUreg && t.Size() == 2 && !t.IsSigned(),
   273  				v.Op == ssaop.OpRISCV64MOVWUreg && t.Size() == 4 && !t.IsSigned():
   274  				// arg is a proper-typed load and already sign/zero-extended
   275  				if rs == rd {
   276  					return
   277  				}
   278  				as = riscv.AMOV
   279  			default:
   280  			}
   281  		}
   282  		p := s.Prog(as)
   283  		p.From.Type = obj.TYPE_REG
   284  		p.From.Reg = rs
   285  		p.To.Type = obj.TYPE_REG
   286  		p.To.Reg = rd
   287  	case ssaop.OpRISCV64ADD, ssaop.OpRISCV64SUB, ssaop.OpRISCV64SUBW, ssaop.OpRISCV64XNOR, ssaop.OpRISCV64XOR,
   288  		ssaop.OpRISCV64OR, ssaop.OpRISCV64ORN, ssaop.OpRISCV64AND, ssaop.OpRISCV64ANDN,
   289  		ssaop.OpRISCV64SLL, ssaop.OpRISCV64SLLW, ssaop.OpRISCV64SRA, ssaop.OpRISCV64SRAW, ssaop.OpRISCV64SRL, ssaop.OpRISCV64SRLW,
   290  		ssaop.OpRISCV64SLT, ssaop.OpRISCV64SLTU, ssaop.OpRISCV64MUL, ssaop.OpRISCV64MULW, ssaop.OpRISCV64MULH,
   291  		ssaop.OpRISCV64MULHU, ssaop.OpRISCV64DIV, ssaop.OpRISCV64DIVU, ssaop.OpRISCV64DIVW,
   292  		ssaop.OpRISCV64DIVUW, ssaop.OpRISCV64REM, ssaop.OpRISCV64REMU, ssaop.OpRISCV64REMW,
   293  		ssaop.OpRISCV64REMUW,
   294  		ssaop.OpRISCV64ROL, ssaop.OpRISCV64ROLW, ssaop.OpRISCV64ROR, ssaop.OpRISCV64RORW,
   295  		ssaop.OpRISCV64FADDS, ssaop.OpRISCV64FSUBS, ssaop.OpRISCV64FMULS, ssaop.OpRISCV64FDIVS,
   296  		ssaop.OpRISCV64FEQS, ssaop.OpRISCV64FNES, ssaop.OpRISCV64FLTS, ssaop.OpRISCV64FLES,
   297  		ssaop.OpRISCV64FADDD, ssaop.OpRISCV64FSUBD, ssaop.OpRISCV64FMULD, ssaop.OpRISCV64FDIVD,
   298  		ssaop.OpRISCV64FEQD, ssaop.OpRISCV64FNED, ssaop.OpRISCV64FLTD, ssaop.OpRISCV64FLED, ssaop.OpRISCV64FSGNJD,
   299  		ssaop.OpRISCV64MIN, ssaop.OpRISCV64MAX, ssaop.OpRISCV64MINU, ssaop.OpRISCV64MAXU,
   300  		ssaop.OpRISCV64SH1ADD, ssaop.OpRISCV64SH2ADD, ssaop.OpRISCV64SH3ADD,
   301  		ssaop.OpRISCV64CZEROEQZ, ssaop.OpRISCV64CZERONEZ:
   302  		r := v.Reg()
   303  		r1 := v.Args[0].Reg()
   304  		r2 := v.Args[1].Reg()
   305  		p := s.Prog(v.Op.Asm())
   306  		p.From.Type = obj.TYPE_REG
   307  		p.From.Reg = r2
   308  		p.Reg = r1
   309  		p.To.Type = obj.TYPE_REG
   310  		p.To.Reg = r
   311  
   312  	case ssaop.OpRISCV64LoweredFMAXD, ssaop.OpRISCV64LoweredFMIND, ssaop.OpRISCV64LoweredFMAXS, ssaop.OpRISCV64LoweredFMINS:
   313  		// Most of FMIN/FMAX result match Go's required behaviour, unless one of the
   314  		// inputs is a NaN. As such, we need to explicitly test for NaN
   315  		// before using FMIN/FMAX.
   316  
   317  		// FADD Rarg0, Rarg1, Rout // FADD is used to propagate a NaN to the result in these cases.
   318  		// FEQ  Rarg0, Rarg0, Rtmp
   319  		// BEQZ Rtmp, end
   320  		// FEQ  Rarg1, Rarg1, Rtmp
   321  		// BEQZ Rtmp, end
   322  		// F(MIN | MAX)
   323  
   324  		r0 := v.Args[0].Reg()
   325  		r1 := v.Args[1].Reg()
   326  		out := v.Reg()
   327  		add, feq := riscv.AFADDD, riscv.AFEQD
   328  		if v.Op == ssaop.OpRISCV64LoweredFMAXS || v.Op == ssaop.OpRISCV64LoweredFMINS {
   329  			add = riscv.AFADDS
   330  			feq = riscv.AFEQS
   331  		}
   332  
   333  		p1 := s.Prog(add)
   334  		p1.From.Type = obj.TYPE_REG
   335  		p1.From.Reg = r0
   336  		p1.Reg = r1
   337  		p1.To.Type = obj.TYPE_REG
   338  		p1.To.Reg = out
   339  
   340  		p2 := s.Prog(feq)
   341  		p2.From.Type = obj.TYPE_REG
   342  		p2.From.Reg = r0
   343  		p2.Reg = r0
   344  		p2.To.Type = obj.TYPE_REG
   345  		p2.To.Reg = riscv.REG_TMP
   346  
   347  		p3 := s.Prog(riscv.ABEQ)
   348  		p3.From.Type = obj.TYPE_REG
   349  		p3.From.Reg = riscv.REG_ZERO
   350  		p3.Reg = riscv.REG_TMP
   351  		p3.To.Type = obj.TYPE_BRANCH
   352  
   353  		p4 := s.Prog(feq)
   354  		p4.From.Type = obj.TYPE_REG
   355  		p4.From.Reg = r1
   356  		p4.Reg = r1
   357  		p4.To.Type = obj.TYPE_REG
   358  		p4.To.Reg = riscv.REG_TMP
   359  
   360  		p5 := s.Prog(riscv.ABEQ)
   361  		p5.From.Type = obj.TYPE_REG
   362  		p5.From.Reg = riscv.REG_ZERO
   363  		p5.Reg = riscv.REG_TMP
   364  		p5.To.Type = obj.TYPE_BRANCH
   365  
   366  		p6 := s.Prog(v.Op.Asm())
   367  		p6.From.Type = obj.TYPE_REG
   368  		p6.From.Reg = r1
   369  		p6.Reg = r0
   370  		p6.To.Type = obj.TYPE_REG
   371  		p6.To.Reg = out
   372  
   373  		nop := s.Prog(obj.ANOP)
   374  		p3.To.SetTarget(nop)
   375  		p5.To.SetTarget(nop)
   376  
   377  	case ssaop.OpRISCV64LoweredRoundToEvenD, ssaop.OpRISCV64LoweredRoundD, ssaop.OpRISCV64LoweredFloorD, ssaop.OpRISCV64LoweredCeilD, ssaop.OpRISCV64LoweredTruncD:
   378  		rm := riscv.RM_RNE
   379  		switch v.Op {
   380  		case ssaop.OpRISCV64LoweredRoundD:
   381  			rm = riscv.RM_RMM
   382  		case ssaop.OpRISCV64LoweredFloorD:
   383  			rm = riscv.RM_RDN
   384  		case ssaop.OpRISCV64LoweredCeilD:
   385  			rm = riscv.RM_RUP
   386  		case ssaop.OpRISCV64LoweredTruncD:
   387  			rm = riscv.RM_RTZ
   388  		}
   389  		arg := v.Args[0].Reg()
   390  		out := v.Reg()
   391  
   392  		p := s.Prog(riscv.AFCVTLD)
   393  		p.Scond = riscv.RoundingModeSuffix(rm)
   394  		p.From.Type = obj.TYPE_REG
   395  		p.From.Reg = arg
   396  		p.To.Type = obj.TYPE_REG
   397  		p.To.Reg = riscv.REG_TMP
   398  
   399  		p = s.Prog(riscv.AFCVTDL)
   400  		p.From.Type = obj.TYPE_REG
   401  		p.From.Reg = riscv.REG_TMP
   402  		p.To.Type = obj.TYPE_REG
   403  		p.To.Reg = out
   404  
   405  		p = s.Prog(riscv.AFSGNJD)
   406  		p.From.Type = obj.TYPE_REG
   407  		p.From.Reg = arg
   408  		p.Reg = out
   409  		p.To.Type = obj.TYPE_REG
   410  		p.To.Reg = out
   411  
   412  	case ssaop.OpRISCV64LoweredMuluhilo:
   413  		r0 := v.Args[0].Reg()
   414  		r1 := v.Args[1].Reg()
   415  		p := s.Prog(riscv.AMULHU)
   416  		p.From.Type = obj.TYPE_REG
   417  		p.From.Reg = r1
   418  		p.Reg = r0
   419  		p.To.Type = obj.TYPE_REG
   420  		p.To.Reg = v.Reg0()
   421  		p1 := s.Prog(riscv.AMUL)
   422  		p1.From.Type = obj.TYPE_REG
   423  		p1.From.Reg = r1
   424  		p1.Reg = r0
   425  		p1.To.Type = obj.TYPE_REG
   426  		p1.To.Reg = v.Reg1()
   427  	case ssaop.OpRISCV64LoweredMuluover:
   428  		r0 := v.Args[0].Reg()
   429  		r1 := v.Args[1].Reg()
   430  		p := s.Prog(riscv.AMULHU)
   431  		p.From.Type = obj.TYPE_REG
   432  		p.From.Reg = r1
   433  		p.Reg = r0
   434  		p.To.Type = obj.TYPE_REG
   435  		p.To.Reg = v.Reg1()
   436  		p1 := s.Prog(riscv.AMUL)
   437  		p1.From.Type = obj.TYPE_REG
   438  		p1.From.Reg = r1
   439  		p1.Reg = r0
   440  		p1.To.Type = obj.TYPE_REG
   441  		p1.To.Reg = v.Reg0()
   442  		p2 := s.Prog(riscv.ASNEZ)
   443  		p2.From.Type = obj.TYPE_REG
   444  		p2.From.Reg = v.Reg1()
   445  		p2.To.Type = obj.TYPE_REG
   446  		p2.To.Reg = v.Reg1()
   447  	case ssaop.OpRISCV64FMADDD, ssaop.OpRISCV64FMSUBD, ssaop.OpRISCV64FNMADDD, ssaop.OpRISCV64FNMSUBD,
   448  		ssaop.OpRISCV64FMADDS, ssaop.OpRISCV64FMSUBS, ssaop.OpRISCV64FNMADDS, ssaop.OpRISCV64FNMSUBS:
   449  		r := v.Reg()
   450  		r1 := v.Args[0].Reg()
   451  		r2 := v.Args[1].Reg()
   452  		r3 := v.Args[2].Reg()
   453  		p := s.Prog(v.Op.Asm())
   454  		p.From.Type = obj.TYPE_REG
   455  		p.From.Reg = r2
   456  		p.Reg = r1
   457  		p.AddRestSource(obj.Addr{Type: obj.TYPE_REG, Reg: r3})
   458  		p.To.Type = obj.TYPE_REG
   459  		p.To.Reg = r
   460  	case ssaop.OpRISCV64FSQRTS, ssaop.OpRISCV64FSQRTD,
   461  		ssaop.OpRISCV64FNEGS, ssaop.OpRISCV64FNEGD,
   462  		ssaop.OpRISCV64FABSS, ssaop.OpRISCV64FABSD,
   463  		ssaop.OpRISCV64FMVSX, ssaop.OpRISCV64FMVXS, ssaop.OpRISCV64FMVDX, ssaop.OpRISCV64FMVXD,
   464  		ssaop.OpRISCV64FCVTSW, ssaop.OpRISCV64FCVTSL, ssaop.OpRISCV64FCVTWS, ssaop.OpRISCV64FCVTLS,
   465  		ssaop.OpRISCV64FCVTDW, ssaop.OpRISCV64FCVTDL, ssaop.OpRISCV64FCVTWD, ssaop.OpRISCV64FCVTLD, ssaop.OpRISCV64FCVTDS, ssaop.OpRISCV64FCVTSD,
   466  		ssaop.OpRISCV64FCLASSS, ssaop.OpRISCV64FCLASSD,
   467  		ssaop.OpRISCV64NOT, ssaop.OpRISCV64NEG, ssaop.OpRISCV64NEGW, ssaop.OpRISCV64CLZ, ssaop.OpRISCV64CLZW, ssaop.OpRISCV64CTZ, ssaop.OpRISCV64CTZW,
   468  		ssaop.OpRISCV64REV8, ssaop.OpRISCV64CPOP, ssaop.OpRISCV64CPOPW:
   469  		p := s.Prog(v.Op.Asm())
   470  		p.From.Type = obj.TYPE_REG
   471  		p.From.Reg = v.Args[0].Reg()
   472  		p.To.Type = obj.TYPE_REG
   473  		p.To.Reg = v.Reg()
   474  	case ssaop.OpRISCV64ADDI, ssaop.OpRISCV64ADDIW, ssaop.OpRISCV64XORI, ssaop.OpRISCV64ORI, ssaop.OpRISCV64ANDI,
   475  		ssaop.OpRISCV64SLLI, ssaop.OpRISCV64SLLIW, ssaop.OpRISCV64SRAI, ssaop.OpRISCV64SRAIW,
   476  		ssaop.OpRISCV64SRLI, ssaop.OpRISCV64SRLIW, ssaop.OpRISCV64SLTI, ssaop.OpRISCV64SLTIU,
   477  		ssaop.OpRISCV64RORI, ssaop.OpRISCV64RORIW:
   478  		p := s.Prog(v.Op.Asm())
   479  		p.From.Type = obj.TYPE_CONST
   480  		p.From.Offset = v.AuxInt
   481  		p.Reg = v.Args[0].Reg()
   482  		p.To.Type = obj.TYPE_REG
   483  		p.To.Reg = v.Reg()
   484  	case ssaop.OpRISCV64MOVDconst:
   485  		p := s.Prog(v.Op.Asm())
   486  		p.From.Type = obj.TYPE_CONST
   487  		p.From.Offset = v.AuxInt
   488  		p.To.Type = obj.TYPE_REG
   489  		p.To.Reg = v.Reg()
   490  	case ssaop.OpRISCV64FMOVDconst, ssaop.OpRISCV64FMOVFconst:
   491  		p := s.Prog(v.Op.Asm())
   492  		p.From.Type = obj.TYPE_FCONST
   493  		p.From.Val = v.AuxFloat()
   494  		p.From.Name = obj.NAME_NONE
   495  		p.From.Reg = obj.REG_NONE
   496  		p.To.Type = obj.TYPE_REG
   497  		p.To.Reg = v.Reg()
   498  	case ssaop.OpRISCV64MOVaddr:
   499  		p := s.Prog(v.Op.Asm())
   500  		p.From.Type = obj.TYPE_ADDR
   501  		p.To.Type = obj.TYPE_REG
   502  		p.To.Reg = v.Reg()
   503  
   504  		var wantreg string
   505  		// MOVW $sym+off(base), R
   506  		switch v.Aux.(type) {
   507  		default:
   508  			v.Fatalf("aux is of unknown type %T", v.Aux)
   509  		case *obj.LSym:
   510  			wantreg = "SB"
   511  			ssagen.AddAux(&p.From, v)
   512  		case *ir.Name:
   513  			wantreg = "SP"
   514  			ssagen.AddAux(&p.From, v)
   515  		case nil:
   516  			// No sym, just MOVW $off(SP), R
   517  			wantreg = "SP"
   518  			p.From.Reg = riscv.REG_SP
   519  			p.From.Offset = v.AuxInt
   520  		}
   521  		if reg := v.Args[0].RegName(); reg != wantreg {
   522  			v.Fatalf("bad reg %s for symbol type %T, want %s", reg, v.Aux, wantreg)
   523  		}
   524  	case ssaop.OpRISCV64MOVBload, ssaop.OpRISCV64MOVHload, ssaop.OpRISCV64MOVWload, ssaop.OpRISCV64MOVDload,
   525  		ssaop.OpRISCV64MOVBUload, ssaop.OpRISCV64MOVHUload, ssaop.OpRISCV64MOVWUload,
   526  		ssaop.OpRISCV64FMOVWload, ssaop.OpRISCV64FMOVDload:
   527  		p := s.Prog(v.Op.Asm())
   528  		p.From.Type = obj.TYPE_MEM
   529  		p.From.Reg = v.Args[0].Reg()
   530  		ssagen.AddAux(&p.From, v)
   531  		p.To.Type = obj.TYPE_REG
   532  		p.To.Reg = v.Reg()
   533  	case ssaop.OpRISCV64MOVBstore, ssaop.OpRISCV64MOVHstore, ssaop.OpRISCV64MOVWstore, ssaop.OpRISCV64MOVDstore,
   534  		ssaop.OpRISCV64FMOVWstore, ssaop.OpRISCV64FMOVDstore:
   535  		p := s.Prog(v.Op.Asm())
   536  		p.From.Type = obj.TYPE_REG
   537  		p.From.Reg = v.Args[1].Reg()
   538  		p.To.Type = obj.TYPE_MEM
   539  		p.To.Reg = v.Args[0].Reg()
   540  		ssagen.AddAux(&p.To, v)
   541  	case ssaop.OpRISCV64MOVBstorezero, ssaop.OpRISCV64MOVHstorezero, ssaop.OpRISCV64MOVWstorezero, ssaop.OpRISCV64MOVDstorezero:
   542  		p := s.Prog(v.Op.Asm())
   543  		p.From.Type = obj.TYPE_REG
   544  		p.From.Reg = riscv.REG_ZERO
   545  		p.To.Type = obj.TYPE_MEM
   546  		p.To.Reg = v.Args[0].Reg()
   547  		ssagen.AddAux(&p.To, v)
   548  	case ssaop.OpRISCV64SEQZ, ssaop.OpRISCV64SNEZ:
   549  		p := s.Prog(v.Op.Asm())
   550  		p.From.Type = obj.TYPE_REG
   551  		p.From.Reg = v.Args[0].Reg()
   552  		p.To.Type = obj.TYPE_REG
   553  		p.To.Reg = v.Reg()
   554  	case ssaop.OpRISCV64CALLstatic, ssaop.OpRISCV64CALLclosure, ssaop.OpRISCV64CALLinter:
   555  		s.Call(v)
   556  	case ssaop.OpRISCV64CALLtail, ssaop.OpRISCV64CALLtailinter:
   557  		s.TailCall(v)
   558  	case ssaop.OpRISCV64LoweredWB:
   559  		p := s.Prog(obj.ACALL)
   560  		p.To.Type = obj.TYPE_MEM
   561  		p.To.Name = obj.NAME_EXTERN
   562  		// AuxInt encodes how many buffer entries we need.
   563  		p.To.Sym = ir.Syms.GCWriteBarrier[v.AuxInt-1]
   564  
   565  	case ssaop.OpRISCV64LoweredPanicBoundsRR, ssaop.OpRISCV64LoweredPanicBoundsRC, ssaop.OpRISCV64LoweredPanicBoundsCR, ssaop.OpRISCV64LoweredPanicBoundsCC:
   566  		// Compute the constant we put in the PCData entry for this call.
   567  		code, signed := ssa.BoundsKind(v.AuxInt).Code()
   568  		xIsReg := false
   569  		yIsReg := false
   570  		xVal := 0
   571  		yVal := 0
   572  		switch v.Op {
   573  		case ssaop.OpRISCV64LoweredPanicBoundsRR:
   574  			xIsReg = true
   575  			xVal = int(v.Args[0].Reg() - riscv.REG_X5)
   576  			yIsReg = true
   577  			yVal = int(v.Args[1].Reg() - riscv.REG_X5)
   578  		case ssaop.OpRISCV64LoweredPanicBoundsRC:
   579  			xIsReg = true
   580  			xVal = int(v.Args[0].Reg() - riscv.REG_X5)
   581  			c := v.Aux.(ssa.PanicBoundsC).C
   582  			if c >= 0 && c <= abi.BoundsMaxConst {
   583  				yVal = int(c)
   584  			} else {
   585  				// Move constant to a register
   586  				yIsReg = true
   587  				if yVal == xVal {
   588  					yVal = 1
   589  				}
   590  				p := s.Prog(riscv.AMOV)
   591  				p.From.Type = obj.TYPE_CONST
   592  				p.From.Offset = c
   593  				p.To.Type = obj.TYPE_REG
   594  				p.To.Reg = riscv.REG_X5 + int16(yVal)
   595  			}
   596  		case ssaop.OpRISCV64LoweredPanicBoundsCR:
   597  			yIsReg = true
   598  			yVal = int(v.Args[0].Reg() - riscv.REG_X5)
   599  			c := v.Aux.(ssa.PanicBoundsC).C
   600  			if c >= 0 && c <= abi.BoundsMaxConst {
   601  				xVal = int(c)
   602  			} else {
   603  				// Move constant to a register
   604  				if xVal == yVal {
   605  					xVal = 1
   606  				}
   607  				p := s.Prog(riscv.AMOV)
   608  				p.From.Type = obj.TYPE_CONST
   609  				p.From.Offset = c
   610  				p.To.Type = obj.TYPE_REG
   611  				p.To.Reg = riscv.REG_X5 + int16(xVal)
   612  			}
   613  		case ssaop.OpRISCV64LoweredPanicBoundsCC:
   614  			c := v.Aux.(ssa.PanicBoundsCC).Cx
   615  			if c >= 0 && c <= abi.BoundsMaxConst {
   616  				xVal = int(c)
   617  			} else {
   618  				// Move constant to a register
   619  				xIsReg = true
   620  				p := s.Prog(riscv.AMOV)
   621  				p.From.Type = obj.TYPE_CONST
   622  				p.From.Offset = c
   623  				p.To.Type = obj.TYPE_REG
   624  				p.To.Reg = riscv.REG_X5 + int16(xVal)
   625  			}
   626  			c = v.Aux.(ssa.PanicBoundsCC).Cy
   627  			if c >= 0 && c <= abi.BoundsMaxConst {
   628  				yVal = int(c)
   629  			} else {
   630  				// Move constant to a register
   631  				yIsReg = true
   632  				yVal = 1
   633  				p := s.Prog(riscv.AMOV)
   634  				p.From.Type = obj.TYPE_CONST
   635  				p.From.Offset = c
   636  				p.To.Type = obj.TYPE_REG
   637  				p.To.Reg = riscv.REG_X5 + int16(yVal)
   638  			}
   639  		}
   640  		c := abi.BoundsEncode(code, signed, xIsReg, yIsReg, xVal, yVal)
   641  
   642  		p := s.Prog(obj.APCDATA)
   643  		p.From.SetConst(abi.PCDATA_PanicBounds)
   644  		p.To.SetConst(int64(c))
   645  		p = s.Prog(obj.ACALL)
   646  		p.To.Type = obj.TYPE_MEM
   647  		p.To.Name = obj.NAME_EXTERN
   648  		p.To.Sym = ir.Syms.PanicBounds
   649  
   650  	case ssaop.OpRISCV64LoweredAtomicLoad8:
   651  		p1 := s.Prog(riscv.AFENCE)
   652  		p1.From.Type = obj.TYPE_SPECIAL
   653  		p1.From.Offset = int64(riscv.SPOP_FENCE_RW)
   654  		p1.To.Type = obj.TYPE_SPECIAL
   655  		p1.To.Offset = int64(riscv.SPOP_FENCE_RW)
   656  
   657  		p := s.Prog(riscv.AMOVBU)
   658  		p.From.Type = obj.TYPE_MEM
   659  		p.From.Reg = v.Args[0].Reg()
   660  		p.To.Type = obj.TYPE_REG
   661  		p.To.Reg = v.Reg0()
   662  
   663  		p2 := s.Prog(riscv.AFENCE)
   664  		p2.From.Type = obj.TYPE_SPECIAL
   665  		p2.From.Offset = int64(riscv.SPOP_FENCE_R)
   666  		p2.To.Type = obj.TYPE_SPECIAL
   667  		p2.To.Offset = int64(riscv.SPOP_FENCE_RW)
   668  
   669  	case ssaop.OpRISCV64LoweredAtomicLoad32, ssaop.OpRISCV64LoweredAtomicLoad64:
   670  		as := riscv.ALRW
   671  		if v.Op == ssaop.OpRISCV64LoweredAtomicLoad64 {
   672  			as = riscv.ALRD
   673  		}
   674  		p := s.Prog(as)
   675  		p.From.Type = obj.TYPE_MEM
   676  		p.From.Reg = v.Args[0].Reg()
   677  		p.To.Type = obj.TYPE_REG
   678  		p.To.Reg = v.Reg0()
   679  
   680  	case ssaop.OpRISCV64LoweredAtomicStore8:
   681  		p1 := s.Prog(riscv.AFENCE)
   682  		p1.From.Type = obj.TYPE_SPECIAL
   683  		p1.From.Offset = int64(riscv.SPOP_FENCE_RW)
   684  		p1.To.Type = obj.TYPE_SPECIAL
   685  		p1.To.Offset = int64(riscv.SPOP_FENCE_W)
   686  
   687  		p := s.Prog(riscv.AMOVB)
   688  		p.From.Type = obj.TYPE_REG
   689  		p.From.Reg = v.Args[1].Reg()
   690  		p.To.Type = obj.TYPE_MEM
   691  		p.To.Reg = v.Args[0].Reg()
   692  
   693  		p2 := s.Prog(riscv.AFENCE)
   694  		p2.From.Type = obj.TYPE_SPECIAL
   695  		p2.From.Offset = int64(riscv.SPOP_FENCE_RW)
   696  		p2.To.Type = obj.TYPE_SPECIAL
   697  		p2.To.Offset = int64(riscv.SPOP_FENCE_RW)
   698  
   699  	case ssaop.OpRISCV64LoweredAtomicStore32, ssaop.OpRISCV64LoweredAtomicStore64:
   700  		as := riscv.AAMOSWAPW
   701  		if v.Op == ssaop.OpRISCV64LoweredAtomicStore64 {
   702  			as = riscv.AAMOSWAPD
   703  		}
   704  		p := s.Prog(as)
   705  		p.From.Type = obj.TYPE_REG
   706  		p.From.Reg = v.Args[1].Reg()
   707  		p.To.Type = obj.TYPE_MEM
   708  		p.To.Reg = v.Args[0].Reg()
   709  		p.RegTo2 = riscv.REG_ZERO
   710  
   711  	case ssaop.OpRISCV64LoweredAtomicAdd32, ssaop.OpRISCV64LoweredAtomicAdd64:
   712  		as := riscv.AAMOADDW
   713  		if v.Op == ssaop.OpRISCV64LoweredAtomicAdd64 {
   714  			as = riscv.AAMOADDD
   715  		}
   716  		p := s.Prog(as)
   717  		p.From.Type = obj.TYPE_REG
   718  		p.From.Reg = v.Args[1].Reg()
   719  		p.To.Type = obj.TYPE_MEM
   720  		p.To.Reg = v.Args[0].Reg()
   721  		p.RegTo2 = riscv.REG_TMP
   722  
   723  		p2 := s.Prog(riscv.AADD)
   724  		p2.From.Type = obj.TYPE_REG
   725  		p2.From.Reg = riscv.REG_TMP
   726  		p2.Reg = v.Args[1].Reg()
   727  		p2.To.Type = obj.TYPE_REG
   728  		p2.To.Reg = v.Reg0()
   729  
   730  	case ssaop.OpRISCV64LoweredAtomicExchange32, ssaop.OpRISCV64LoweredAtomicExchange64:
   731  		as := riscv.AAMOSWAPW
   732  		if v.Op == ssaop.OpRISCV64LoweredAtomicExchange64 {
   733  			as = riscv.AAMOSWAPD
   734  		}
   735  		p := s.Prog(as)
   736  		p.From.Type = obj.TYPE_REG
   737  		p.From.Reg = v.Args[1].Reg()
   738  		p.To.Type = obj.TYPE_MEM
   739  		p.To.Reg = v.Args[0].Reg()
   740  		p.RegTo2 = v.Reg0()
   741  
   742  	case ssaop.OpRISCV64LoweredAtomicCas32, ssaop.OpRISCV64LoweredAtomicCas64:
   743  		// MOV  ZERO, Rout
   744  		// LR	(Rarg0), Rtmp
   745  		// BNE	Rtmp, Rarg1, 3(PC)
   746  		// SC	Rarg2, (Rarg0), Rtmp
   747  		// BNE	Rtmp, ZERO, -3(PC)
   748  		// MOV	$1, Rout
   749  
   750  		lr := riscv.ALRW
   751  		sc := riscv.ASCW
   752  		if v.Op == ssaop.OpRISCV64LoweredAtomicCas64 {
   753  			lr = riscv.ALRD
   754  			sc = riscv.ASCD
   755  		}
   756  
   757  		r0 := v.Args[0].Reg()
   758  		r1 := v.Args[1].Reg()
   759  		r2 := v.Args[2].Reg()
   760  		out := v.Reg0()
   761  
   762  		p := s.Prog(riscv.AMOV)
   763  		p.From.Type = obj.TYPE_REG
   764  		p.From.Reg = riscv.REG_ZERO
   765  		p.To.Type = obj.TYPE_REG
   766  		p.To.Reg = out
   767  
   768  		p1 := s.Prog(lr)
   769  		p1.From.Type = obj.TYPE_MEM
   770  		p1.From.Reg = r0
   771  		p1.To.Type = obj.TYPE_REG
   772  		p1.To.Reg = riscv.REG_TMP
   773  
   774  		p2 := s.Prog(riscv.ABNE)
   775  		p2.From.Type = obj.TYPE_REG
   776  		p2.From.Reg = r1
   777  		p2.Reg = riscv.REG_TMP
   778  		p2.To.Type = obj.TYPE_BRANCH
   779  
   780  		p3 := s.Prog(sc)
   781  		p3.From.Type = obj.TYPE_REG
   782  		p3.From.Reg = r2
   783  		p3.To.Type = obj.TYPE_MEM
   784  		p3.To.Reg = r0
   785  		p3.RegTo2 = riscv.REG_TMP
   786  
   787  		p4 := s.Prog(riscv.ABNE)
   788  		p4.From.Type = obj.TYPE_REG
   789  		p4.From.Reg = riscv.REG_TMP
   790  		p4.Reg = riscv.REG_ZERO
   791  		p4.To.Type = obj.TYPE_BRANCH
   792  		p4.To.SetTarget(p1)
   793  
   794  		p5 := s.Prog(riscv.AMOV)
   795  		p5.From.Type = obj.TYPE_CONST
   796  		p5.From.Offset = 1
   797  		p5.To.Type = obj.TYPE_REG
   798  		p5.To.Reg = out
   799  
   800  		p6 := s.Prog(obj.ANOP)
   801  		p2.To.SetTarget(p6)
   802  
   803  	case ssaop.OpRISCV64LoweredAtomicAnd32, ssaop.OpRISCV64LoweredAtomicOr32:
   804  		p := s.Prog(v.Op.Asm())
   805  		p.From.Type = obj.TYPE_REG
   806  		p.From.Reg = v.Args[1].Reg()
   807  		p.To.Type = obj.TYPE_MEM
   808  		p.To.Reg = v.Args[0].Reg()
   809  		p.RegTo2 = riscv.REG_ZERO
   810  
   811  	case ssaop.OpRISCV64LoweredAtomicAnd32value, ssaop.OpRISCV64LoweredAtomicAnd64value,
   812  		ssaop.OpRISCV64LoweredAtomicOr32value, ssaop.OpRISCV64LoweredAtomicOr64value:
   813  		p := s.Prog(v.Op.Asm())
   814  		p.From.Type = obj.TYPE_REG
   815  		p.From.Reg = v.Args[1].Reg()
   816  		p.To.Type = obj.TYPE_MEM
   817  		p.To.Reg = v.Args[0].Reg()
   818  		p.RegTo2 = v.Reg0()
   819  
   820  	case ssaop.OpRISCV64LoweredZero:
   821  		ptr := v.Args[0].Reg()
   822  		n, align := v.AuxSizeAndAlign()
   823  		mov, sz := largestMove(align)
   824  
   825  		// mov	ZERO, (offset)(Rarg0)
   826  		var off int64
   827  		for n >= sz {
   828  			zeroOp(s, mov, ptr, off)
   829  			off += sz
   830  			n -= sz
   831  		}
   832  
   833  		for i := len(fracMovOps) - 1; i >= 0; i-- {
   834  			tsz := int64(1 << i)
   835  			if n < tsz {
   836  				continue
   837  			}
   838  			zeroOp(s, fracMovOps[i], ptr, off)
   839  			off += tsz
   840  			n -= tsz
   841  		}
   842  
   843  	case ssaop.OpRISCV64LoweredZeroLoop:
   844  		ptr := v.Args[0].Reg()
   845  		n, align := v.AuxSizeAndAlign()
   846  		mov, sz := largestMove(align)
   847  		chunk := 8 * sz
   848  
   849  		if n <= 3*chunk {
   850  			v.Fatalf("ZeroLoop too small:%d, expect:%d", n, 3*chunk)
   851  		}
   852  
   853  		tmp := v.RegTmp()
   854  
   855  		if n >= 1<<31 {
   856  			p := s.Prog(riscv.AMOV)
   857  			p.From.Type = obj.TYPE_CONST
   858  			p.From.Offset = n - n%chunk
   859  			p.To.Type = obj.TYPE_REG
   860  			p.To.Reg = tmp
   861  		}
   862  		p := s.Prog(riscv.AADD)
   863  		if n >= 1<<31 {
   864  			p.From.Type = obj.TYPE_REG
   865  			p.From.Reg = tmp
   866  		} else {
   867  			p.From.Type = obj.TYPE_CONST
   868  			p.From.Offset = n - n%chunk
   869  		}
   870  		p.Reg = ptr
   871  		p.To.Type = obj.TYPE_REG
   872  		p.To.Reg = tmp
   873  
   874  		for i := int64(0); i < 8; i++ {
   875  			zeroOp(s, mov, ptr, sz*i)
   876  		}
   877  
   878  		p2 := s.Prog(riscv.AADD)
   879  		p2.From.Type = obj.TYPE_CONST
   880  		p2.From.Offset = chunk
   881  		p2.To.Type = obj.TYPE_REG
   882  		p2.To.Reg = ptr
   883  
   884  		p3 := s.Prog(riscv.ABNE)
   885  		p3.From.Reg = tmp
   886  		p3.From.Type = obj.TYPE_REG
   887  		p3.Reg = ptr
   888  		p3.To.Type = obj.TYPE_BRANCH
   889  		p3.To.SetTarget(p.Link)
   890  
   891  		n %= chunk
   892  
   893  		// mov	ZERO, (offset)(Rarg0)
   894  		var off int64
   895  		for n >= sz {
   896  			zeroOp(s, mov, ptr, off)
   897  			off += sz
   898  			n -= sz
   899  		}
   900  
   901  		for i := len(fracMovOps) - 1; i >= 0; i-- {
   902  			tsz := int64(1 << i)
   903  			if n < tsz {
   904  				continue
   905  			}
   906  			zeroOp(s, fracMovOps[i], ptr, off)
   907  			off += tsz
   908  			n -= tsz
   909  		}
   910  
   911  	case ssaop.OpRISCV64LoweredMove:
   912  		dst := v.Args[0].Reg()
   913  		src := v.Args[1].Reg()
   914  		if dst == src {
   915  			break
   916  		}
   917  
   918  		n, align := v.AuxSizeAndAlign()
   919  		mov, sz := largestMove(align)
   920  
   921  		var off int64
   922  		tmp := int16(riscv.REG_X5)
   923  		for n >= sz {
   924  			moveOp(s, mov, dst, src, tmp, off)
   925  			off += sz
   926  			n -= sz
   927  		}
   928  
   929  		for i := len(fracMovOps) - 1; i >= 0; i-- {
   930  			tsz := int64(1 << i)
   931  			if n < tsz {
   932  				continue
   933  			}
   934  			moveOp(s, fracMovOps[i], dst, src, tmp, off)
   935  			off += tsz
   936  			n -= tsz
   937  		}
   938  
   939  	case ssaop.OpRISCV64LoweredMoveLoop:
   940  		dst := v.Args[0].Reg()
   941  		src := v.Args[1].Reg()
   942  		if dst == src {
   943  			break
   944  		}
   945  
   946  		n, align := v.AuxSizeAndAlign()
   947  		mov, sz := largestMove(align)
   948  		chunk := 8 * sz
   949  
   950  		if n <= 3*chunk {
   951  			v.Fatalf("MoveLoop too small:%d, expect:%d", n, 3*chunk)
   952  		}
   953  		tmp := int16(riscv.REG_X5)
   954  
   955  		if n >= 1<<31 {
   956  			p := s.Prog(riscv.AMOV)
   957  			p.From.Type = obj.TYPE_CONST
   958  			p.From.Offset = n - n%chunk
   959  			p.To.Type = obj.TYPE_REG
   960  			p.To.Reg = riscv.REG_X6
   961  		}
   962  		p := s.Prog(riscv.AADD)
   963  		if n >= 1<<31 {
   964  			p.From.Type = obj.TYPE_REG
   965  			p.From.Reg = riscv.REG_X6
   966  		} else {
   967  			p.From.Type = obj.TYPE_CONST
   968  			p.From.Offset = n - n%chunk
   969  		}
   970  		p.Reg = src
   971  		p.To.Type = obj.TYPE_REG
   972  		p.To.Reg = riscv.REG_X6
   973  
   974  		for i := int64(0); i < 8; i++ {
   975  			moveOp(s, mov, dst, src, tmp, sz*i)
   976  		}
   977  
   978  		p1 := s.Prog(riscv.AADD)
   979  		p1.From.Type = obj.TYPE_CONST
   980  		p1.From.Offset = chunk
   981  		p1.To.Type = obj.TYPE_REG
   982  		p1.To.Reg = src
   983  
   984  		p2 := s.Prog(riscv.AADD)
   985  		p2.From.Type = obj.TYPE_CONST
   986  		p2.From.Offset = chunk
   987  		p2.To.Type = obj.TYPE_REG
   988  		p2.To.Reg = dst
   989  
   990  		p3 := s.Prog(riscv.ABNE)
   991  		p3.From.Reg = riscv.REG_X6
   992  		p3.From.Type = obj.TYPE_REG
   993  		p3.Reg = src
   994  		p3.To.Type = obj.TYPE_BRANCH
   995  		p3.To.SetTarget(p.Link)
   996  
   997  		n %= chunk
   998  
   999  		var off int64
  1000  		for n >= sz {
  1001  			moveOp(s, mov, dst, src, tmp, off)
  1002  			off += sz
  1003  			n -= sz
  1004  		}
  1005  
  1006  		for i := len(fracMovOps) - 1; i >= 0; i-- {
  1007  			tsz := int64(1 << i)
  1008  			if n < tsz {
  1009  				continue
  1010  			}
  1011  			moveOp(s, fracMovOps[i], dst, src, tmp, off)
  1012  			off += tsz
  1013  			n -= tsz
  1014  		}
  1015  
  1016  	case ssaop.OpRISCV64LoweredNilCheck:
  1017  		// Issue a load which will fault if arg is nil.
  1018  		p := s.Prog(riscv.AMOVB)
  1019  		p.From.Type = obj.TYPE_MEM
  1020  		p.From.Reg = v.Args[0].Reg()
  1021  		ssagen.AddAux(&p.From, v)
  1022  		p.To.Type = obj.TYPE_REG
  1023  		p.To.Reg = riscv.REG_ZERO
  1024  		if logopt.Enabled() {
  1025  			logopt.LogOpt(v.Pos, "nilcheck", "genssa", v.Block.Func.Name)
  1026  		}
  1027  		if base.Debug.Nil != 0 && v.Pos.Line() > 1 { // v.Pos == 1 in generated wrappers
  1028  			base.WarnfAt(v.Pos, "generated nil check")
  1029  		}
  1030  
  1031  	case ssaop.OpRISCV64LoweredGetClosurePtr:
  1032  		// Closure pointer is S10 (riscv.REG_CTXT).
  1033  		ssagen.CheckLoweredGetClosurePtr(v)
  1034  
  1035  	case ssaop.OpRISCV64LoweredGetCallerSP:
  1036  		// caller's SP is FixedFrameSize below the address of the first arg
  1037  		p := s.Prog(riscv.AMOV)
  1038  		p.From.Type = obj.TYPE_ADDR
  1039  		p.From.Offset = -base.Ctxt.Arch.FixedFrameSize
  1040  		p.From.Name = obj.NAME_PARAM
  1041  		p.To.Type = obj.TYPE_REG
  1042  		p.To.Reg = v.Reg()
  1043  
  1044  	case ssaop.OpRISCV64LoweredGetCallerPC:
  1045  		p := s.Prog(obj.AGETCALLERPC)
  1046  		p.To.Type = obj.TYPE_REG
  1047  		p.To.Reg = v.Reg()
  1048  
  1049  	case ssaop.OpRISCV64LoweredPubBarrier:
  1050  		// FENCE W, W
  1051  		p := s.Prog(v.Op.Asm())
  1052  		p.From.Type = obj.TYPE_SPECIAL
  1053  		p.From.Offset = int64(riscv.SPOP_FENCE_W)
  1054  		p.To.Type = obj.TYPE_SPECIAL
  1055  		p.To.Offset = int64(riscv.SPOP_FENCE_W)
  1056  
  1057  	case ssaop.OpRISCV64LoweredRound32F, ssaop.OpRISCV64LoweredRound64F:
  1058  		// input is already rounded
  1059  
  1060  	case ssaop.OpClobber, ssaop.OpClobberReg:
  1061  		// TODO: implement for clobberdead experiment. Nop is ok for now.
  1062  
  1063  	default:
  1064  		v.Fatalf("Unhandled op %v", v.Op)
  1065  	}
  1066  }
  1067  
  1068  var blockBranch = [...]obj.As{
  1069  	block.BlockRISCV64BEQ:  riscv.ABEQ,
  1070  	block.BlockRISCV64BEQZ: riscv.ABEQZ,
  1071  	block.BlockRISCV64BGE:  riscv.ABGE,
  1072  	block.BlockRISCV64BGEU: riscv.ABGEU,
  1073  	block.BlockRISCV64BGEZ: riscv.ABGEZ,
  1074  	block.BlockRISCV64BGTZ: riscv.ABGTZ,
  1075  	block.BlockRISCV64BLEZ: riscv.ABLEZ,
  1076  	block.BlockRISCV64BLT:  riscv.ABLT,
  1077  	block.BlockRISCV64BLTU: riscv.ABLTU,
  1078  	block.BlockRISCV64BLTZ: riscv.ABLTZ,
  1079  	block.BlockRISCV64BNE:  riscv.ABNE,
  1080  	block.BlockRISCV64BNEZ: riscv.ABNEZ,
  1081  }
  1082  
  1083  func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
  1084  	s.SetPos(b.Pos)
  1085  
  1086  	switch b.Kind {
  1087  	case block.BlockPlain, block.BlockDefer:
  1088  		if b.Succs[0].Block() != next {
  1089  			p := s.Prog(obj.AJMP)
  1090  			p.To.Type = obj.TYPE_BRANCH
  1091  			s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
  1092  		}
  1093  	case block.BlockExit, block.BlockRetJmp:
  1094  	case block.BlockRet:
  1095  		s.Prog(obj.ARET)
  1096  	case block.BlockRISCV64BEQ, block.BlockRISCV64BEQZ, block.BlockRISCV64BNE, block.BlockRISCV64BNEZ,
  1097  		block.BlockRISCV64BLT, block.BlockRISCV64BLEZ, block.BlockRISCV64BGE, block.BlockRISCV64BGEZ,
  1098  		block.BlockRISCV64BLTZ, block.BlockRISCV64BGTZ, block.BlockRISCV64BLTU, block.BlockRISCV64BGEU:
  1099  
  1100  		as := blockBranch[b.Kind]
  1101  		invAs := riscv.InvertBranch(as)
  1102  
  1103  		var p *obj.Prog
  1104  		switch next {
  1105  		case b.Succs[0].Block():
  1106  			p = s.Br(invAs, b.Succs[1].Block())
  1107  		case b.Succs[1].Block():
  1108  			p = s.Br(as, b.Succs[0].Block())
  1109  		default:
  1110  			if b.Likely != ssa.BranchUnlikely {
  1111  				p = s.Br(as, b.Succs[0].Block())
  1112  				s.Br(obj.AJMP, b.Succs[1].Block())
  1113  			} else {
  1114  				p = s.Br(invAs, b.Succs[1].Block())
  1115  				s.Br(obj.AJMP, b.Succs[0].Block())
  1116  			}
  1117  		}
  1118  
  1119  		p.From.Type = obj.TYPE_REG
  1120  		switch b.Kind {
  1121  		case block.BlockRISCV64BEQ, block.BlockRISCV64BNE, block.BlockRISCV64BLT, block.BlockRISCV64BGE, block.BlockRISCV64BLTU, block.BlockRISCV64BGEU:
  1122  			if b.NumControls() != 2 {
  1123  				b.Fatalf("Unexpected number of controls (%d != 2): %s", b.NumControls(), b.LongString())
  1124  			}
  1125  			p.From.Reg = b.Controls[0].Reg()
  1126  			p.Reg = b.Controls[1].Reg()
  1127  
  1128  		case block.BlockRISCV64BEQZ, block.BlockRISCV64BNEZ, block.BlockRISCV64BGEZ, block.BlockRISCV64BLEZ, block.BlockRISCV64BLTZ, block.BlockRISCV64BGTZ:
  1129  			if b.NumControls() != 1 {
  1130  				b.Fatalf("Unexpected number of controls (%d != 1): %s", b.NumControls(), b.LongString())
  1131  			}
  1132  			p.From.Reg = b.Controls[0].Reg()
  1133  		}
  1134  
  1135  	case block.BlockRISCV64JUMPTABLE:
  1136  		// Jump table:
  1137  		// TMP = base + index*8 (SH3ADD if Zba else SLLI+ADD).
  1138  		// Load slot into TMP, then indirect JMP through TMP.
  1139  		var p *obj.Prog
  1140  		if buildcfg.GORISCV64 >= 22 {
  1141  			p = s.Prog(riscv.ASH3ADD)
  1142  			p.From.Type = obj.TYPE_REG
  1143  			p.From.Reg = b.Controls[1].Reg()
  1144  			p.Reg = b.Controls[0].Reg()
  1145  			p.To.Type = obj.TYPE_REG
  1146  			p.To.Reg = riscv.REG_TMP
  1147  		} else {
  1148  			p = s.Prog(riscv.ASLLI)
  1149  			p.From.Type = obj.TYPE_CONST
  1150  			p.From.Offset = 3
  1151  			p.Reg = b.Controls[0].Reg()
  1152  			p.To.Type = obj.TYPE_REG
  1153  			p.To.Reg = riscv.REG_TMP
  1154  
  1155  			p = s.Prog(riscv.AADD)
  1156  			p.From.Type = obj.TYPE_REG
  1157  			p.From.Reg = riscv.REG_TMP
  1158  			p.Reg = b.Controls[1].Reg()
  1159  			p.To.Type = obj.TYPE_REG
  1160  			p.To.Reg = riscv.REG_TMP
  1161  		}
  1162  
  1163  		p = s.Prog(riscv.AMOV)
  1164  		p.From.Type = obj.TYPE_MEM
  1165  		p.From.Reg = riscv.REG_TMP
  1166  		p.To.Type = obj.TYPE_REG
  1167  		p.To.Reg = riscv.REG_TMP
  1168  
  1169  		p = s.Prog(obj.AJMP)
  1170  		p.To.Type = obj.TYPE_MEM
  1171  		p.To.Reg = riscv.REG_TMP
  1172  		// Save jump tables for later resolution of the target blocks.
  1173  		s.JumpTables = append(s.JumpTables, b)
  1174  
  1175  	default:
  1176  		b.Fatalf("Unhandled block: %s", b.LongString())
  1177  	}
  1178  }
  1179  
  1180  func loadRegResult(s *ssagen.State, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  1181  	p := s.Prog(loadByType(t))
  1182  	p.From.Type = obj.TYPE_MEM
  1183  	p.From.Name = obj.NAME_AUTO
  1184  	p.From.Sym = n.Linksym()
  1185  	p.From.Offset = n.FrameOffset() + off
  1186  	p.To.Type = obj.TYPE_REG
  1187  	p.To.Reg = reg
  1188  	return p
  1189  }
  1190  
  1191  func spillArgReg(pp *objw.Progs, p *obj.Prog, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  1192  	p = pp.Append(p, storeByType(t), obj.TYPE_REG, reg, 0, obj.TYPE_MEM, 0, n.FrameOffset()+off)
  1193  	p.To.Name = obj.NAME_PARAM
  1194  	p.To.Sym = n.Linksym()
  1195  	p.Pos = p.Pos.WithNotStmt()
  1196  	return p
  1197  }
  1198  
  1199  func zeroOp(s *ssagen.State, mov obj.As, reg int16, off int64) {
  1200  	p := s.Prog(mov)
  1201  	p.From.Type = obj.TYPE_REG
  1202  	p.From.Reg = riscv.REG_ZERO
  1203  	p.To.Type = obj.TYPE_MEM
  1204  	p.To.Reg = reg
  1205  	p.To.Offset = off
  1206  	return
  1207  }
  1208  
  1209  func moveOp(s *ssagen.State, mov obj.As, dst int16, src int16, tmp int16, off int64) {
  1210  	p := s.Prog(mov)
  1211  	p.From.Type = obj.TYPE_MEM
  1212  	p.From.Reg = src
  1213  	p.From.Offset = off
  1214  	p.To.Type = obj.TYPE_REG
  1215  	p.To.Reg = tmp
  1216  
  1217  	p1 := s.Prog(mov)
  1218  	p1.From.Type = obj.TYPE_REG
  1219  	p1.From.Reg = tmp
  1220  	p1.To.Type = obj.TYPE_MEM
  1221  	p1.To.Reg = dst
  1222  	p1.To.Offset = off
  1223  
  1224  	return
  1225  }
  1226  

View as plain text