Source file src/cmd/compile/internal/amd64/ssa.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package amd64
     6  
     7  import (
     8  	"fmt"
     9  	"math"
    10  
    11  	"cmd/compile/internal/base"
    12  	"cmd/compile/internal/ir"
    13  	"cmd/compile/internal/logopt"
    14  	"cmd/compile/internal/objw"
    15  	"cmd/compile/internal/ssa"
    16  	"cmd/compile/internal/ssa/block"
    17  	"cmd/compile/internal/ssa/ssaop"
    18  	"cmd/compile/internal/ssagen"
    19  	"cmd/compile/internal/types"
    20  	"cmd/internal/obj"
    21  	"cmd/internal/obj/x86"
    22  	"internal/abi"
    23  	"internal/buildcfg"
    24  )
    25  
    26  // ssaMarkMoves marks any MOVXconst ops that need to avoid clobbering flags.
    27  func ssaMarkMoves(s *ssagen.State, b *ssa.Block) {
    28  	flive := b.FlagsLiveAtEnd
    29  	for _, c := range b.ControlValues() {
    30  		flive = c.Type.IsFlags() || flive
    31  	}
    32  	for i := len(b.Values) - 1; i >= 0; i-- {
    33  		v := b.Values[i]
    34  		if flive && (v.Op == ssaop.OpAMD64MOVLconst || v.Op == ssaop.OpAMD64MOVQconst) {
    35  			// The "mark" is any non-nil Aux value.
    36  			v.Aux = ssa.AuxMark
    37  		}
    38  		if v.Type.IsFlags() {
    39  			flive = false
    40  		}
    41  		for _, a := range v.Args {
    42  			if a.Type.IsFlags() {
    43  				flive = true
    44  			}
    45  		}
    46  	}
    47  }
    48  
    49  func isGPReg(r int16) bool {
    50  	return x86.REG_AL <= r && r <= x86.REG_R15
    51  }
    52  
    53  func isFPReg(r int16) bool {
    54  	return x86.REG_X0 <= r && r <= x86.REG_Z31
    55  }
    56  
    57  func isKReg(r int16) bool {
    58  	return x86.REG_K0 <= r && r <= x86.REG_K7
    59  }
    60  
    61  func isLowFPReg(r int16) bool {
    62  	return x86.REG_X0 <= r && r <= x86.REG_X15
    63  }
    64  
    65  func isHighFPReg(r int16) bool {
    66  	return x86.REG_X16 <= r && r <= x86.REG_X31 || x86.REG_Y16 <= r && r <= x86.REG_Y31 || x86.REG_Z16 <= r && r <= x86.REG_Z31
    67  }
    68  
    69  // loadByRegWidth returns the load instruction of the given register of a given width.
    70  // avx says whether AVX is known to be present, in which case VEX encodings are
    71  // used for otherwise legacy-SSE instructions to avoid AVX-SSE transition
    72  // penalties. See issue #80835.
    73  func loadByRegWidth(r int16, width int64, avx bool) obj.As {
    74  	// Avoid partial register write for GPR
    75  	if !isFPReg(r) && !isKReg(r) {
    76  		switch width {
    77  		case 1:
    78  			return x86.AMOVBLZX
    79  		case 2:
    80  			return x86.AMOVWLZX
    81  		}
    82  	}
    83  	// Otherwise, there's no difference between load and store opcodes.
    84  	return storeByRegWidth(r, width, avx)
    85  }
    86  
    87  // storeByRegWidth returns the store instruction of the given register of a given width.
    88  // It's also used for loading const to a reg.
    89  // See loadByRegWidth for the meaning of avx.
    90  func storeByRegWidth(r int16, width int64, avx bool) obj.As {
    91  	if isHighFPReg(r) {
    92  		// High registers require AVX512 instruction
    93  		return x86.AVMOVDQU64
    94  	}
    95  	if isFPReg(r) {
    96  		if avx {
    97  			switch width {
    98  			case 4:
    99  				return x86.AVMOVSS
   100  			case 8:
   101  				return x86.AVMOVSD
   102  			case 16:
   103  				return x86.AVMOVUPS
   104  			}
   105  		}
   106  		switch width {
   107  		case 4:
   108  			return x86.AMOVSS
   109  		case 8:
   110  			return x86.AMOVSD
   111  		case 16:
   112  			// int128s are in SSE registers
   113  			return x86.AMOVUPS
   114  		case 32:
   115  			return x86.AVMOVDQU
   116  		case 64:
   117  			return x86.AVMOVDQU64
   118  		}
   119  	}
   120  	if isKReg(r) {
   121  		return x86.AKMOVQ
   122  	}
   123  	// gp
   124  	switch width {
   125  	case 1:
   126  		return x86.AMOVB
   127  	case 2:
   128  		return x86.AMOVW
   129  	case 4:
   130  		return x86.AMOVL
   131  	case 8:
   132  		return x86.AMOVQ
   133  	}
   134  	panic(fmt.Sprintf("bad store reg=%v, width=%d", r, width))
   135  }
   136  
   137  // moveByRegsWidth returns the reg->reg move instruction of the given dest/src registers of a given width.
   138  // See loadByRegWidth for the meaning of avx.
   139  func moveByRegsWidth(dest, src int16, width int64, avx bool) obj.As {
   140  	// fp -> fp
   141  	if isFPReg(dest) && isFPReg(src) {
   142  		if isHighFPReg(src) || isHighFPReg(dest) {
   143  			// High registers require AVX512 instruction
   144  			return x86.AVMOVDQU64
   145  		}
   146  		// Moving the whole sse2 register is faster
   147  		// than moving just the correct low portion of it.
   148  		// There is no xmm->xmm move with 1 byte opcode,
   149  		// so use movups, which has 2 byte opcode.
   150  		if width <= 16 {
   151  			if avx {
   152  				return x86.AVMOVUPS
   153  			}
   154  			return x86.AMOVUPS
   155  		}
   156  		if width <= 32 {
   157  			return x86.AVMOVDQU
   158  		}
   159  		return x86.AVMOVDQU64
   160  	}
   161  	// k -> gp, gp -> k, k -> k
   162  	if isKReg(dest) || isKReg(src) {
   163  		if isFPReg(dest) || isFPReg(src) {
   164  			panic(fmt.Sprintf("bad move, src=%v, dest=%v, width=%d", src, dest, width))
   165  		}
   166  		return x86.AKMOVQ
   167  	}
   168  	// gp -> fp, fp -> gp
   169  	if avx && (isFPReg(dest) || isFPReg(src)) {
   170  		switch width {
   171  		case 1, 2, 4:
   172  			return x86.AVMOVD
   173  		case 8:
   174  			return x86.AVMOVQ
   175  		}
   176  	}
   177  	// gp -> fp, fp -> gp, gp -> gp
   178  	switch width {
   179  	case 1:
   180  		// Avoids partial register write
   181  		return x86.AMOVL
   182  	case 2:
   183  		return x86.AMOVL
   184  	case 4:
   185  		return x86.AMOVL
   186  	case 8:
   187  		return x86.AMOVQ
   188  	case 16:
   189  		// int128s are in SSE registers
   190  		return x86.AMOVUPS
   191  	case 32:
   192  		return x86.AVMOVDQU
   193  	case 64:
   194  		return x86.AVMOVDQU64
   195  	}
   196  	panic(fmt.Sprintf("bad move, src=%v, dest=%v, width=%d", src, dest, width))
   197  }
   198  
   199  // opregreg emits instructions for
   200  //
   201  //	dest := dest(To) op src(From)
   202  //
   203  // and also returns the created obj.Prog so it
   204  // may be further adjusted (offset, scale, etc).
   205  func opregreg(s *ssagen.State, op obj.As, dest, src int16) *obj.Prog {
   206  	p := s.Prog(op)
   207  	p.From.Type = obj.TYPE_REG
   208  	p.To.Type = obj.TYPE_REG
   209  	p.To.Reg = dest
   210  	p.From.Reg = src
   211  	return p
   212  }
   213  
   214  // memIdx fills out a as an indexed memory reference for v.
   215  // It assumes that the base register and the index register
   216  // are v.Args[0].Reg() and v.Args[1].Reg(), respectively.
   217  // The caller must still use gc.AddAux/gc.AddAux2 to handle v.Aux as necessary.
   218  func memIdx(a *obj.Addr, v *ssa.Value) {
   219  	r, i := v.Args[0].Reg(), v.Args[1].Reg()
   220  	a.Type = obj.TYPE_MEM
   221  	a.Scale = v.Op.Scale()
   222  	if a.Scale == 1 && i == x86.REG_SP {
   223  		r, i = i, r
   224  	}
   225  	a.Reg = r
   226  	a.Index = i
   227  }
   228  
   229  func getgFromTLS(s *ssagen.State, r int16) {
   230  	// See the comments in cmd/internal/obj/x86/obj6.go
   231  	// near CanUse1InsnTLS for a detailed explanation of these instructions.
   232  	if x86.CanUse1InsnTLS(base.Ctxt) {
   233  		// MOVQ (TLS), r
   234  		p := s.Prog(x86.AMOVQ)
   235  		p.From.Type = obj.TYPE_MEM
   236  		p.From.Reg = x86.REG_TLS
   237  		p.To.Type = obj.TYPE_REG
   238  		p.To.Reg = r
   239  	} else {
   240  		// MOVQ TLS, r
   241  		// MOVQ (r)(TLS*1), r
   242  		p := s.Prog(x86.AMOVQ)
   243  		p.From.Type = obj.TYPE_REG
   244  		p.From.Reg = x86.REG_TLS
   245  		p.To.Type = obj.TYPE_REG
   246  		p.To.Reg = r
   247  		q := s.Prog(x86.AMOVQ)
   248  		q.From.Type = obj.TYPE_MEM
   249  		q.From.Reg = r
   250  		q.From.Index = x86.REG_TLS
   251  		q.From.Scale = 1
   252  		q.To.Type = obj.TYPE_REG
   253  		q.To.Reg = r
   254  	}
   255  }
   256  
   257  func ssaGenValue(s *ssagen.State, v *ssa.Value) {
   258  	switch v.Op {
   259  	case ssaop.OpAMD64VFMADD231SD, ssaop.OpAMD64VFMADD231SS, ssaop.OpAMD64VFMSUB231SD, ssaop.OpAMD64VFMSUB231SS, ssaop.OpAMD64VFNMADD231SD, ssaop.OpAMD64VFNMADD231SS:
   260  		p := s.Prog(v.Op.Asm())
   261  		p.From = obj.Addr{Type: obj.TYPE_REG, Reg: v.Args[2].Reg()}
   262  		p.To = obj.Addr{Type: obj.TYPE_REG, Reg: v.Reg()}
   263  		p.AddRestSourceReg(v.Args[1].Reg())
   264  	case ssaop.OpAMD64ADDQ, ssaop.OpAMD64ADDL:
   265  		r := v.Reg()
   266  		r1 := v.Args[0].Reg()
   267  		r2 := v.Args[1].Reg()
   268  		switch {
   269  		case r == r1:
   270  			p := s.Prog(v.Op.Asm())
   271  			p.From.Type = obj.TYPE_REG
   272  			p.From.Reg = r2
   273  			p.To.Type = obj.TYPE_REG
   274  			p.To.Reg = r
   275  		case r == r2:
   276  			p := s.Prog(v.Op.Asm())
   277  			p.From.Type = obj.TYPE_REG
   278  			p.From.Reg = r1
   279  			p.To.Type = obj.TYPE_REG
   280  			p.To.Reg = r
   281  		default:
   282  			var asm obj.As
   283  			if v.Op == ssaop.OpAMD64ADDQ {
   284  				asm = x86.ALEAQ
   285  			} else {
   286  				asm = x86.ALEAL
   287  			}
   288  			p := s.Prog(asm)
   289  			p.From.Type = obj.TYPE_MEM
   290  			p.From.Reg = r1
   291  			p.From.Scale = 1
   292  			p.From.Index = r2
   293  			p.To.Type = obj.TYPE_REG
   294  			p.To.Reg = r
   295  		}
   296  	// 2-address opcode arithmetic
   297  	case ssaop.OpAMD64SUBQ, ssaop.OpAMD64SUBL,
   298  		ssaop.OpAMD64MULQ, ssaop.OpAMD64MULL,
   299  		ssaop.OpAMD64ANDQ, ssaop.OpAMD64ANDL,
   300  		ssaop.OpAMD64ORQ, ssaop.OpAMD64ORL,
   301  		ssaop.OpAMD64XORQ, ssaop.OpAMD64XORL,
   302  		ssaop.OpAMD64SHLQ, ssaop.OpAMD64SHLL,
   303  		ssaop.OpAMD64SHRQ, ssaop.OpAMD64SHRL, ssaop.OpAMD64SHRW, ssaop.OpAMD64SHRB,
   304  		ssaop.OpAMD64SARQ, ssaop.OpAMD64SARL, ssaop.OpAMD64SARW, ssaop.OpAMD64SARB,
   305  		ssaop.OpAMD64ROLQ, ssaop.OpAMD64ROLL, ssaop.OpAMD64ROLW, ssaop.OpAMD64ROLB,
   306  		ssaop.OpAMD64RORQ, ssaop.OpAMD64RORL, ssaop.OpAMD64RORW, ssaop.OpAMD64RORB,
   307  		ssaop.OpAMD64ADDSS, ssaop.OpAMD64ADDSD, ssaop.OpAMD64SUBSS, ssaop.OpAMD64SUBSD,
   308  		ssaop.OpAMD64MULSS, ssaop.OpAMD64MULSD, ssaop.OpAMD64DIVSS, ssaop.OpAMD64DIVSD,
   309  		ssaop.OpAMD64MINSS, ssaop.OpAMD64MINSD,
   310  		ssaop.OpAMD64MAXSS, ssaop.OpAMD64MAXSD,
   311  		ssaop.OpAMD64POR, ssaop.OpAMD64PXOR,
   312  		ssaop.OpAMD64BTSL, ssaop.OpAMD64BTSQ,
   313  		ssaop.OpAMD64BTCL, ssaop.OpAMD64BTCQ,
   314  		ssaop.OpAMD64BTRL, ssaop.OpAMD64BTRQ,
   315  		ssaop.OpAMD64PCMPEQB, ssaop.OpAMD64PSIGNB,
   316  		ssaop.OpAMD64PUNPCKLBW:
   317  		opregreg(s, v.Op.Asm(), v.Reg(), v.Args[1].Reg())
   318  
   319  	case ssaop.OpAMD64PSHUFLW:
   320  		p := s.Prog(v.Op.Asm())
   321  		imm := v.AuxInt
   322  		if imm < 0 || imm > 255 {
   323  			v.Fatalf("Invalid source selection immediate")
   324  		}
   325  		p.From.Offset = imm
   326  		p.From.Type = obj.TYPE_CONST
   327  		p.AddRestSourceReg(v.Args[0].Reg())
   328  		p.To.Type = obj.TYPE_REG
   329  		p.To.Reg = v.Reg()
   330  
   331  	case ssaop.OpAMD64PSHUFBbroadcast:
   332  		// PSHUFB with a control mask of zero copies byte 0 to all
   333  		// bytes in the register.
   334  		//
   335  		// X15 is always zero with ABIInternal.
   336  		if s.ABI != obj.ABIInternal {
   337  			// zero X15 manually
   338  			opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
   339  		}
   340  
   341  		p := s.Prog(v.Op.Asm())
   342  		p.From.Type = obj.TYPE_REG
   343  		p.To.Type = obj.TYPE_REG
   344  		p.To.Reg = v.Reg()
   345  		p.From.Reg = x86.REG_X15
   346  
   347  	case ssaop.OpAMD64BLSIQ, ssaop.OpAMD64BLSIL,
   348  		ssaop.OpAMD64BLSMSKQ, ssaop.OpAMD64BLSMSKL,
   349  		ssaop.OpAMD64BLSRQ, ssaop.OpAMD64BLSRL:
   350  		p := s.Prog(v.Op.Asm())
   351  		p.From.Type = obj.TYPE_REG
   352  		p.From.Reg = v.Args[0].Reg()
   353  		p.To.Type = obj.TYPE_REG
   354  		switch v.Op {
   355  		case ssaop.OpAMD64BLSRQ, ssaop.OpAMD64BLSRL:
   356  			p.To.Reg = v.Reg0()
   357  		default:
   358  			p.To.Reg = v.Reg()
   359  		}
   360  
   361  	case ssaop.OpAMD64ANDNQ, ssaop.OpAMD64ANDNL:
   362  		p := s.Prog(v.Op.Asm())
   363  		p.From.Type = obj.TYPE_REG
   364  		p.From.Reg = v.Args[0].Reg()
   365  		p.To.Type = obj.TYPE_REG
   366  		p.To.Reg = v.Reg()
   367  		p.AddRestSourceReg(v.Args[1].Reg())
   368  
   369  	case ssaop.OpAMD64SARXL, ssaop.OpAMD64SARXQ,
   370  		ssaop.OpAMD64SHLXL, ssaop.OpAMD64SHLXQ,
   371  		ssaop.OpAMD64SHRXL, ssaop.OpAMD64SHRXQ:
   372  		p := opregreg(s, v.Op.Asm(), v.Reg(), v.Args[1].Reg())
   373  		p.AddRestSourceReg(v.Args[0].Reg())
   374  
   375  	case ssaop.OpAMD64SHLXLload, ssaop.OpAMD64SHLXQload,
   376  		ssaop.OpAMD64SHRXLload, ssaop.OpAMD64SHRXQload,
   377  		ssaop.OpAMD64SARXLload, ssaop.OpAMD64SARXQload:
   378  		p := opregreg(s, v.Op.Asm(), v.Reg(), v.Args[1].Reg())
   379  		m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[0].Reg()}
   380  		ssagen.AddAux(&m, v)
   381  		p.AddRestSource(m)
   382  
   383  	case ssaop.OpAMD64SHLXLloadidx1, ssaop.OpAMD64SHLXLloadidx4, ssaop.OpAMD64SHLXLloadidx8,
   384  		ssaop.OpAMD64SHRXLloadidx1, ssaop.OpAMD64SHRXLloadidx4, ssaop.OpAMD64SHRXLloadidx8,
   385  		ssaop.OpAMD64SARXLloadidx1, ssaop.OpAMD64SARXLloadidx4, ssaop.OpAMD64SARXLloadidx8,
   386  		ssaop.OpAMD64SHLXQloadidx1, ssaop.OpAMD64SHLXQloadidx8,
   387  		ssaop.OpAMD64SHRXQloadidx1, ssaop.OpAMD64SHRXQloadidx8,
   388  		ssaop.OpAMD64SARXQloadidx1, ssaop.OpAMD64SARXQloadidx8:
   389  		p := opregreg(s, v.Op.Asm(), v.Reg(), v.Args[2].Reg())
   390  		m := obj.Addr{Type: obj.TYPE_MEM}
   391  		memIdx(&m, v)
   392  		ssagen.AddAux(&m, v)
   393  		p.AddRestSource(m)
   394  
   395  	case ssaop.OpAMD64DIVQU, ssaop.OpAMD64DIVLU, ssaop.OpAMD64DIVWU:
   396  		// Arg[0] (the dividend) is in AX.
   397  		// Arg[1] (the divisor) can be in any other register.
   398  		// Result[0] (the quotient) is in AX.
   399  		// Result[1] (the remainder) is in DX.
   400  		r := v.Args[1].Reg()
   401  
   402  		// Zero extend dividend.
   403  		opregreg(s, x86.AXORL, x86.REG_DX, x86.REG_DX)
   404  
   405  		// Issue divide.
   406  		p := s.Prog(v.Op.Asm())
   407  		p.From.Type = obj.TYPE_REG
   408  		p.From.Reg = r
   409  
   410  	case ssaop.OpAMD64DIVQ, ssaop.OpAMD64DIVL, ssaop.OpAMD64DIVW:
   411  		// Arg[0] (the dividend) is in AX.
   412  		// Arg[1] (the divisor) can be in any other register.
   413  		// Result[0] (the quotient) is in AX.
   414  		// Result[1] (the remainder) is in DX.
   415  		r := v.Args[1].Reg()
   416  
   417  		var opCMP, opNEG, opSXD obj.As
   418  		switch v.Op {
   419  		case ssaop.OpAMD64DIVQ:
   420  			opCMP, opNEG, opSXD = x86.ACMPQ, x86.ANEGQ, x86.ACQO
   421  		case ssaop.OpAMD64DIVL:
   422  			opCMP, opNEG, opSXD = x86.ACMPL, x86.ANEGL, x86.ACDQ
   423  		case ssaop.OpAMD64DIVW:
   424  			opCMP, opNEG, opSXD = x86.ACMPW, x86.ANEGW, x86.ACWD
   425  		}
   426  
   427  		// CPU faults upon signed overflow, which occurs when the most
   428  		// negative int is divided by -1. Handle divide by -1 as a special case.
   429  		var j1, j2 *obj.Prog
   430  		if ssa.DivisionNeedsFixUp(v) {
   431  			c := s.Prog(opCMP)
   432  			c.From.Type = obj.TYPE_REG
   433  			c.From.Reg = r
   434  			c.To.Type = obj.TYPE_CONST
   435  			c.To.Offset = -1
   436  
   437  			// Divisor is not -1, proceed with normal division.
   438  			j1 = s.Prog(x86.AJNE)
   439  			j1.To.Type = obj.TYPE_BRANCH
   440  
   441  			// Divisor is -1, manually compute quotient and remainder via fixup code.
   442  			// n / -1 = -n
   443  			n1 := s.Prog(opNEG)
   444  			n1.To.Type = obj.TYPE_REG
   445  			n1.To.Reg = x86.REG_AX
   446  
   447  			// n % -1 == 0
   448  			opregreg(s, x86.AXORL, x86.REG_DX, x86.REG_DX)
   449  
   450  			// TODO(khr): issue only the -1 fixup code we need.
   451  			// For instance, if only the quotient is used, no point in zeroing the remainder.
   452  
   453  			// Skip over normal division.
   454  			j2 = s.Prog(obj.AJMP)
   455  			j2.To.Type = obj.TYPE_BRANCH
   456  		}
   457  
   458  		// Sign extend dividend and perform division.
   459  		p := s.Prog(opSXD)
   460  		if j1 != nil {
   461  			j1.To.SetTarget(p)
   462  		}
   463  		p = s.Prog(v.Op.Asm())
   464  		p.From.Type = obj.TYPE_REG
   465  		p.From.Reg = r
   466  
   467  		if j2 != nil {
   468  			j2.To.SetTarget(s.Pc())
   469  		}
   470  
   471  	case ssaop.OpAMD64HMULQ, ssaop.OpAMD64HMULL, ssaop.OpAMD64HMULQU, ssaop.OpAMD64HMULLU:
   472  		// the frontend rewrites constant division by 8/16/32 bit integers into
   473  		// HMUL by a constant
   474  		// SSA rewrites generate the 64 bit versions
   475  
   476  		// Arg[0] is already in AX as it's the only register we allow
   477  		// and DX is the only output we care about (the high bits)
   478  		p := s.Prog(v.Op.Asm())
   479  		p.From.Type = obj.TYPE_REG
   480  		p.From.Reg = v.Args[1].Reg()
   481  
   482  		// IMULB puts the high portion in AH instead of DL,
   483  		// so move it to DL for consistency
   484  		if v.Type.Size() == 1 {
   485  			m := s.Prog(x86.AMOVB)
   486  			m.From.Type = obj.TYPE_REG
   487  			m.From.Reg = x86.REG_AH
   488  			m.To.Type = obj.TYPE_REG
   489  			m.To.Reg = x86.REG_DX
   490  		}
   491  
   492  	case ssaop.OpAMD64MULQU, ssaop.OpAMD64MULLU:
   493  		// Arg[0] is already in AX as it's the only register we allow
   494  		// results lo in AX
   495  		p := s.Prog(v.Op.Asm())
   496  		p.From.Type = obj.TYPE_REG
   497  		p.From.Reg = v.Args[1].Reg()
   498  
   499  	case ssaop.OpAMD64MULQU2:
   500  		// Arg[0] is already in AX as it's the only register we allow
   501  		// results hi in DX, lo in AX
   502  		p := s.Prog(v.Op.Asm())
   503  		p.From.Type = obj.TYPE_REG
   504  		p.From.Reg = v.Args[1].Reg()
   505  
   506  	case ssaop.OpAMD64MULXQ:
   507  		// Arg[0] is already in DX (the implicit operand); Arg[1] is any GP/mem.
   508  		// SSA outputs are (hi, lo) -> Reg0()=hi, Reg1()=lo.
   509  		// Go assembler syntax: MULXQ src, lo, hi (encodes vvvv=lo, reg=hi).
   510  		p := s.Prog(v.Op.Asm())
   511  		p.From.Type = obj.TYPE_REG
   512  		p.From.Reg = v.Args[1].Reg()
   513  		p.AddRestSourceReg(v.Reg1())
   514  		p.To.Type = obj.TYPE_REG
   515  		p.To.Reg = v.Reg0()
   516  
   517  	case ssaop.OpAMD64DIVQU2:
   518  		// Arg[0], Arg[1] are already in Dx, AX, as they're the only registers we allow
   519  		// results q in AX, r in DX
   520  		p := s.Prog(v.Op.Asm())
   521  		p.From.Type = obj.TYPE_REG
   522  		p.From.Reg = v.Args[2].Reg()
   523  
   524  	case ssaop.OpAMD64AVGQU:
   525  		// compute (x+y)/2 unsigned.
   526  		// Do a 64-bit add, the overflow goes into the carry.
   527  		// Shift right once and pull the carry back into the 63rd bit.
   528  		p := s.Prog(x86.AADDQ)
   529  		p.From.Type = obj.TYPE_REG
   530  		p.To.Type = obj.TYPE_REG
   531  		p.To.Reg = v.Reg()
   532  		p.From.Reg = v.Args[1].Reg()
   533  		p = s.Prog(x86.ARCRQ)
   534  		p.From.Type = obj.TYPE_CONST
   535  		p.From.Offset = 1
   536  		p.To.Type = obj.TYPE_REG
   537  		p.To.Reg = v.Reg()
   538  
   539  	case ssaop.OpAMD64ADDQcarry, ssaop.OpAMD64ADCQ:
   540  		r := v.Reg0()
   541  		r0 := v.Args[0].Reg()
   542  		r1 := v.Args[1].Reg()
   543  		switch r {
   544  		case r0:
   545  			p := s.Prog(v.Op.Asm())
   546  			p.From.Type = obj.TYPE_REG
   547  			p.From.Reg = r1
   548  			p.To.Type = obj.TYPE_REG
   549  			p.To.Reg = r
   550  		case r1:
   551  			p := s.Prog(v.Op.Asm())
   552  			p.From.Type = obj.TYPE_REG
   553  			p.From.Reg = r0
   554  			p.To.Type = obj.TYPE_REG
   555  			p.To.Reg = r
   556  		default:
   557  			v.Fatalf("output not in same register as an input %s", v.LongString())
   558  		}
   559  
   560  	case ssaop.OpAMD64SUBQborrow, ssaop.OpAMD64SBBQ:
   561  		p := s.Prog(v.Op.Asm())
   562  		p.From.Type = obj.TYPE_REG
   563  		p.From.Reg = v.Args[1].Reg()
   564  		p.To.Type = obj.TYPE_REG
   565  		p.To.Reg = v.Reg0()
   566  
   567  	case ssaop.OpAMD64ADDQconstcarry, ssaop.OpAMD64ADCQconst, ssaop.OpAMD64SUBQconstborrow, ssaop.OpAMD64SBBQconst:
   568  		p := s.Prog(v.Op.Asm())
   569  		p.From.Type = obj.TYPE_CONST
   570  		p.From.Offset = v.AuxInt
   571  		p.To.Type = obj.TYPE_REG
   572  		p.To.Reg = v.Reg0()
   573  
   574  	case ssaop.OpAMD64ADDQconst, ssaop.OpAMD64ADDLconst:
   575  		r := v.Reg()
   576  		a := v.Args[0].Reg()
   577  		if r == a {
   578  			switch v.AuxInt {
   579  			case 1:
   580  				var asm obj.As
   581  				// Software optimization manual recommends add $1,reg.
   582  				// But inc/dec is 1 byte smaller. ICC always uses inc
   583  				// Clang/GCC choose depending on flags, but prefer add.
   584  				// Experiments show that inc/dec is both a little faster
   585  				// and make a binary a little smaller.
   586  				if v.Op == ssaop.OpAMD64ADDQconst {
   587  					asm = x86.AINCQ
   588  				} else {
   589  					asm = x86.AINCL
   590  				}
   591  				p := s.Prog(asm)
   592  				p.To.Type = obj.TYPE_REG
   593  				p.To.Reg = r
   594  				return
   595  			case -1:
   596  				var asm obj.As
   597  				if v.Op == ssaop.OpAMD64ADDQconst {
   598  					asm = x86.ADECQ
   599  				} else {
   600  					asm = x86.ADECL
   601  				}
   602  				p := s.Prog(asm)
   603  				p.To.Type = obj.TYPE_REG
   604  				p.To.Reg = r
   605  				return
   606  			case 0x80:
   607  				// 'SUBQ $-0x80, r' is shorter to encode than
   608  				// and functionally equivalent to 'ADDQ $0x80, r'.
   609  				asm := x86.ASUBL
   610  				if v.Op == ssaop.OpAMD64ADDQconst {
   611  					asm = x86.ASUBQ
   612  				}
   613  				p := s.Prog(asm)
   614  				p.From.Type = obj.TYPE_CONST
   615  				p.From.Offset = -0x80
   616  				p.To.Type = obj.TYPE_REG
   617  				p.To.Reg = r
   618  				return
   619  
   620  			}
   621  			p := s.Prog(v.Op.Asm())
   622  			p.From.Type = obj.TYPE_CONST
   623  			p.From.Offset = v.AuxInt
   624  			p.To.Type = obj.TYPE_REG
   625  			p.To.Reg = r
   626  			return
   627  		}
   628  		var asm obj.As
   629  		if v.Op == ssaop.OpAMD64ADDQconst {
   630  			asm = x86.ALEAQ
   631  		} else {
   632  			asm = x86.ALEAL
   633  		}
   634  		p := s.Prog(asm)
   635  		p.From.Type = obj.TYPE_MEM
   636  		p.From.Reg = a
   637  		p.From.Offset = v.AuxInt
   638  		p.To.Type = obj.TYPE_REG
   639  		p.To.Reg = r
   640  
   641  	case ssaop.OpAMD64CMOVQEQ, ssaop.OpAMD64CMOVLEQ, ssaop.OpAMD64CMOVWEQ,
   642  		ssaop.OpAMD64CMOVQLT, ssaop.OpAMD64CMOVLLT, ssaop.OpAMD64CMOVWLT,
   643  		ssaop.OpAMD64CMOVQNE, ssaop.OpAMD64CMOVLNE, ssaop.OpAMD64CMOVWNE,
   644  		ssaop.OpAMD64CMOVQGT, ssaop.OpAMD64CMOVLGT, ssaop.OpAMD64CMOVWGT,
   645  		ssaop.OpAMD64CMOVQLE, ssaop.OpAMD64CMOVLLE, ssaop.OpAMD64CMOVWLE,
   646  		ssaop.OpAMD64CMOVQGE, ssaop.OpAMD64CMOVLGE, ssaop.OpAMD64CMOVWGE,
   647  		ssaop.OpAMD64CMOVQHI, ssaop.OpAMD64CMOVLHI, ssaop.OpAMD64CMOVWHI,
   648  		ssaop.OpAMD64CMOVQLS, ssaop.OpAMD64CMOVLLS, ssaop.OpAMD64CMOVWLS,
   649  		ssaop.OpAMD64CMOVQCC, ssaop.OpAMD64CMOVLCC, ssaop.OpAMD64CMOVWCC,
   650  		ssaop.OpAMD64CMOVQCS, ssaop.OpAMD64CMOVLCS, ssaop.OpAMD64CMOVWCS,
   651  		ssaop.OpAMD64CMOVQGTF, ssaop.OpAMD64CMOVLGTF, ssaop.OpAMD64CMOVWGTF,
   652  		ssaop.OpAMD64CMOVQGEF, ssaop.OpAMD64CMOVLGEF, ssaop.OpAMD64CMOVWGEF:
   653  		p := s.Prog(v.Op.Asm())
   654  		p.From.Type = obj.TYPE_REG
   655  		p.From.Reg = v.Args[1].Reg()
   656  		p.To.Type = obj.TYPE_REG
   657  		p.To.Reg = v.Reg()
   658  
   659  	case ssaop.OpAMD64CMOVQNEF, ssaop.OpAMD64CMOVLNEF, ssaop.OpAMD64CMOVWNEF:
   660  		// Flag condition: ^ZERO || PARITY
   661  		// Generate:
   662  		//   CMOV*NE  SRC,DST
   663  		//   CMOV*PS  SRC,DST
   664  		p := s.Prog(v.Op.Asm())
   665  		p.From.Type = obj.TYPE_REG
   666  		p.From.Reg = v.Args[1].Reg()
   667  		p.To.Type = obj.TYPE_REG
   668  		p.To.Reg = v.Reg()
   669  		var q *obj.Prog
   670  		if v.Op == ssaop.OpAMD64CMOVQNEF {
   671  			q = s.Prog(x86.ACMOVQPS)
   672  		} else if v.Op == ssaop.OpAMD64CMOVLNEF {
   673  			q = s.Prog(x86.ACMOVLPS)
   674  		} else {
   675  			q = s.Prog(x86.ACMOVWPS)
   676  		}
   677  		q.From.Type = obj.TYPE_REG
   678  		q.From.Reg = v.Args[1].Reg()
   679  		q.To.Type = obj.TYPE_REG
   680  		q.To.Reg = v.Reg()
   681  
   682  	case ssaop.OpAMD64CMOVQEQF, ssaop.OpAMD64CMOVLEQF, ssaop.OpAMD64CMOVWEQF:
   683  		// Flag condition: ZERO && !PARITY
   684  		// Generate:
   685  		//   MOV      SRC,TMP
   686  		//   CMOV*NE  DST,TMP
   687  		//   CMOV*PC  TMP,DST
   688  		//
   689  		// TODO(rasky): we could generate:
   690  		//   CMOV*NE  DST,SRC
   691  		//   CMOV*PC  SRC,DST
   692  		// But this requires a way for regalloc to know that SRC might be
   693  		// clobbered by this instruction.
   694  		t := v.RegTmp()
   695  		opregreg(s, moveByRegsWidth(t, v.Args[1].Reg(), v.Type.Size(), hasAVX(v)), t, v.Args[1].Reg())
   696  
   697  		p := s.Prog(v.Op.Asm())
   698  		p.From.Type = obj.TYPE_REG
   699  		p.From.Reg = v.Reg()
   700  		p.To.Type = obj.TYPE_REG
   701  		p.To.Reg = t
   702  		var q *obj.Prog
   703  		if v.Op == ssaop.OpAMD64CMOVQEQF {
   704  			q = s.Prog(x86.ACMOVQPC)
   705  		} else if v.Op == ssaop.OpAMD64CMOVLEQF {
   706  			q = s.Prog(x86.ACMOVLPC)
   707  		} else {
   708  			q = s.Prog(x86.ACMOVWPC)
   709  		}
   710  		q.From.Type = obj.TYPE_REG
   711  		q.From.Reg = t
   712  		q.To.Type = obj.TYPE_REG
   713  		q.To.Reg = v.Reg()
   714  
   715  	case ssaop.OpAMD64MULQconst, ssaop.OpAMD64MULLconst:
   716  		r := v.Reg()
   717  		p := s.Prog(v.Op.Asm())
   718  		p.From.Type = obj.TYPE_CONST
   719  		p.From.Offset = v.AuxInt
   720  		p.To.Type = obj.TYPE_REG
   721  		p.To.Reg = r
   722  		p.AddRestSourceReg(v.Args[0].Reg())
   723  
   724  	case ssaop.OpAMD64ANDQconst:
   725  		asm := v.Op.Asm()
   726  		// If the constant is positive and fits into 32 bits, use ANDL.
   727  		// This saves a few bytes of encoding.
   728  		if 0 <= v.AuxInt && v.AuxInt <= (1<<32-1) {
   729  			asm = x86.AANDL
   730  		}
   731  		p := s.Prog(asm)
   732  		p.From.Type = obj.TYPE_CONST
   733  		p.From.Offset = v.AuxInt
   734  		p.To.Type = obj.TYPE_REG
   735  		p.To.Reg = v.Reg()
   736  
   737  	case ssaop.OpAMD64SUBQconst, ssaop.OpAMD64SUBLconst,
   738  		ssaop.OpAMD64ANDLconst,
   739  		ssaop.OpAMD64ORQconst, ssaop.OpAMD64ORLconst,
   740  		ssaop.OpAMD64XORQconst, ssaop.OpAMD64XORLconst:
   741  		p := s.Prog(v.Op.Asm())
   742  		p.From.Type = obj.TYPE_CONST
   743  		p.From.Offset = v.AuxInt
   744  		p.To.Type = obj.TYPE_REG
   745  		p.To.Reg = v.Reg()
   746  
   747  	case ssaop.OpAMD64SHLQconst, ssaop.OpAMD64SHLLconst,
   748  		ssaop.OpAMD64SHRQconst, ssaop.OpAMD64SHRLconst, ssaop.OpAMD64SHRWconst, ssaop.OpAMD64SHRBconst,
   749  		ssaop.OpAMD64SARQconst, ssaop.OpAMD64SARLconst, ssaop.OpAMD64SARWconst, ssaop.OpAMD64SARBconst,
   750  		ssaop.OpAMD64ROLQconst, ssaop.OpAMD64ROLLconst, ssaop.OpAMD64ROLWconst, ssaop.OpAMD64ROLBconst:
   751  		var maxShift int64
   752  		switch v.Op {
   753  		case ssaop.OpAMD64SHLQconst, ssaop.OpAMD64SHRQconst, ssaop.OpAMD64SARQconst, ssaop.OpAMD64ROLQconst:
   754  			maxShift = 63
   755  		case ssaop.OpAMD64SHLLconst, ssaop.OpAMD64SHRLconst, ssaop.OpAMD64SARLconst, ssaop.OpAMD64ROLLconst:
   756  			maxShift = 31
   757  		case ssaop.OpAMD64SHRWconst, ssaop.OpAMD64SARWconst, ssaop.OpAMD64ROLWconst:
   758  			maxShift = 15
   759  		case ssaop.OpAMD64SHRBconst, ssaop.OpAMD64SARBconst, ssaop.OpAMD64ROLBconst:
   760  			maxShift = 7
   761  		default:
   762  			panic("unreachable")
   763  		}
   764  		if v.AuxInt < 0 || v.AuxInt > maxShift {
   765  			v.Fatalf("shift amount out of range [0,%d]: %d", maxShift, v.AuxInt)
   766  		}
   767  		p := s.Prog(v.Op.Asm())
   768  		p.From.Type = obj.TYPE_CONST
   769  		p.From.Offset = v.AuxInt
   770  		p.To.Type = obj.TYPE_REG
   771  		p.To.Reg = v.Reg()
   772  	case ssaop.OpAMD64SBBQcarrymask, ssaop.OpAMD64SBBLcarrymask:
   773  		r := v.Reg()
   774  		p := s.Prog(v.Op.Asm())
   775  		p.From.Type = obj.TYPE_REG
   776  		p.From.Reg = r
   777  		p.To.Type = obj.TYPE_REG
   778  		p.To.Reg = r
   779  	case ssaop.OpAMD64LEAQ1, ssaop.OpAMD64LEAQ2, ssaop.OpAMD64LEAQ4, ssaop.OpAMD64LEAQ8,
   780  		ssaop.OpAMD64LEAL1, ssaop.OpAMD64LEAL2, ssaop.OpAMD64LEAL4, ssaop.OpAMD64LEAL8,
   781  		ssaop.OpAMD64LEAW1, ssaop.OpAMD64LEAW2, ssaop.OpAMD64LEAW4, ssaop.OpAMD64LEAW8:
   782  		p := s.Prog(v.Op.Asm())
   783  		memIdx(&p.From, v)
   784  		ssagen.AddAux(&p.From, v)
   785  		p.To.Type = obj.TYPE_REG
   786  		p.To.Reg = v.Reg()
   787  	case ssaop.OpAMD64LEAQ, ssaop.OpAMD64LEAL, ssaop.OpAMD64LEAW:
   788  		p := s.Prog(v.Op.Asm())
   789  		p.From.Type = obj.TYPE_MEM
   790  		p.From.Reg = v.Args[0].Reg()
   791  		ssagen.AddAux(&p.From, v)
   792  		p.To.Type = obj.TYPE_REG
   793  		p.To.Reg = v.Reg()
   794  	case ssaop.OpAMD64CMPQ, ssaop.OpAMD64CMPL, ssaop.OpAMD64CMPW, ssaop.OpAMD64CMPB,
   795  		ssaop.OpAMD64TESTQ, ssaop.OpAMD64TESTL, ssaop.OpAMD64TESTW, ssaop.OpAMD64TESTB,
   796  		ssaop.OpAMD64BTL, ssaop.OpAMD64BTQ:
   797  		opregreg(s, v.Op.Asm(), v.Args[1].Reg(), v.Args[0].Reg())
   798  	case ssaop.OpAMD64UCOMISS, ssaop.OpAMD64UCOMISD:
   799  		// Go assembler has swapped operands for UCOMISx relative to CMP,
   800  		// must account for that right here.
   801  		opregreg(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg())
   802  	case ssaop.OpAMD64CMPQconst, ssaop.OpAMD64CMPLconst, ssaop.OpAMD64CMPWconst, ssaop.OpAMD64CMPBconst:
   803  		p := s.Prog(v.Op.Asm())
   804  		p.From.Type = obj.TYPE_REG
   805  		p.From.Reg = v.Args[0].Reg()
   806  		p.To.Type = obj.TYPE_CONST
   807  		p.To.Offset = v.AuxInt
   808  	case ssaop.OpAMD64BTLconst, ssaop.OpAMD64BTQconst,
   809  		ssaop.OpAMD64TESTQconst, ssaop.OpAMD64TESTLconst, ssaop.OpAMD64TESTWconst, ssaop.OpAMD64TESTBconst,
   810  		ssaop.OpAMD64BTSQconst,
   811  		ssaop.OpAMD64BTCQconst,
   812  		ssaop.OpAMD64BTRQconst:
   813  		op := v.Op
   814  		if op == ssaop.OpAMD64BTQconst && v.AuxInt < 32 {
   815  			// Emit 32-bit version because it's shorter
   816  			op = ssaop.OpAMD64BTLconst
   817  		}
   818  		p := s.Prog(op.Asm())
   819  		p.From.Type = obj.TYPE_CONST
   820  		p.From.Offset = v.AuxInt
   821  		p.To.Type = obj.TYPE_REG
   822  		p.To.Reg = v.Args[0].Reg()
   823  	case ssaop.OpAMD64CMPQload, ssaop.OpAMD64CMPLload, ssaop.OpAMD64CMPWload, ssaop.OpAMD64CMPBload:
   824  		p := s.Prog(v.Op.Asm())
   825  		p.From.Type = obj.TYPE_MEM
   826  		p.From.Reg = v.Args[0].Reg()
   827  		ssagen.AddAux(&p.From, v)
   828  		p.To.Type = obj.TYPE_REG
   829  		p.To.Reg = v.Args[1].Reg()
   830  	case ssaop.OpAMD64CMPQconstload, ssaop.OpAMD64CMPLconstload, ssaop.OpAMD64CMPWconstload, ssaop.OpAMD64CMPBconstload:
   831  		sc := v.AuxValAndOff()
   832  		p := s.Prog(v.Op.Asm())
   833  		p.From.Type = obj.TYPE_MEM
   834  		p.From.Reg = v.Args[0].Reg()
   835  		ssagen.AddAux2(&p.From, v, sc.Off64())
   836  		p.To.Type = obj.TYPE_CONST
   837  		p.To.Offset = sc.Val64()
   838  	case ssaop.OpAMD64CMPQloadidx8, ssaop.OpAMD64CMPQloadidx1, ssaop.OpAMD64CMPLloadidx4, ssaop.OpAMD64CMPLloadidx1, ssaop.OpAMD64CMPWloadidx2, ssaop.OpAMD64CMPWloadidx1, ssaop.OpAMD64CMPBloadidx1:
   839  		p := s.Prog(v.Op.Asm())
   840  		memIdx(&p.From, v)
   841  		ssagen.AddAux(&p.From, v)
   842  		p.To.Type = obj.TYPE_REG
   843  		p.To.Reg = v.Args[2].Reg()
   844  	case ssaop.OpAMD64CMPQconstloadidx8, ssaop.OpAMD64CMPQconstloadidx1, ssaop.OpAMD64CMPLconstloadidx4, ssaop.OpAMD64CMPLconstloadidx1, ssaop.OpAMD64CMPWconstloadidx2, ssaop.OpAMD64CMPWconstloadidx1, ssaop.OpAMD64CMPBconstloadidx1:
   845  		sc := v.AuxValAndOff()
   846  		p := s.Prog(v.Op.Asm())
   847  		memIdx(&p.From, v)
   848  		ssagen.AddAux2(&p.From, v, sc.Off64())
   849  		p.To.Type = obj.TYPE_CONST
   850  		p.To.Offset = sc.Val64()
   851  	case ssaop.OpAMD64MOVLconst, ssaop.OpAMD64MOVQconst:
   852  		x := v.Reg()
   853  
   854  		// If flags aren't live (indicated by v.Aux == nil),
   855  		// then we can rewrite MOV $0, AX into XOR AX, AX.
   856  		if v.AuxInt == 0 && v.Aux == nil {
   857  			opregreg(s, x86.AXORL, x, x)
   858  			break
   859  		}
   860  
   861  		asm := v.Op.Asm()
   862  		// Use MOVL to move a small constant into a register
   863  		// when the constant is positive and fits into 32 bits.
   864  		if 0 <= v.AuxInt && v.AuxInt <= (1<<32-1) {
   865  			// The upper 32bit are zeroed automatically when using MOVL.
   866  			asm = x86.AMOVL
   867  		}
   868  		p := s.Prog(asm)
   869  		p.From.Type = obj.TYPE_CONST
   870  		p.From.Offset = v.AuxInt
   871  		p.To.Type = obj.TYPE_REG
   872  		p.To.Reg = x
   873  
   874  	case ssaop.OpAMD64MOVSSconst, ssaop.OpAMD64MOVSDconst:
   875  		x := v.Reg()
   876  		if !isFPReg(x) && v.AuxInt == 0 && v.Aux == nil {
   877  			opregreg(s, x86.AXORL, x, x)
   878  			break
   879  		}
   880  		p := s.Prog(storeByRegWidth(x, v.Type.Size(), hasAVX(v)))
   881  		p.From.Type = obj.TYPE_FCONST
   882  		p.From.Val = math.Float64frombits(uint64(v.AuxInt))
   883  		p.To.Type = obj.TYPE_REG
   884  		p.To.Reg = x
   885  	case ssaop.OpAMD64MOVOload:
   886  		asm := v.Op.Asm()
   887  		if hasAVX(v) {
   888  			asm = x86.AVMOVUPS
   889  		}
   890  		p := s.Prog(asm)
   891  		p.From.Type = obj.TYPE_MEM
   892  		p.From.Reg = v.Args[0].Reg()
   893  		ssagen.AddAux(&p.From, v)
   894  		p.To.Type = obj.TYPE_REG
   895  		p.To.Reg = v.Reg()
   896  	case ssaop.OpAMD64MOVQload, ssaop.OpAMD64MOVLload, ssaop.OpAMD64MOVWload, ssaop.OpAMD64MOVBload,
   897  		ssaop.OpAMD64MOVSSload, ssaop.OpAMD64MOVSDload, ssaop.OpAMD64MOVBQSXload, ssaop.OpAMD64MOVWQSXload, ssaop.OpAMD64MOVLQSXload,
   898  		ssaop.OpAMD64MOVBEQload, ssaop.OpAMD64MOVBELload:
   899  		p := s.Prog(v.Op.Asm())
   900  		p.From.Type = obj.TYPE_MEM
   901  		p.From.Reg = v.Args[0].Reg()
   902  		ssagen.AddAux(&p.From, v)
   903  		p.To.Type = obj.TYPE_REG
   904  		p.To.Reg = v.Reg()
   905  	case ssaop.OpAMD64MOVBloadidx1, ssaop.OpAMD64MOVWloadidx1, ssaop.OpAMD64MOVLloadidx1, ssaop.OpAMD64MOVQloadidx1, ssaop.OpAMD64MOVSSloadidx1, ssaop.OpAMD64MOVSDloadidx1,
   906  		ssaop.OpAMD64MOVQloadidx8, ssaop.OpAMD64MOVSDloadidx8, ssaop.OpAMD64MOVLloadidx8, ssaop.OpAMD64MOVLloadidx4, ssaop.OpAMD64MOVSSloadidx4, ssaop.OpAMD64MOVWloadidx2,
   907  		ssaop.OpAMD64MOVBELloadidx1, ssaop.OpAMD64MOVBELloadidx4, ssaop.OpAMD64MOVBELloadidx8, ssaop.OpAMD64MOVBEQloadidx1, ssaop.OpAMD64MOVBEQloadidx8:
   908  		p := s.Prog(v.Op.Asm())
   909  		memIdx(&p.From, v)
   910  		ssagen.AddAux(&p.From, v)
   911  		p.To.Type = obj.TYPE_REG
   912  		p.To.Reg = v.Reg()
   913  	case ssaop.OpAMD64MOVOstore:
   914  		asm := v.Op.Asm()
   915  		if hasAVX(v) {
   916  			asm = x86.AVMOVUPS
   917  		}
   918  		p := s.Prog(asm)
   919  		p.From.Type = obj.TYPE_REG
   920  		p.From.Reg = v.Args[1].Reg()
   921  		p.To.Type = obj.TYPE_MEM
   922  		p.To.Reg = v.Args[0].Reg()
   923  		ssagen.AddAux(&p.To, v)
   924  	case ssaop.OpAMD64MOVQstore, ssaop.OpAMD64MOVSSstore, ssaop.OpAMD64MOVSDstore, ssaop.OpAMD64MOVLstore, ssaop.OpAMD64MOVWstore, ssaop.OpAMD64MOVBstore,
   925  		ssaop.OpAMD64ADDQmodify, ssaop.OpAMD64SUBQmodify, ssaop.OpAMD64ANDQmodify, ssaop.OpAMD64ORQmodify, ssaop.OpAMD64XORQmodify,
   926  		ssaop.OpAMD64ADDLmodify, ssaop.OpAMD64SUBLmodify, ssaop.OpAMD64ANDLmodify, ssaop.OpAMD64ORLmodify, ssaop.OpAMD64XORLmodify,
   927  		ssaop.OpAMD64MOVBEQstore, ssaop.OpAMD64MOVBELstore, ssaop.OpAMD64MOVBEWstore:
   928  		p := s.Prog(v.Op.Asm())
   929  		p.From.Type = obj.TYPE_REG
   930  		p.From.Reg = v.Args[1].Reg()
   931  		p.To.Type = obj.TYPE_MEM
   932  		p.To.Reg = v.Args[0].Reg()
   933  		ssagen.AddAux(&p.To, v)
   934  	case ssaop.OpAMD64MOVBstoreidx1, ssaop.OpAMD64MOVWstoreidx1, ssaop.OpAMD64MOVLstoreidx1, ssaop.OpAMD64MOVQstoreidx1, ssaop.OpAMD64MOVSSstoreidx1, ssaop.OpAMD64MOVSDstoreidx1,
   935  		ssaop.OpAMD64MOVQstoreidx8, ssaop.OpAMD64MOVSDstoreidx8, ssaop.OpAMD64MOVLstoreidx8, ssaop.OpAMD64MOVSSstoreidx4, ssaop.OpAMD64MOVLstoreidx4, ssaop.OpAMD64MOVWstoreidx2,
   936  		ssaop.OpAMD64ADDLmodifyidx1, ssaop.OpAMD64ADDLmodifyidx4, ssaop.OpAMD64ADDLmodifyidx8, ssaop.OpAMD64ADDQmodifyidx1, ssaop.OpAMD64ADDQmodifyidx8,
   937  		ssaop.OpAMD64SUBLmodifyidx1, ssaop.OpAMD64SUBLmodifyidx4, ssaop.OpAMD64SUBLmodifyidx8, ssaop.OpAMD64SUBQmodifyidx1, ssaop.OpAMD64SUBQmodifyidx8,
   938  		ssaop.OpAMD64ANDLmodifyidx1, ssaop.OpAMD64ANDLmodifyidx4, ssaop.OpAMD64ANDLmodifyidx8, ssaop.OpAMD64ANDQmodifyidx1, ssaop.OpAMD64ANDQmodifyidx8,
   939  		ssaop.OpAMD64ORLmodifyidx1, ssaop.OpAMD64ORLmodifyidx4, ssaop.OpAMD64ORLmodifyidx8, ssaop.OpAMD64ORQmodifyidx1, ssaop.OpAMD64ORQmodifyidx8,
   940  		ssaop.OpAMD64XORLmodifyidx1, ssaop.OpAMD64XORLmodifyidx4, ssaop.OpAMD64XORLmodifyidx8, ssaop.OpAMD64XORQmodifyidx1, ssaop.OpAMD64XORQmodifyidx8,
   941  		ssaop.OpAMD64MOVBEWstoreidx1, ssaop.OpAMD64MOVBEWstoreidx2, ssaop.OpAMD64MOVBELstoreidx1, ssaop.OpAMD64MOVBELstoreidx4, ssaop.OpAMD64MOVBELstoreidx8, ssaop.OpAMD64MOVBEQstoreidx1, ssaop.OpAMD64MOVBEQstoreidx8:
   942  		p := s.Prog(v.Op.Asm())
   943  		p.From.Type = obj.TYPE_REG
   944  		p.From.Reg = v.Args[2].Reg()
   945  		memIdx(&p.To, v)
   946  		ssagen.AddAux(&p.To, v)
   947  	case ssaop.OpAMD64ADDQconstmodify, ssaop.OpAMD64ADDLconstmodify,
   948  		ssaop.OpAMD64ADDWconstmodify, ssaop.OpAMD64ADDBconstmodify:
   949  		sc := v.AuxValAndOff()
   950  		off := sc.Off64()
   951  		val := sc.Val()
   952  		if val == 1 || val == -1 {
   953  			var asm obj.As
   954  			switch v.Op {
   955  			case ssaop.OpAMD64ADDQconstmodify:
   956  				asm = x86.AINCQ
   957  				if val == -1 {
   958  					asm = x86.ADECQ
   959  				}
   960  			case ssaop.OpAMD64ADDLconstmodify:
   961  				asm = x86.AINCL
   962  				if val == -1 {
   963  					asm = x86.ADECL
   964  				}
   965  			case ssaop.OpAMD64ADDWconstmodify:
   966  				asm = x86.AINCW
   967  				if val == -1 {
   968  					asm = x86.ADECW
   969  				}
   970  			default:
   971  				asm = x86.AINCB
   972  				if val == -1 {
   973  					asm = x86.ADECB
   974  				}
   975  			}
   976  			p := s.Prog(asm)
   977  			p.To.Type = obj.TYPE_MEM
   978  			p.To.Reg = v.Args[0].Reg()
   979  			ssagen.AddAux2(&p.To, v, off)
   980  			break
   981  		}
   982  		fallthrough
   983  	case ssaop.OpAMD64ANDQconstmodify, ssaop.OpAMD64ANDLconstmodify, ssaop.OpAMD64ORQconstmodify, ssaop.OpAMD64ORLconstmodify,
   984  		ssaop.OpAMD64XORQconstmodify, ssaop.OpAMD64XORLconstmodify,
   985  		ssaop.OpAMD64ANDWconstmodify, ssaop.OpAMD64ANDBconstmodify, ssaop.OpAMD64ORWconstmodify, ssaop.OpAMD64ORBconstmodify,
   986  		ssaop.OpAMD64XORWconstmodify, ssaop.OpAMD64XORBconstmodify,
   987  		ssaop.OpAMD64BTSQconstmodify, ssaop.OpAMD64BTRQconstmodify, ssaop.OpAMD64BTCQconstmodify:
   988  		sc := v.AuxValAndOff()
   989  		off := sc.Off64()
   990  		val := sc.Val64()
   991  		p := s.Prog(v.Op.Asm())
   992  		p.From.Type = obj.TYPE_CONST
   993  		p.From.Offset = val
   994  		p.To.Type = obj.TYPE_MEM
   995  		p.To.Reg = v.Args[0].Reg()
   996  		ssagen.AddAux2(&p.To, v, off)
   997  
   998  	case ssaop.OpAMD64MOVQstoreconst, ssaop.OpAMD64MOVLstoreconst, ssaop.OpAMD64MOVWstoreconst, ssaop.OpAMD64MOVBstoreconst:
   999  		sc := v.AuxValAndOff()
  1000  		p := s.Prog(v.Op.Asm())
  1001  		if sc.Val() == 0 && s.ABI == obj.ABIInternal && buildcfg.GOOS != "plan9" && (v.Op == ssaop.OpAMD64MOVQstoreconst || v.Op == ssaop.OpAMD64MOVLstoreconst) {
  1002  			p.From.Type = obj.TYPE_REG
  1003  			p.From.Reg = x86.REG_X15
  1004  		} else {
  1005  			p.From.Type = obj.TYPE_CONST
  1006  			p.From.Offset = sc.Val64()
  1007  		}
  1008  		p.To.Type = obj.TYPE_MEM
  1009  		p.To.Reg = v.Args[0].Reg()
  1010  		ssagen.AddAux2(&p.To, v, sc.Off64())
  1011  	case ssaop.OpAMD64MOVOstoreconst:
  1012  		sc := v.AuxValAndOff()
  1013  		if sc.Val() != 0 {
  1014  			v.Fatalf("MOVO for non zero constants not implemented: %s", v.LongString())
  1015  		}
  1016  
  1017  		avx := hasAVX(v)
  1018  		if s.ABI != obj.ABIInternal {
  1019  			// zero X15 manually
  1020  			zeroX15Low(s, avx)
  1021  		}
  1022  		asm := v.Op.Asm()
  1023  		if avx {
  1024  			asm = x86.AVMOVUPS
  1025  		}
  1026  		p := s.Prog(asm)
  1027  		p.From.Type = obj.TYPE_REG
  1028  		p.From.Reg = x86.REG_X15
  1029  		p.To.Type = obj.TYPE_MEM
  1030  		p.To.Reg = v.Args[0].Reg()
  1031  		ssagen.AddAux2(&p.To, v, sc.Off64())
  1032  
  1033  	case ssaop.OpAMD64MOVQstoreconstidx1, ssaop.OpAMD64MOVQstoreconstidx8, ssaop.OpAMD64MOVLstoreconstidx1, ssaop.OpAMD64MOVLstoreconstidx4, ssaop.OpAMD64MOVWstoreconstidx1, ssaop.OpAMD64MOVWstoreconstidx2, ssaop.OpAMD64MOVBstoreconstidx1,
  1034  		ssaop.OpAMD64ADDLconstmodifyidx1, ssaop.OpAMD64ADDLconstmodifyidx4, ssaop.OpAMD64ADDLconstmodifyidx8, ssaop.OpAMD64ADDQconstmodifyidx1, ssaop.OpAMD64ADDQconstmodifyidx8,
  1035  		ssaop.OpAMD64ANDLconstmodifyidx1, ssaop.OpAMD64ANDLconstmodifyidx4, ssaop.OpAMD64ANDLconstmodifyidx8, ssaop.OpAMD64ANDQconstmodifyidx1, ssaop.OpAMD64ANDQconstmodifyidx8,
  1036  		ssaop.OpAMD64ORLconstmodifyidx1, ssaop.OpAMD64ORLconstmodifyidx4, ssaop.OpAMD64ORLconstmodifyidx8, ssaop.OpAMD64ORQconstmodifyidx1, ssaop.OpAMD64ORQconstmodifyidx8,
  1037  		ssaop.OpAMD64XORLconstmodifyidx1, ssaop.OpAMD64XORLconstmodifyidx4, ssaop.OpAMD64XORLconstmodifyidx8, ssaop.OpAMD64XORQconstmodifyidx1, ssaop.OpAMD64XORQconstmodifyidx8,
  1038  		ssaop.OpAMD64ADDWconstmodifyidx1, ssaop.OpAMD64ADDWconstmodifyidx2, ssaop.OpAMD64ADDBconstmodifyidx1,
  1039  		ssaop.OpAMD64ANDWconstmodifyidx1, ssaop.OpAMD64ANDWconstmodifyidx2, ssaop.OpAMD64ANDBconstmodifyidx1,
  1040  		ssaop.OpAMD64ORWconstmodifyidx1, ssaop.OpAMD64ORWconstmodifyidx2, ssaop.OpAMD64ORBconstmodifyidx1,
  1041  		ssaop.OpAMD64XORWconstmodifyidx1, ssaop.OpAMD64XORWconstmodifyidx2, ssaop.OpAMD64XORBconstmodifyidx1:
  1042  		p := s.Prog(v.Op.Asm())
  1043  		p.From.Type = obj.TYPE_CONST
  1044  		sc := v.AuxValAndOff()
  1045  		p.From.Offset = sc.Val64()
  1046  		if sc.Val() == 0 && s.ABI == obj.ABIInternal && buildcfg.GOOS != "plan9" {
  1047  			switch v.Op {
  1048  			case ssaop.OpAMD64MOVQstoreconstidx1, ssaop.OpAMD64MOVQstoreconstidx8,
  1049  				ssaop.OpAMD64MOVLstoreconstidx1, ssaop.OpAMD64MOVLstoreconstidx4:
  1050  				p.From.Type = obj.TYPE_REG
  1051  				p.From.Reg = x86.REG_X15
  1052  			}
  1053  		}
  1054  		switch {
  1055  		case p.As == x86.AADDQ && p.From.Offset == 1:
  1056  			p.As = x86.AINCQ
  1057  			p.From.Type = obj.TYPE_NONE
  1058  		case p.As == x86.AADDQ && p.From.Offset == -1:
  1059  			p.As = x86.ADECQ
  1060  			p.From.Type = obj.TYPE_NONE
  1061  		case p.As == x86.AADDL && p.From.Offset == 1:
  1062  			p.As = x86.AINCL
  1063  			p.From.Type = obj.TYPE_NONE
  1064  		case p.As == x86.AADDL && p.From.Offset == -1:
  1065  			p.As = x86.ADECL
  1066  			p.From.Type = obj.TYPE_NONE
  1067  		case p.As == x86.AADDW && p.From.Offset == 1:
  1068  			p.As = x86.AINCW
  1069  			p.From.Type = obj.TYPE_NONE
  1070  		case p.As == x86.AADDW && p.From.Offset == -1:
  1071  			p.As = x86.ADECW
  1072  			p.From.Type = obj.TYPE_NONE
  1073  		case p.As == x86.AADDB && p.From.Offset == 1:
  1074  			p.As = x86.AINCB
  1075  			p.From.Type = obj.TYPE_NONE
  1076  		case p.As == x86.AADDB && p.From.Offset == -1:
  1077  			p.As = x86.ADECB
  1078  			p.From.Type = obj.TYPE_NONE
  1079  		}
  1080  		memIdx(&p.To, v)
  1081  		ssagen.AddAux2(&p.To, v, sc.Off64())
  1082  	case ssaop.OpAMD64MOVLQSX, ssaop.OpAMD64MOVWQSX, ssaop.OpAMD64MOVBQSX, ssaop.OpAMD64MOVLQZX, ssaop.OpAMD64MOVWQZX, ssaop.OpAMD64MOVBQZX,
  1083  		ssaop.OpAMD64CVTTSS2SL, ssaop.OpAMD64CVTTSD2SL, ssaop.OpAMD64CVTTSS2SQ, ssaop.OpAMD64CVTTSD2SQ,
  1084  		ssaop.OpAMD64CVTSS2SD, ssaop.OpAMD64CVTSD2SS, ssaop.OpAMD64VPBROADCASTB, ssaop.OpAMD64PMOVMSKB:
  1085  		opregreg(s, v.Op.Asm(), v.Reg(), v.Args[0].Reg())
  1086  	case ssaop.OpAMD64CVTSL2SD, ssaop.OpAMD64CVTSQ2SD, ssaop.OpAMD64CVTSQ2SS, ssaop.OpAMD64CVTSL2SS:
  1087  		r := v.Reg()
  1088  		// Break false dependency on destination register.
  1089  		opregreg(s, x86.AXORPS, r, r)
  1090  		opregreg(s, v.Op.Asm(), r, v.Args[0].Reg())
  1091  	case ssaop.OpAMD64MOVQi2f, ssaop.OpAMD64MOVQf2i, ssaop.OpAMD64MOVLi2f, ssaop.OpAMD64MOVLf2i:
  1092  		avx := hasAVX(v)
  1093  		var p *obj.Prog
  1094  		switch v.Op {
  1095  		case ssaop.OpAMD64MOVQi2f, ssaop.OpAMD64MOVQf2i:
  1096  			if avx {
  1097  				p = s.Prog(x86.AVMOVQ)
  1098  			} else {
  1099  				p = s.Prog(x86.AMOVQ)
  1100  			}
  1101  		case ssaop.OpAMD64MOVLi2f, ssaop.OpAMD64MOVLf2i:
  1102  			if avx {
  1103  				p = s.Prog(x86.AVMOVD)
  1104  			} else {
  1105  				p = s.Prog(x86.AMOVL)
  1106  			}
  1107  		}
  1108  		p.From.Type = obj.TYPE_REG
  1109  		p.From.Reg = v.Args[0].Reg()
  1110  		p.To.Type = obj.TYPE_REG
  1111  		p.To.Reg = v.Reg()
  1112  	case ssaop.OpAMD64ADDQload, ssaop.OpAMD64ADDLload, ssaop.OpAMD64SUBQload, ssaop.OpAMD64SUBLload,
  1113  		ssaop.OpAMD64ANDQload, ssaop.OpAMD64ANDLload, ssaop.OpAMD64ORQload, ssaop.OpAMD64ORLload,
  1114  		ssaop.OpAMD64XORQload, ssaop.OpAMD64XORLload, ssaop.OpAMD64ADDSDload, ssaop.OpAMD64ADDSSload,
  1115  		ssaop.OpAMD64SUBSDload, ssaop.OpAMD64SUBSSload, ssaop.OpAMD64MULSDload, ssaop.OpAMD64MULSSload,
  1116  		ssaop.OpAMD64DIVSDload, ssaop.OpAMD64DIVSSload:
  1117  		p := s.Prog(v.Op.Asm())
  1118  		p.From.Type = obj.TYPE_MEM
  1119  		p.From.Reg = v.Args[1].Reg()
  1120  		ssagen.AddAux(&p.From, v)
  1121  		p.To.Type = obj.TYPE_REG
  1122  		p.To.Reg = v.Reg()
  1123  	case ssaop.OpAMD64ADDLloadidx1, ssaop.OpAMD64ADDLloadidx4, ssaop.OpAMD64ADDLloadidx8, ssaop.OpAMD64ADDQloadidx1, ssaop.OpAMD64ADDQloadidx8,
  1124  		ssaop.OpAMD64SUBLloadidx1, ssaop.OpAMD64SUBLloadidx4, ssaop.OpAMD64SUBLloadidx8, ssaop.OpAMD64SUBQloadidx1, ssaop.OpAMD64SUBQloadidx8,
  1125  		ssaop.OpAMD64ANDLloadidx1, ssaop.OpAMD64ANDLloadidx4, ssaop.OpAMD64ANDLloadidx8, ssaop.OpAMD64ANDQloadidx1, ssaop.OpAMD64ANDQloadidx8,
  1126  		ssaop.OpAMD64ORLloadidx1, ssaop.OpAMD64ORLloadidx4, ssaop.OpAMD64ORLloadidx8, ssaop.OpAMD64ORQloadidx1, ssaop.OpAMD64ORQloadidx8,
  1127  		ssaop.OpAMD64XORLloadidx1, ssaop.OpAMD64XORLloadidx4, ssaop.OpAMD64XORLloadidx8, ssaop.OpAMD64XORQloadidx1, ssaop.OpAMD64XORQloadidx8,
  1128  		ssaop.OpAMD64ADDSSloadidx1, ssaop.OpAMD64ADDSSloadidx4, ssaop.OpAMD64ADDSDloadidx1, ssaop.OpAMD64ADDSDloadidx8,
  1129  		ssaop.OpAMD64SUBSSloadidx1, ssaop.OpAMD64SUBSSloadidx4, ssaop.OpAMD64SUBSDloadidx1, ssaop.OpAMD64SUBSDloadidx8,
  1130  		ssaop.OpAMD64MULSSloadidx1, ssaop.OpAMD64MULSSloadidx4, ssaop.OpAMD64MULSDloadidx1, ssaop.OpAMD64MULSDloadidx8,
  1131  		ssaop.OpAMD64DIVSSloadidx1, ssaop.OpAMD64DIVSSloadidx4, ssaop.OpAMD64DIVSDloadidx1, ssaop.OpAMD64DIVSDloadidx8:
  1132  		p := s.Prog(v.Op.Asm())
  1133  
  1134  		r, i := v.Args[1].Reg(), v.Args[2].Reg()
  1135  		p.From.Type = obj.TYPE_MEM
  1136  		p.From.Scale = v.Op.Scale()
  1137  		if p.From.Scale == 1 && i == x86.REG_SP {
  1138  			r, i = i, r
  1139  		}
  1140  		p.From.Reg = r
  1141  		p.From.Index = i
  1142  
  1143  		ssagen.AddAux(&p.From, v)
  1144  		p.To.Type = obj.TYPE_REG
  1145  		p.To.Reg = v.Reg()
  1146  
  1147  	case ssaop.OpAMD64LoweredZero:
  1148  		avx := hasAVX(v)
  1149  		if s.ABI != obj.ABIInternal {
  1150  			// zero X15 manually
  1151  			zeroX15Low(s, avx)
  1152  		}
  1153  		ptrReg := v.Args[0].Reg()
  1154  		n := v.AuxInt
  1155  		if n < 16 {
  1156  			v.Fatalf("Zero too small %d", n)
  1157  		}
  1158  		zero16 := func(off int64) {
  1159  			zero16(s, ptrReg, off, avx)
  1160  		}
  1161  
  1162  		// Generate zeroing instructions.
  1163  		var off int64
  1164  		for n >= 16 {
  1165  			zero16(off)
  1166  			off += 16
  1167  			n -= 16
  1168  		}
  1169  		if n != 0 {
  1170  			// use partially overlapped write.
  1171  			// TODO: n <= 8, use smaller write?
  1172  			zero16(off + n - 16)
  1173  		}
  1174  
  1175  	case ssaop.OpAMD64LoweredZeroLoop:
  1176  		avx := hasAVX(v)
  1177  		if s.ABI != obj.ABIInternal {
  1178  			// zero X15 manually
  1179  			zeroX15Low(s, avx)
  1180  		}
  1181  		ptrReg := v.Args[0].Reg()
  1182  		countReg := v.RegTmp()
  1183  		n := v.AuxInt
  1184  		loopSize := int64(64)
  1185  		if n < 3*loopSize {
  1186  			// - a loop count of 0 won't work.
  1187  			// - a loop count of 1 is useless.
  1188  			// - a loop count of 2 is a code size ~tie
  1189  			//     4 instructions to implement the loop
  1190  			//     4 instructions in the loop body
  1191  			//   vs
  1192  			//     8 instructions in the straightline code
  1193  			//   Might as well use straightline code.
  1194  			v.Fatalf("ZeroLoop size too small %d", n)
  1195  		}
  1196  		zero16 := func(off int64) {
  1197  			zero16(s, ptrReg, off, avx)
  1198  		}
  1199  
  1200  		// Put iteration count in a register.
  1201  		//   MOVL    $n, countReg
  1202  		p := s.Prog(x86.AMOVL)
  1203  		p.From.Type = obj.TYPE_CONST
  1204  		p.From.Offset = n / loopSize
  1205  		p.To.Type = obj.TYPE_REG
  1206  		p.To.Reg = countReg
  1207  		cntInit := p
  1208  
  1209  		// Zero loopSize bytes starting at ptrReg.
  1210  		for i := range loopSize / 16 {
  1211  			zero16(i * 16)
  1212  		}
  1213  		//   ADDQ    $loopSize, ptrReg
  1214  		p = s.Prog(x86.AADDQ)
  1215  		p.From.Type = obj.TYPE_CONST
  1216  		p.From.Offset = loopSize
  1217  		p.To.Type = obj.TYPE_REG
  1218  		p.To.Reg = ptrReg
  1219  		//   DECL    countReg
  1220  		p = s.Prog(x86.ADECL)
  1221  		p.To.Type = obj.TYPE_REG
  1222  		p.To.Reg = countReg
  1223  		// Jump to first instruction in loop if we're not done yet.
  1224  		//   JNE     head
  1225  		p = s.Prog(x86.AJNE)
  1226  		p.To.Type = obj.TYPE_BRANCH
  1227  		p.To.SetTarget(cntInit.Link)
  1228  
  1229  		// Multiples of the loop size are now done.
  1230  		n %= loopSize
  1231  
  1232  		// Write any fractional portion.
  1233  		var off int64
  1234  		for n >= 16 {
  1235  			zero16(off)
  1236  			off += 16
  1237  			n -= 16
  1238  		}
  1239  		if n != 0 {
  1240  			// Use partially-overlapping write.
  1241  			// TODO: n <= 8, use smaller write?
  1242  			zero16(off + n - 16)
  1243  		}
  1244  
  1245  	case ssaop.OpAMD64LoweredMove:
  1246  		dstReg := v.Args[0].Reg()
  1247  		srcReg := v.Args[1].Reg()
  1248  		if dstReg == srcReg {
  1249  			break
  1250  		}
  1251  		tmpReg := int16(x86.REG_X14)
  1252  		n := v.AuxInt
  1253  		if n < 16 {
  1254  			v.Fatalf("Move too small %d", n)
  1255  		}
  1256  		avx := hasAVX(v)
  1257  		// move 16 bytes from srcReg+off to dstReg+off.
  1258  		move16 := func(off int64) {
  1259  			move16(s, srcReg, dstReg, tmpReg, off, avx)
  1260  		}
  1261  
  1262  		// Generate copying instructions.
  1263  		var off int64
  1264  		for n >= 16 {
  1265  			move16(off)
  1266  			off += 16
  1267  			n -= 16
  1268  		}
  1269  		if n != 0 {
  1270  			// use partially overlapped read/write.
  1271  			// TODO: use smaller operations when we can?
  1272  			move16(off + n - 16)
  1273  		}
  1274  
  1275  	case ssaop.OpAMD64LoweredMoveLoop:
  1276  		dstReg := v.Args[0].Reg()
  1277  		srcReg := v.Args[1].Reg()
  1278  		if dstReg == srcReg {
  1279  			break
  1280  		}
  1281  		countReg := v.RegTmp()
  1282  		tmpReg := int16(x86.REG_X14)
  1283  		n := v.AuxInt
  1284  		loopSize := int64(64)
  1285  		if n < 3*loopSize {
  1286  			// - a loop count of 0 won't work.
  1287  			// - a loop count of 1 is useless.
  1288  			// - a loop count of 2 is a code size ~tie
  1289  			//     4 instructions to implement the loop
  1290  			//     4 instructions in the loop body
  1291  			//   vs
  1292  			//     8 instructions in the straightline code
  1293  			//   Might as well use straightline code.
  1294  			v.Fatalf("ZeroLoop size too small %d", n)
  1295  		}
  1296  		avx := hasAVX(v)
  1297  		// move 16 bytes from srcReg+off to dstReg+off.
  1298  		move16 := func(off int64) {
  1299  			move16(s, srcReg, dstReg, tmpReg, off, avx)
  1300  		}
  1301  
  1302  		// Put iteration count in a register.
  1303  		//   MOVL    $n, countReg
  1304  		p := s.Prog(x86.AMOVL)
  1305  		p.From.Type = obj.TYPE_CONST
  1306  		p.From.Offset = n / loopSize
  1307  		p.To.Type = obj.TYPE_REG
  1308  		p.To.Reg = countReg
  1309  		cntInit := p
  1310  
  1311  		// Copy loopSize bytes starting at srcReg to dstReg.
  1312  		for i := range loopSize / 16 {
  1313  			move16(i * 16)
  1314  		}
  1315  		//   ADDQ    $loopSize, srcReg
  1316  		p = s.Prog(x86.AADDQ)
  1317  		p.From.Type = obj.TYPE_CONST
  1318  		p.From.Offset = loopSize
  1319  		p.To.Type = obj.TYPE_REG
  1320  		p.To.Reg = srcReg
  1321  		//   ADDQ    $loopSize, dstReg
  1322  		p = s.Prog(x86.AADDQ)
  1323  		p.From.Type = obj.TYPE_CONST
  1324  		p.From.Offset = loopSize
  1325  		p.To.Type = obj.TYPE_REG
  1326  		p.To.Reg = dstReg
  1327  		//   DECL    countReg
  1328  		p = s.Prog(x86.ADECL)
  1329  		p.To.Type = obj.TYPE_REG
  1330  		p.To.Reg = countReg
  1331  		// Jump to loop header if we're not done yet.
  1332  		//   JNE     head
  1333  		p = s.Prog(x86.AJNE)
  1334  		p.To.Type = obj.TYPE_BRANCH
  1335  		p.To.SetTarget(cntInit.Link)
  1336  
  1337  		// Multiples of the loop size are now done.
  1338  		n %= loopSize
  1339  
  1340  		// Copy any fractional portion.
  1341  		var off int64
  1342  		for n >= 16 {
  1343  			move16(off)
  1344  			off += 16
  1345  			n -= 16
  1346  		}
  1347  		if n != 0 {
  1348  			// Use partially-overlapping copy.
  1349  			move16(off + n - 16)
  1350  		}
  1351  
  1352  	case ssaop.OpCopy: // TODO: use MOVQreg for reg->reg copies instead of OpCopy?
  1353  		if v.Type.IsMemory() {
  1354  			return
  1355  		}
  1356  		arg := v.Args[0]
  1357  		x := arg.Reg()
  1358  		y := v.Reg()
  1359  		if v.Type.IsSIMD() {
  1360  			x = simdOrMaskReg(arg)
  1361  			y = simdOrMaskReg(v)
  1362  		}
  1363  		if x != y {
  1364  			width := v.Type.Size()
  1365  			if width == 8 && isGPReg(y) && ssa.ZeroUpper32Bits(arg) {
  1366  				// The source was naturally zext-ed from 32 to 64 bits,
  1367  				// but we are asked to do a full 64-bit copy.
  1368  				// Save the REX prefix byte in I-CACHE by using a 32-bit move,
  1369  				// since it zeroes the upper 32 bits anyway.
  1370  				width = 4
  1371  			}
  1372  			opregreg(s, moveByRegsWidth(y, x, width, hasAVX(v)), y, x)
  1373  		}
  1374  	case ssaop.OpLoadReg:
  1375  		if v.Type.IsFlags() {
  1376  			v.Fatalf("load flags not implemented: %v", v.LongString())
  1377  			return
  1378  		}
  1379  		r := v.Reg()
  1380  		p := s.Prog(loadByRegWidth(r, v.Type.Size(), hasAVX(v)))
  1381  		ssagen.AddrAuto(&p.From, v.Args[0])
  1382  		p.To.Type = obj.TYPE_REG
  1383  		if v.Type.IsSIMD() {
  1384  			r = simdOrMaskReg(v)
  1385  		}
  1386  		p.To.Reg = r
  1387  
  1388  	case ssaop.OpStoreReg:
  1389  		if v.Type.IsFlags() {
  1390  			v.Fatalf("store flags not implemented: %v", v.LongString())
  1391  			return
  1392  		}
  1393  		r := v.Args[0].Reg()
  1394  		if v.Type.IsSIMD() {
  1395  			r = simdOrMaskReg(v.Args[0])
  1396  		}
  1397  		p := s.Prog(storeByRegWidth(r, v.Type.Size(), hasAVX(v)))
  1398  		p.From.Type = obj.TYPE_REG
  1399  		p.From.Reg = r
  1400  		ssagen.AddrAuto(&p.To, v)
  1401  	case ssaop.OpAMD64LoweredHasCPUFeature:
  1402  		// If this load changes width, update zeroUpperBits in AMD64Ops.go.
  1403  		p := s.Prog(x86.AMOVBLZX)
  1404  		p.From.Type = obj.TYPE_MEM
  1405  		ssagen.AddAux(&p.From, v)
  1406  		p.To.Type = obj.TYPE_REG
  1407  		p.To.Reg = v.Reg()
  1408  	case ssaop.OpArgIntReg, ssaop.OpArgFloatReg:
  1409  		// The assembler needs to wrap the entry safepoint/stack growth code with spill/unspill
  1410  		// The loop only runs once.
  1411  		for _, ap := range v.Block.Func.RegArgs {
  1412  			// Pass the spill/unspill information along to the assembler, offset by size of return PC pushed on stack.
  1413  			addr := ssagen.SpillSlotAddr(ap, x86.REG_SP, v.Block.Func.Config.PtrSize)
  1414  			reg := ap.Reg
  1415  			t := ap.Type
  1416  			sz := t.Size()
  1417  			if t.IsSIMD() {
  1418  				reg = simdRegBySize(reg, sz)
  1419  			}
  1420  			// These spills execute in the prologue, so only the
  1421  			// entry block's CPU features apply.
  1422  			avx := v.Block.Func.Entry.CPUfeatures.HasFeature(ssa.CPUavx)
  1423  			s.FuncInfo().AddSpill(
  1424  				obj.RegSpill{Reg: reg, Addr: addr, Unspill: loadByRegWidth(reg, sz, avx), Spill: storeByRegWidth(reg, sz, avx)})
  1425  		}
  1426  		v.Block.Func.RegArgs = nil
  1427  		ssagen.CheckArgReg(v)
  1428  	case ssaop.OpAMD64LoweredGetClosurePtr:
  1429  		// Closure pointer is DX.
  1430  		ssagen.CheckLoweredGetClosurePtr(v)
  1431  	case ssaop.OpAMD64LoweredGetG:
  1432  		if s.ABI == obj.ABIInternal {
  1433  			v.Fatalf("LoweredGetG should not appear in ABIInternal")
  1434  		}
  1435  		r := v.Reg()
  1436  		getgFromTLS(s, r)
  1437  	case ssaop.OpAMD64CALLstatic, ssaop.OpAMD64CALLtail, ssaop.OpAMD64CALLtailinter:
  1438  		if s.ABI == obj.ABI0 && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABIInternal {
  1439  			// zeroing X15 when entering ABIInternal from ABI0
  1440  			zeroX15(s)
  1441  			// set G register from TLS
  1442  			getgFromTLS(s, x86.REG_R14)
  1443  		}
  1444  		if v.Op == ssaop.OpAMD64CALLtail || v.Op == ssaop.OpAMD64CALLtailinter {
  1445  			s.TailCall(v)
  1446  			break
  1447  		}
  1448  		s.Call(v)
  1449  		if s.ABI == obj.ABIInternal && v.Aux.(*ssa.AuxCall).Fn.ABI() == obj.ABI0 {
  1450  			// zeroing X15 when entering ABIInternal from ABI0
  1451  			zeroX15(s)
  1452  			// set G register from TLS
  1453  			getgFromTLS(s, x86.REG_R14)
  1454  		}
  1455  	case ssaop.OpAMD64CALLclosure, ssaop.OpAMD64CALLinter:
  1456  		s.Call(v)
  1457  
  1458  	case ssaop.OpAMD64LoweredGetCallerPC:
  1459  		p := s.Prog(x86.AMOVQ)
  1460  		p.From.Type = obj.TYPE_MEM
  1461  		p.From.Offset = -8 // PC is stored 8 bytes below first parameter.
  1462  		p.From.Name = obj.NAME_PARAM
  1463  		p.To.Type = obj.TYPE_REG
  1464  		p.To.Reg = v.Reg()
  1465  
  1466  	case ssaop.OpAMD64LoweredGetCallerSP:
  1467  		// caller's SP is the address of the first arg
  1468  		mov := x86.AMOVQ
  1469  		if types.PtrSize == 4 {
  1470  			mov = x86.AMOVL
  1471  		}
  1472  		p := s.Prog(mov)
  1473  		p.From.Type = obj.TYPE_ADDR
  1474  		p.From.Offset = -base.Ctxt.Arch.FixedFrameSize // 0 on amd64, just to be consistent with other architectures
  1475  		p.From.Name = obj.NAME_PARAM
  1476  		p.To.Type = obj.TYPE_REG
  1477  		p.To.Reg = v.Reg()
  1478  
  1479  	case ssaop.OpAMD64LoweredWB:
  1480  		p := s.Prog(obj.ACALL)
  1481  		p.To.Type = obj.TYPE_MEM
  1482  		p.To.Name = obj.NAME_EXTERN
  1483  		// AuxInt encodes how many buffer entries we need.
  1484  		p.To.Sym = ir.Syms.GCWriteBarrier[v.AuxInt-1]
  1485  
  1486  	case ssaop.OpAMD64LoweredPanicBoundsRR, ssaop.OpAMD64LoweredPanicBoundsRC, ssaop.OpAMD64LoweredPanicBoundsCR, ssaop.OpAMD64LoweredPanicBoundsCC:
  1487  		// Compute the constant we put in the PCData entry for this call.
  1488  		code, signed := ssa.BoundsKind(v.AuxInt).Code()
  1489  		xIsReg := false
  1490  		yIsReg := false
  1491  		xVal := 0
  1492  		yVal := 0
  1493  		switch v.Op {
  1494  		case ssaop.OpAMD64LoweredPanicBoundsRR:
  1495  			xIsReg = true
  1496  			xVal = int(v.Args[0].Reg() - x86.REG_AX)
  1497  			yIsReg = true
  1498  			yVal = int(v.Args[1].Reg() - x86.REG_AX)
  1499  		case ssaop.OpAMD64LoweredPanicBoundsRC:
  1500  			xIsReg = true
  1501  			xVal = int(v.Args[0].Reg() - x86.REG_AX)
  1502  			c := v.Aux.(ssa.PanicBoundsC).C
  1503  			if c >= 0 && c <= abi.BoundsMaxConst {
  1504  				yVal = int(c)
  1505  			} else {
  1506  				// Move constant to a register
  1507  				yIsReg = true
  1508  				if yVal == xVal {
  1509  					yVal = 1
  1510  				}
  1511  				p := s.Prog(x86.AMOVQ)
  1512  				p.From.Type = obj.TYPE_CONST
  1513  				p.From.Offset = c
  1514  				p.To.Type = obj.TYPE_REG
  1515  				p.To.Reg = x86.REG_AX + int16(yVal)
  1516  			}
  1517  		case ssaop.OpAMD64LoweredPanicBoundsCR:
  1518  			yIsReg = true
  1519  			yVal = int(v.Args[0].Reg() - x86.REG_AX)
  1520  			c := v.Aux.(ssa.PanicBoundsC).C
  1521  			if c >= 0 && c <= abi.BoundsMaxConst {
  1522  				xVal = int(c)
  1523  			} else {
  1524  				// Move constant to a register
  1525  				xIsReg = true
  1526  				if xVal == yVal {
  1527  					xVal = 1
  1528  				}
  1529  				p := s.Prog(x86.AMOVQ)
  1530  				p.From.Type = obj.TYPE_CONST
  1531  				p.From.Offset = c
  1532  				p.To.Type = obj.TYPE_REG
  1533  				p.To.Reg = x86.REG_AX + int16(xVal)
  1534  			}
  1535  		case ssaop.OpAMD64LoweredPanicBoundsCC:
  1536  			c := v.Aux.(ssa.PanicBoundsCC).Cx
  1537  			if c >= 0 && c <= abi.BoundsMaxConst {
  1538  				xVal = int(c)
  1539  			} else {
  1540  				// Move constant to a register
  1541  				xIsReg = true
  1542  				p := s.Prog(x86.AMOVQ)
  1543  				p.From.Type = obj.TYPE_CONST
  1544  				p.From.Offset = c
  1545  				p.To.Type = obj.TYPE_REG
  1546  				p.To.Reg = x86.REG_AX + int16(xVal)
  1547  			}
  1548  			c = v.Aux.(ssa.PanicBoundsCC).Cy
  1549  			if c >= 0 && c <= abi.BoundsMaxConst {
  1550  				yVal = int(c)
  1551  			} else {
  1552  				// Move constant to a register
  1553  				yIsReg = true
  1554  				yVal = 1
  1555  				p := s.Prog(x86.AMOVQ)
  1556  				p.From.Type = obj.TYPE_CONST
  1557  				p.From.Offset = c
  1558  				p.To.Type = obj.TYPE_REG
  1559  				p.To.Reg = x86.REG_AX + int16(yVal)
  1560  			}
  1561  		}
  1562  		c := abi.BoundsEncode(code, signed, xIsReg, yIsReg, xVal, yVal)
  1563  
  1564  		p := s.Prog(obj.APCDATA)
  1565  		p.From.SetConst(abi.PCDATA_PanicBounds)
  1566  		p.To.SetConst(int64(c))
  1567  		p = s.Prog(obj.ACALL)
  1568  		p.To.Type = obj.TYPE_MEM
  1569  		p.To.Name = obj.NAME_EXTERN
  1570  		p.To.Sym = ir.Syms.PanicBounds
  1571  
  1572  	case ssaop.OpAMD64NEGQ, ssaop.OpAMD64NEGL,
  1573  		ssaop.OpAMD64BSWAPQ, ssaop.OpAMD64BSWAPL,
  1574  		ssaop.OpAMD64NOTQ, ssaop.OpAMD64NOTL:
  1575  		p := s.Prog(v.Op.Asm())
  1576  		p.To.Type = obj.TYPE_REG
  1577  		p.To.Reg = v.Reg()
  1578  
  1579  	case ssaop.OpAMD64NEGLflags:
  1580  		p := s.Prog(v.Op.Asm())
  1581  		p.To.Type = obj.TYPE_REG
  1582  		p.To.Reg = v.Reg0()
  1583  
  1584  	case ssaop.OpAMD64ADDQconstflags, ssaop.OpAMD64ADDLconstflags:
  1585  		p := s.Prog(v.Op.Asm())
  1586  		p.From.Type = obj.TYPE_CONST
  1587  		p.From.Offset = v.AuxInt
  1588  		// Note: the inc/dec instructions do not modify
  1589  		// the carry flag like add$1 / sub$1 do.
  1590  		// We currently never use the CF/OF flags from
  1591  		// these instructions, so that is ok.
  1592  		switch {
  1593  		case p.As == x86.AADDQ && p.From.Offset == 1:
  1594  			p.As = x86.AINCQ
  1595  			p.From.Type = obj.TYPE_NONE
  1596  		case p.As == x86.AADDQ && p.From.Offset == -1:
  1597  			p.As = x86.ADECQ
  1598  			p.From.Type = obj.TYPE_NONE
  1599  		case p.As == x86.AADDL && p.From.Offset == 1:
  1600  			p.As = x86.AINCL
  1601  			p.From.Type = obj.TYPE_NONE
  1602  		case p.As == x86.AADDL && p.From.Offset == -1:
  1603  			p.As = x86.ADECL
  1604  			p.From.Type = obj.TYPE_NONE
  1605  		}
  1606  		p.To.Type = obj.TYPE_REG
  1607  		p.To.Reg = v.Reg0()
  1608  
  1609  	case ssaop.OpAMD64BSFQinit, ssaop.OpAMD64BSRQinit:
  1610  		p := s.Prog(v.Op.Asm())
  1611  		p.From.Type = obj.TYPE_REG
  1612  		p.From.Reg = v.Args[1].Reg()
  1613  		p.To.Type = obj.TYPE_REG
  1614  		p.To.Reg = v.Reg()
  1615  	case ssaop.OpAMD64BSFQ, ssaop.OpAMD64BSRQ, ssaop.OpAMD64BSFL, ssaop.OpAMD64BSRL:
  1616  		if v.Args[0].Reg() != v.Reg() {
  1617  			// BSF/BSR depend on the previous destination value.
  1618  			// Clear the destination to break the dependency.
  1619  			opregreg(s, x86.AXORL, v.Reg(), v.Reg())
  1620  		}
  1621  		p := s.Prog(v.Op.Asm())
  1622  		p.From.Type = obj.TYPE_REG
  1623  		p.From.Reg = v.Args[0].Reg()
  1624  		p.To.Type = obj.TYPE_REG
  1625  		p.To.Reg = v.Reg()
  1626  	case ssaop.OpAMD64SQRTSD, ssaop.OpAMD64SQRTSS:
  1627  		p := s.Prog(v.Op.Asm())
  1628  		p.From.Type = obj.TYPE_REG
  1629  		p.From.Reg = v.Args[0].Reg()
  1630  		p.To.Type = obj.TYPE_REG
  1631  		p.To.Reg = v.Reg()
  1632  	case ssaop.OpAMD64LoweredRound32F, ssaop.OpAMD64LoweredRound64F:
  1633  		// input is already rounded
  1634  	case ssaop.OpAMD64ROUNDSD, ssaop.OpAMD64ROUNDSS:
  1635  		p := s.Prog(v.Op.Asm())
  1636  		val := v.AuxInt
  1637  		// 0 means math.RoundToEven, 1 Floor, 2 Ceil, 3 Trunc
  1638  		if val < 0 || val > 3 {
  1639  			v.Fatalf("Invalid rounding mode")
  1640  		}
  1641  		p.From.Offset = val
  1642  		p.From.Type = obj.TYPE_CONST
  1643  		p.AddRestSourceReg(v.Args[0].Reg())
  1644  		p.To.Type = obj.TYPE_REG
  1645  		p.To.Reg = v.Reg()
  1646  	case ssaop.OpAMD64POPCNTQ, ssaop.OpAMD64POPCNTL,
  1647  		ssaop.OpAMD64TZCNTQ, ssaop.OpAMD64TZCNTL,
  1648  		ssaop.OpAMD64LZCNTQ, ssaop.OpAMD64LZCNTL:
  1649  		if v.Args[0].Reg() != v.Reg() {
  1650  			// POPCNT/TZCNT/LZCNT have a false dependency on the destination register on Intel cpus.
  1651  			// TZCNT/LZCNT problem affects pre-Skylake models. See discussion at https://gcc.gnu.org/bugzilla/show_bug.cgi?id=62011#c7.
  1652  			// Xor register with itself to break the dependency.
  1653  			opregreg(s, x86.AXORL, v.Reg(), v.Reg())
  1654  		}
  1655  		p := s.Prog(v.Op.Asm())
  1656  		p.From.Type = obj.TYPE_REG
  1657  		p.From.Reg = v.Args[0].Reg()
  1658  		p.To.Type = obj.TYPE_REG
  1659  		p.To.Reg = v.Reg()
  1660  
  1661  	case ssaop.OpAMD64SETEQ, ssaop.OpAMD64SETNE,
  1662  		ssaop.OpAMD64SETL, ssaop.OpAMD64SETLE,
  1663  		ssaop.OpAMD64SETG, ssaop.OpAMD64SETGE,
  1664  		ssaop.OpAMD64SETGF, ssaop.OpAMD64SETGEF,
  1665  		ssaop.OpAMD64SETB, ssaop.OpAMD64SETBE,
  1666  		ssaop.OpAMD64SETORD, ssaop.OpAMD64SETNAN,
  1667  		ssaop.OpAMD64SETA, ssaop.OpAMD64SETAE,
  1668  		ssaop.OpAMD64SETO:
  1669  		p := s.Prog(v.Op.Asm())
  1670  		p.To.Type = obj.TYPE_REG
  1671  		p.To.Reg = v.Reg()
  1672  
  1673  	case ssaop.OpAMD64SETEQstore, ssaop.OpAMD64SETNEstore,
  1674  		ssaop.OpAMD64SETLstore, ssaop.OpAMD64SETLEstore,
  1675  		ssaop.OpAMD64SETGstore, ssaop.OpAMD64SETGEstore,
  1676  		ssaop.OpAMD64SETBstore, ssaop.OpAMD64SETBEstore,
  1677  		ssaop.OpAMD64SETAstore, ssaop.OpAMD64SETAEstore:
  1678  		p := s.Prog(v.Op.Asm())
  1679  		p.To.Type = obj.TYPE_MEM
  1680  		p.To.Reg = v.Args[0].Reg()
  1681  		ssagen.AddAux(&p.To, v)
  1682  
  1683  	case ssaop.OpAMD64SETEQstoreidx1, ssaop.OpAMD64SETNEstoreidx1,
  1684  		ssaop.OpAMD64SETLstoreidx1, ssaop.OpAMD64SETLEstoreidx1,
  1685  		ssaop.OpAMD64SETGstoreidx1, ssaop.OpAMD64SETGEstoreidx1,
  1686  		ssaop.OpAMD64SETBstoreidx1, ssaop.OpAMD64SETBEstoreidx1,
  1687  		ssaop.OpAMD64SETAstoreidx1, ssaop.OpAMD64SETAEstoreidx1:
  1688  		p := s.Prog(v.Op.Asm())
  1689  		memIdx(&p.To, v)
  1690  		ssagen.AddAux(&p.To, v)
  1691  
  1692  	case ssaop.OpAMD64SETNEF:
  1693  		t := v.RegTmp()
  1694  		p := s.Prog(v.Op.Asm())
  1695  		p.To.Type = obj.TYPE_REG
  1696  		p.To.Reg = v.Reg()
  1697  		q := s.Prog(x86.ASETPS)
  1698  		q.To.Type = obj.TYPE_REG
  1699  		q.To.Reg = t
  1700  		// ORL avoids partial register write and is smaller than ORQ, used by old compiler
  1701  		opregreg(s, x86.AORL, v.Reg(), t)
  1702  
  1703  	case ssaop.OpAMD64SETEQF:
  1704  		t := v.RegTmp()
  1705  		p := s.Prog(v.Op.Asm())
  1706  		p.To.Type = obj.TYPE_REG
  1707  		p.To.Reg = v.Reg()
  1708  		q := s.Prog(x86.ASETPC)
  1709  		q.To.Type = obj.TYPE_REG
  1710  		q.To.Reg = t
  1711  		// ANDL avoids partial register write and is smaller than ANDQ, used by old compiler
  1712  		opregreg(s, x86.AANDL, v.Reg(), t)
  1713  
  1714  	case ssaop.OpAMD64InvertFlags:
  1715  		v.Fatalf("InvertFlags should never make it to codegen %v", v.LongString())
  1716  	case ssaop.OpAMD64FlagEQ, ssaop.OpAMD64FlagLT_ULT, ssaop.OpAMD64FlagLT_UGT, ssaop.OpAMD64FlagGT_ULT, ssaop.OpAMD64FlagGT_UGT:
  1717  		v.Fatalf("Flag* ops should never make it to codegen %v", v.LongString())
  1718  	case ssaop.OpAMD64AddTupleFirst32, ssaop.OpAMD64AddTupleFirst64:
  1719  		v.Fatalf("AddTupleFirst* should never make it to codegen %v", v.LongString())
  1720  	case ssaop.OpAMD64REPSTOSQ:
  1721  		s.Prog(x86.AREP)
  1722  		s.Prog(x86.ASTOSQ)
  1723  	case ssaop.OpAMD64REPMOVSQ:
  1724  		s.Prog(x86.AREP)
  1725  		s.Prog(x86.AMOVSQ)
  1726  	case ssaop.OpAMD64LoweredNilCheck:
  1727  		// Issue a load which will fault if the input is nil.
  1728  		// TODO: We currently use the 2-byte instruction TESTB AX, (reg).
  1729  		// Should we use the 3-byte TESTB $0, (reg) instead? It is larger
  1730  		// but it doesn't have false dependency on AX.
  1731  		// Or maybe allocate an output register and use MOVL (reg),reg2 ?
  1732  		// That trades clobbering flags for clobbering a register.
  1733  		p := s.Prog(x86.ATESTB)
  1734  		p.From.Type = obj.TYPE_REG
  1735  		p.From.Reg = x86.REG_AX
  1736  		p.To.Type = obj.TYPE_MEM
  1737  		p.To.Reg = v.Args[0].Reg()
  1738  		if logopt.Enabled() {
  1739  			logopt.LogOpt(v.Pos, "nilcheck", "genssa", v.Block.Func.Name)
  1740  		}
  1741  		if base.Debug.Nil != 0 && v.Pos.Line() > 1 { // v.Pos.Line()==1 in generated wrappers
  1742  			base.WarnfAt(v.Pos, "generated nil check")
  1743  		}
  1744  	case ssaop.OpAMD64MOVBatomicload, ssaop.OpAMD64MOVLatomicload, ssaop.OpAMD64MOVQatomicload:
  1745  		p := s.Prog(v.Op.Asm())
  1746  		p.From.Type = obj.TYPE_MEM
  1747  		p.From.Reg = v.Args[0].Reg()
  1748  		ssagen.AddAux(&p.From, v)
  1749  		p.To.Type = obj.TYPE_REG
  1750  		p.To.Reg = v.Reg0()
  1751  	case ssaop.OpAMD64XCHGB, ssaop.OpAMD64XCHGL, ssaop.OpAMD64XCHGQ:
  1752  		p := s.Prog(v.Op.Asm())
  1753  		p.From.Type = obj.TYPE_REG
  1754  		p.From.Reg = v.Reg0()
  1755  		p.To.Type = obj.TYPE_MEM
  1756  		p.To.Reg = v.Args[1].Reg()
  1757  		ssagen.AddAux(&p.To, v)
  1758  	case ssaop.OpAMD64XADDLlock, ssaop.OpAMD64XADDQlock:
  1759  		s.Prog(x86.ALOCK)
  1760  		p := s.Prog(v.Op.Asm())
  1761  		p.From.Type = obj.TYPE_REG
  1762  		p.From.Reg = v.Reg0()
  1763  		p.To.Type = obj.TYPE_MEM
  1764  		p.To.Reg = v.Args[1].Reg()
  1765  		ssagen.AddAux(&p.To, v)
  1766  	case ssaop.OpAMD64CMPXCHGLlock, ssaop.OpAMD64CMPXCHGQlock:
  1767  		if v.Args[1].Reg() != x86.REG_AX {
  1768  			v.Fatalf("input[1] not in AX %s", v.LongString())
  1769  		}
  1770  		s.Prog(x86.ALOCK)
  1771  		p := s.Prog(v.Op.Asm())
  1772  		p.From.Type = obj.TYPE_REG
  1773  		p.From.Reg = v.Args[2].Reg()
  1774  		p.To.Type = obj.TYPE_MEM
  1775  		p.To.Reg = v.Args[0].Reg()
  1776  		ssagen.AddAux(&p.To, v)
  1777  		p = s.Prog(x86.ASETEQ)
  1778  		p.To.Type = obj.TYPE_REG
  1779  		p.To.Reg = v.Reg0()
  1780  	case ssaop.OpAMD64ANDBlock, ssaop.OpAMD64ANDLlock, ssaop.OpAMD64ANDQlock,
  1781  		ssaop.OpAMD64ORBlock, ssaop.OpAMD64ORLlock, ssaop.OpAMD64ORQlock,
  1782  		ssaop.OpAMD64ADDLlock, ssaop.OpAMD64ADDQlock,
  1783  		ssaop.OpAMD64SUBLlock, ssaop.OpAMD64SUBQlock:
  1784  		// Atomic memory operations that don't need to return the old value.
  1785  		s.Prog(x86.ALOCK)
  1786  		p := s.Prog(v.Op.Asm())
  1787  		p.From.Type = obj.TYPE_REG
  1788  		p.From.Reg = v.Args[1].Reg()
  1789  		p.To.Type = obj.TYPE_MEM
  1790  		p.To.Reg = v.Args[0].Reg()
  1791  		ssagen.AddAux(&p.To, v)
  1792  	case ssaop.OpAMD64INCLlock, ssaop.OpAMD64INCQlock,
  1793  		ssaop.OpAMD64DECLlock, ssaop.OpAMD64DECQlock:
  1794  		// Unary atomic memory operations that don't need to return the old value.
  1795  		s.Prog(x86.ALOCK)
  1796  		p := s.Prog(v.Op.Asm())
  1797  		p.To.Type = obj.TYPE_MEM
  1798  		p.To.Reg = v.Args[0].Reg()
  1799  		ssagen.AddAux(&p.To, v)
  1800  	case ssaop.OpAMD64LoweredAtomicAnd64, ssaop.OpAMD64LoweredAtomicOr64, ssaop.OpAMD64LoweredAtomicAnd32, ssaop.OpAMD64LoweredAtomicOr32:
  1801  		// Atomic memory operations that need to return the old value.
  1802  		// We need to do these with compare-and-exchange to get access to the old value.
  1803  		// loop:
  1804  		// MOVQ mask, tmp
  1805  		// MOVQ (addr), AX
  1806  		// ANDQ AX, tmp
  1807  		// LOCK CMPXCHGQ tmp, (addr) : note that AX is implicit old value to compare against
  1808  		// JNE loop
  1809  		// : result in AX
  1810  		//
  1811  		// If the width written to AX changes, update zeroUpperBits in AMD64Ops.go.
  1812  		mov := x86.AMOVQ
  1813  		op := x86.AANDQ
  1814  		cmpxchg := x86.ACMPXCHGQ
  1815  		switch v.Op {
  1816  		case ssaop.OpAMD64LoweredAtomicOr64:
  1817  			op = x86.AORQ
  1818  		case ssaop.OpAMD64LoweredAtomicAnd32:
  1819  			mov = x86.AMOVL
  1820  			op = x86.AANDL
  1821  			cmpxchg = x86.ACMPXCHGL
  1822  		case ssaop.OpAMD64LoweredAtomicOr32:
  1823  			mov = x86.AMOVL
  1824  			op = x86.AORL
  1825  			cmpxchg = x86.ACMPXCHGL
  1826  		}
  1827  		addr := v.Args[0].Reg()
  1828  		mask := v.Args[1].Reg()
  1829  		tmp := v.RegTmp()
  1830  		p1 := s.Prog(mov)
  1831  		p1.From.Type = obj.TYPE_REG
  1832  		p1.From.Reg = mask
  1833  		p1.To.Type = obj.TYPE_REG
  1834  		p1.To.Reg = tmp
  1835  		p2 := s.Prog(mov)
  1836  		p2.From.Type = obj.TYPE_MEM
  1837  		p2.From.Reg = addr
  1838  		ssagen.AddAux(&p2.From, v)
  1839  		p2.To.Type = obj.TYPE_REG
  1840  		p2.To.Reg = x86.REG_AX
  1841  		p3 := s.Prog(op)
  1842  		p3.From.Type = obj.TYPE_REG
  1843  		p3.From.Reg = x86.REG_AX
  1844  		p3.To.Type = obj.TYPE_REG
  1845  		p3.To.Reg = tmp
  1846  		s.Prog(x86.ALOCK)
  1847  		p5 := s.Prog(cmpxchg)
  1848  		p5.From.Type = obj.TYPE_REG
  1849  		p5.From.Reg = tmp
  1850  		p5.To.Type = obj.TYPE_MEM
  1851  		p5.To.Reg = addr
  1852  		ssagen.AddAux(&p5.To, v)
  1853  		p6 := s.Prog(x86.AJNE)
  1854  		p6.To.Type = obj.TYPE_BRANCH
  1855  		p6.To.SetTarget(p1)
  1856  	case ssaop.OpAMD64PrefetchT0, ssaop.OpAMD64PrefetchNTA:
  1857  		p := s.Prog(v.Op.Asm())
  1858  		p.From.Type = obj.TYPE_MEM
  1859  		p.From.Reg = v.Args[0].Reg()
  1860  	case ssaop.OpClobber:
  1861  		p := s.Prog(x86.AMOVL)
  1862  		p.From.Type = obj.TYPE_CONST
  1863  		p.From.Offset = 0xdeaddead
  1864  		p.To.Type = obj.TYPE_MEM
  1865  		p.To.Reg = x86.REG_SP
  1866  		ssagen.AddAux(&p.To, v)
  1867  		p = s.Prog(x86.AMOVL)
  1868  		p.From.Type = obj.TYPE_CONST
  1869  		p.From.Offset = 0xdeaddead
  1870  		p.To.Type = obj.TYPE_MEM
  1871  		p.To.Reg = x86.REG_SP
  1872  		ssagen.AddAux(&p.To, v)
  1873  		p.To.Offset += 4
  1874  	case ssaop.OpClobberReg:
  1875  		x := uint64(0xdeaddeaddeaddead)
  1876  		p := s.Prog(x86.AMOVQ)
  1877  		p.From.Type = obj.TYPE_CONST
  1878  		p.From.Offset = int64(x)
  1879  		p.To.Type = obj.TYPE_REG
  1880  		p.To.Reg = v.Reg()
  1881  
  1882  	// SIMD ops
  1883  	case ssaop.OpAMD64VZEROUPPER, ssaop.OpAMD64VZEROALL:
  1884  		s.Prog(v.Op.Asm())
  1885  
  1886  	case ssaop.OpAMD64Zero128, ssaop.OpAMD64Zero256, ssaop.OpAMD64Zero512: // no code emitted
  1887  
  1888  	case ssaop.OpAMD64VMOVSSf2v, ssaop.OpAMD64VMOVSDf2v:
  1889  		// These are for initializing the least 32/64 bits of a SIMD register from a "float".
  1890  		p := s.Prog(v.Op.Asm())
  1891  		p.From.Type = obj.TYPE_REG
  1892  		p.From.Reg = v.Args[0].Reg()
  1893  		p.AddRestSourceReg(x86.REG_X15)
  1894  		p.To.Type = obj.TYPE_REG
  1895  		p.To.Reg = simdReg(v)
  1896  
  1897  	case ssaop.OpAMD64VMOVQload, ssaop.OpAMD64VMOVDload,
  1898  		ssaop.OpAMD64VMOVSSload, ssaop.OpAMD64VMOVSDload:
  1899  		p := s.Prog(v.Op.Asm())
  1900  		p.From.Type = obj.TYPE_MEM
  1901  		p.From.Reg = v.Args[0].Reg()
  1902  		ssagen.AddAux(&p.From, v)
  1903  		p.To.Type = obj.TYPE_REG
  1904  		p.To.Reg = simdReg(v)
  1905  
  1906  	case ssaop.OpAMD64VMOVSSconst, ssaop.OpAMD64VMOVSDconst:
  1907  		// for loading constants directly into SIMD registers
  1908  		x := simdReg(v)
  1909  		p := s.Prog(v.Op.Asm())
  1910  		p.From.Type = obj.TYPE_FCONST
  1911  		p.From.Val = math.Float64frombits(uint64(v.AuxInt))
  1912  		p.To.Type = obj.TYPE_REG
  1913  		p.To.Reg = x
  1914  
  1915  	case ssaop.OpAMD64VMOVD, ssaop.OpAMD64VMOVQ:
  1916  		// These are for initializing the least 32/64 bits of a SIMD register from an "int".
  1917  		p := s.Prog(v.Op.Asm())
  1918  		p.From.Type = obj.TYPE_REG
  1919  		p.From.Reg = v.Args[0].Reg()
  1920  		p.To.Type = obj.TYPE_REG
  1921  		p.To.Reg = simdReg(v)
  1922  
  1923  	case ssaop.OpAMD64VMOVDQUload128, ssaop.OpAMD64VMOVDQUload256, ssaop.OpAMD64VMOVDQUload512,
  1924  		ssaop.OpAMD64KMOVBload, ssaop.OpAMD64KMOVWload, ssaop.OpAMD64KMOVDload, ssaop.OpAMD64KMOVQload:
  1925  		p := s.Prog(v.Op.Asm())
  1926  		p.From.Type = obj.TYPE_MEM
  1927  		p.From.Reg = v.Args[0].Reg()
  1928  		ssagen.AddAux(&p.From, v)
  1929  		p.To.Type = obj.TYPE_REG
  1930  		p.To.Reg = simdOrMaskReg(v)
  1931  	case ssaop.OpAMD64VMOVDQUstore128, ssaop.OpAMD64VMOVDQUstore256, ssaop.OpAMD64VMOVDQUstore512,
  1932  		ssaop.OpAMD64KMOVBstore, ssaop.OpAMD64KMOVWstore, ssaop.OpAMD64KMOVDstore, ssaop.OpAMD64KMOVQstore:
  1933  		p := s.Prog(v.Op.Asm())
  1934  		p.From.Type = obj.TYPE_REG
  1935  		p.From.Reg = simdOrMaskReg(v.Args[1])
  1936  		p.To.Type = obj.TYPE_MEM
  1937  		p.To.Reg = v.Args[0].Reg()
  1938  		ssagen.AddAux(&p.To, v)
  1939  
  1940  	case ssaop.OpAMD64VPMASK32load128, ssaop.OpAMD64VPMASK64load128, ssaop.OpAMD64VPMASK32load256, ssaop.OpAMD64VPMASK64load256:
  1941  		p := s.Prog(v.Op.Asm())
  1942  		p.From.Type = obj.TYPE_MEM
  1943  		p.From.Reg = v.Args[0].Reg()
  1944  		ssagen.AddAux(&p.From, v)
  1945  		p.To.Type = obj.TYPE_REG
  1946  		p.To.Reg = simdReg(v)
  1947  		p.AddRestSourceReg(simdReg(v.Args[1])) // masking simd reg
  1948  
  1949  	case ssaop.OpAMD64VPMASK32store128, ssaop.OpAMD64VPMASK64store128, ssaop.OpAMD64VPMASK32store256, ssaop.OpAMD64VPMASK64store256:
  1950  		p := s.Prog(v.Op.Asm())
  1951  		p.From.Type = obj.TYPE_REG
  1952  		p.From.Reg = simdReg(v.Args[2])
  1953  		p.To.Type = obj.TYPE_MEM
  1954  		p.To.Reg = v.Args[0].Reg()
  1955  		ssagen.AddAux(&p.To, v)
  1956  		p.AddRestSourceReg(simdReg(v.Args[1])) // masking simd reg
  1957  
  1958  	case ssaop.OpAMD64VPMASK64load512, ssaop.OpAMD64VPMASK32load512, ssaop.OpAMD64VPMASK16load512, ssaop.OpAMD64VPMASK8load512:
  1959  		p := s.Prog(v.Op.Asm())
  1960  		p.From.Type = obj.TYPE_MEM
  1961  		p.From.Reg = v.Args[0].Reg()
  1962  		ssagen.AddAux(&p.From, v)
  1963  		p.To.Type = obj.TYPE_REG
  1964  		p.To.Reg = simdReg(v)
  1965  		p.AddRestSourceReg(v.Args[1].Reg()) // simd mask reg
  1966  		x86.ParseSuffix(p, "Z")             // must be zero if not in mask
  1967  
  1968  	case ssaop.OpAMD64KANDB, ssaop.OpAMD64KANDW, ssaop.OpAMD64KANDD, ssaop.OpAMD64KANDQ,
  1969  		ssaop.OpAMD64KANDNB, ssaop.OpAMD64KANDNW, ssaop.OpAMD64KANDND, ssaop.OpAMD64KANDNQ,
  1970  		ssaop.OpAMD64KORB, ssaop.OpAMD64KORW, ssaop.OpAMD64KORD, ssaop.OpAMD64KORQ,
  1971  		ssaop.OpAMD64KXORB, ssaop.OpAMD64KXORW, ssaop.OpAMD64KXORD, ssaop.OpAMD64KXORQ,
  1972  		ssaop.OpAMD64KXNORB, ssaop.OpAMD64KXNORW, ssaop.OpAMD64KXNORD, ssaop.OpAMD64KXNORQ: // XNOR == EQ
  1973  		p := s.Prog(v.Op.Asm())
  1974  		p.From.Type = obj.TYPE_REG
  1975  		p.From.Reg = v.Args[0].Reg()
  1976  		p.To.Type = obj.TYPE_REG
  1977  		p.To.Reg = v.Reg()
  1978  		p.AddRestSourceReg(v.Args[1].Reg()) // masking simd reg
  1979  
  1980  	case ssaop.OpAMD64VPMASK64store512, ssaop.OpAMD64VPMASK32store512, ssaop.OpAMD64VPMASK16store512, ssaop.OpAMD64VPMASK8store512:
  1981  		p := s.Prog(v.Op.Asm())
  1982  		p.From.Type = obj.TYPE_REG
  1983  		p.From.Reg = simdReg(v.Args[2])
  1984  		p.To.Type = obj.TYPE_MEM
  1985  		p.To.Reg = v.Args[0].Reg()
  1986  		ssagen.AddAux(&p.To, v)
  1987  		p.AddRestSourceReg(v.Args[1].Reg()) // simd mask reg
  1988  
  1989  	case ssaop.OpAMD64VPMOVMToVec8x16,
  1990  		ssaop.OpAMD64VPMOVMToVec8x32,
  1991  		ssaop.OpAMD64VPMOVMToVec8x64,
  1992  		ssaop.OpAMD64VPMOVMToVec16x8,
  1993  		ssaop.OpAMD64VPMOVMToVec16x16,
  1994  		ssaop.OpAMD64VPMOVMToVec16x32,
  1995  		ssaop.OpAMD64VPMOVMToVec32x4,
  1996  		ssaop.OpAMD64VPMOVMToVec32x8,
  1997  		ssaop.OpAMD64VPMOVMToVec32x16,
  1998  		ssaop.OpAMD64VPMOVMToVec64x2,
  1999  		ssaop.OpAMD64VPMOVMToVec64x4,
  2000  		ssaop.OpAMD64VPMOVMToVec64x8:
  2001  		p := s.Prog(v.Op.Asm())
  2002  		p.From.Type = obj.TYPE_REG
  2003  		p.From.Reg = v.Args[0].Reg()
  2004  		p.To.Type = obj.TYPE_REG
  2005  		p.To.Reg = simdReg(v)
  2006  
  2007  	case ssaop.OpAMD64VPMOVVec8x16ToM,
  2008  		ssaop.OpAMD64VPMOVVec8x32ToM,
  2009  		ssaop.OpAMD64VPMOVVec8x64ToM,
  2010  		ssaop.OpAMD64VPMOVVec16x8ToM,
  2011  		ssaop.OpAMD64VPMOVVec16x16ToM,
  2012  		ssaop.OpAMD64VPMOVVec16x32ToM,
  2013  		ssaop.OpAMD64VPMOVVec32x4ToM,
  2014  		ssaop.OpAMD64VPMOVVec32x8ToM,
  2015  		ssaop.OpAMD64VPMOVVec32x16ToM,
  2016  		ssaop.OpAMD64VPMOVVec64x2ToM,
  2017  		ssaop.OpAMD64VPMOVVec64x4ToM,
  2018  		ssaop.OpAMD64VPMOVVec64x8ToM,
  2019  		ssaop.OpAMD64VPMOVMSKB128,
  2020  		ssaop.OpAMD64VPMOVMSKB256,
  2021  		ssaop.OpAMD64VMOVMSKPS128,
  2022  		ssaop.OpAMD64VMOVMSKPS256,
  2023  		ssaop.OpAMD64VMOVMSKPD128,
  2024  		ssaop.OpAMD64VMOVMSKPD256:
  2025  		p := s.Prog(v.Op.Asm())
  2026  		p.From.Type = obj.TYPE_REG
  2027  		p.From.Reg = simdReg(v.Args[0])
  2028  		p.To.Type = obj.TYPE_REG
  2029  		p.To.Reg = v.Reg()
  2030  
  2031  	case ssaop.OpAMD64KMOVQk, ssaop.OpAMD64KMOVDk, ssaop.OpAMD64KMOVWk, ssaop.OpAMD64KMOVBk,
  2032  		ssaop.OpAMD64KMOVQi, ssaop.OpAMD64KMOVDi, ssaop.OpAMD64KMOVWi, ssaop.OpAMD64KMOVBi,
  2033  		ssaop.OpAMD64KNOTB, ssaop.OpAMD64KNOTW, ssaop.OpAMD64KNOTD, ssaop.OpAMD64KNOTQ:
  2034  		// See also ssa.OpAMD64KMOVQload
  2035  		p := s.Prog(v.Op.Asm())
  2036  		p.From.Type = obj.TYPE_REG
  2037  		p.From.Reg = v.Args[0].Reg()
  2038  		p.To.Type = obj.TYPE_REG
  2039  		p.To.Reg = v.Reg()
  2040  	case ssaop.OpAMD64VPTEST:
  2041  		// Some instructions setting flags put their second operand into the destination reg.
  2042  		// See also CMP[BWDQ].
  2043  		p := s.Prog(v.Op.Asm())
  2044  		p.From.Type = obj.TYPE_REG
  2045  		p.From.Reg = simdReg(v.Args[0])
  2046  		p.To.Type = obj.TYPE_REG
  2047  		p.To.Reg = simdReg(v.Args[1])
  2048  
  2049  	default:
  2050  		if !ssaGenSIMDValue(s, v) {
  2051  			v.Fatalf("genValue not implemented: %s", v.LongString())
  2052  		}
  2053  	}
  2054  }
  2055  
  2056  // vxorpsX15 zeroes the whole X15 register (including the high bits)
  2057  // using a VEX encoding. AVX must be present.
  2058  func vxorpsX15(s *ssagen.State) {
  2059  	p := s.Prog(x86.AVXORPS)
  2060  	p.From.Type = obj.TYPE_REG
  2061  	p.From.Reg = x86.REG_X15
  2062  	p.AddRestSourceReg(x86.REG_X15)
  2063  	p.To.Type = obj.TYPE_REG
  2064  	p.To.Reg = x86.REG_X15
  2065  }
  2066  
  2067  // zeroX15Low zeroes the low 16 bytes of the X15 register, using a VEX
  2068  // encoding if avx says AVX is known to be present.
  2069  func zeroX15Low(s *ssagen.State, avx bool) {
  2070  	if avx {
  2071  		vxorpsX15(s)
  2072  		return
  2073  	}
  2074  	opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
  2075  }
  2076  
  2077  // zeroX15 zeroes the X15 register.
  2078  func zeroX15(s *ssagen.State) {
  2079  	if buildcfg.GOAMD64 >= 3 {
  2080  		vxorpsX15(s)
  2081  		return
  2082  	}
  2083  	opregreg(s, x86.AXORPS, x86.REG_X15, x86.REG_X15)
  2084  	// AVX may not be available, check before zeroing the high bits.
  2085  	p := s.Prog(x86.ACMPB)
  2086  	p.From.Type = obj.TYPE_MEM
  2087  	p.From.Name = obj.NAME_EXTERN
  2088  	p.From.Sym = ir.Syms.X86HasAVX
  2089  	p.To.Type = obj.TYPE_CONST
  2090  	p.To.Offset = 1
  2091  	jmp := s.Prog(x86.AJNE)
  2092  	jmp.To.Type = obj.TYPE_BRANCH
  2093  	vxorpsX15(s)
  2094  	end := s.Prog(obj.ANOP)
  2095  	jmp.To.SetTarget(end)
  2096  }
  2097  
  2098  // Example instruction: VRSQRTPS X1, X1
  2099  func simdV11(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2100  	p := s.Prog(v.Op.Asm())
  2101  	p.From.Type = obj.TYPE_REG
  2102  	p.From.Reg = simdReg(v.Args[0])
  2103  	p.To.Type = obj.TYPE_REG
  2104  	p.To.Reg = simdReg(v)
  2105  	return p
  2106  }
  2107  
  2108  // Example instruction: VPSUBD X1, X2, X3
  2109  func simdV21(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2110  	p := s.Prog(v.Op.Asm())
  2111  	p.From.Type = obj.TYPE_REG
  2112  	// Vector registers operands follows a right-to-left order.
  2113  	// e.g. VPSUBD X1, X2, X3 means X3 = X2 - X1.
  2114  	p.From.Reg = simdReg(v.Args[1])
  2115  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2116  	p.To.Type = obj.TYPE_REG
  2117  	p.To.Reg = simdReg(v)
  2118  	return p
  2119  }
  2120  
  2121  // This function is to accustomize the shifts.
  2122  // The 2nd arg is an XMM, and this function merely checks that.
  2123  // Example instruction: VPSLLQ Z1, X1, Z2
  2124  func simdVfpv(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2125  	p := s.Prog(v.Op.Asm())
  2126  	p.From.Type = obj.TYPE_REG
  2127  	// Vector registers operands follows a right-to-left order.
  2128  	// e.g. VPSUBD X1, X2, X3 means X3 = X2 - X1.
  2129  	p.From.Reg = v.Args[1].Reg()
  2130  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2131  	p.To.Type = obj.TYPE_REG
  2132  	p.To.Reg = simdReg(v)
  2133  	return p
  2134  }
  2135  
  2136  // Example instruction: VPCMPEQW Z26, Z30, K4
  2137  func simdV2k(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2138  	p := s.Prog(v.Op.Asm())
  2139  	p.From.Type = obj.TYPE_REG
  2140  	p.From.Reg = simdReg(v.Args[1])
  2141  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2142  	p.To.Type = obj.TYPE_REG
  2143  	p.To.Reg = maskReg(v)
  2144  	return p
  2145  }
  2146  
  2147  // Example instruction: VPMINUQ X21, X3, K3, X31
  2148  func simdV2kv(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2149  	p := s.Prog(v.Op.Asm())
  2150  	p.From.Type = obj.TYPE_REG
  2151  	p.From.Reg = simdReg(v.Args[1])
  2152  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2153  	// These "simd*" series of functions assumes:
  2154  	// Any "K" register that serves as the write-mask
  2155  	// or "predicate" for "predicated AVX512 instructions"
  2156  	// sits right at the end of the operand list.
  2157  	// TODO: verify this assumption.
  2158  	p.AddRestSourceReg(maskReg(v.Args[2]))
  2159  	p.To.Type = obj.TYPE_REG
  2160  	p.To.Reg = simdReg(v)
  2161  	return p
  2162  }
  2163  
  2164  // Example instruction: VPABSB X1, X2, K3 (masking merging)
  2165  func simdV2kvResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2166  	p := s.Prog(v.Op.Asm())
  2167  	p.From.Type = obj.TYPE_REG
  2168  	p.From.Reg = simdReg(v.Args[1])
  2169  	// These "simd*" series of functions assumes:
  2170  	// Any "K" register that serves as the write-mask
  2171  	// or "predicate" for "predicated AVX512 instructions"
  2172  	// sits right at the end of the operand list.
  2173  	// TODO: verify this assumption.
  2174  	p.AddRestSourceReg(maskReg(v.Args[2]))
  2175  	p.To.Type = obj.TYPE_REG
  2176  	p.To.Reg = simdReg(v)
  2177  	return p
  2178  }
  2179  
  2180  // This function is to accustomize the shifts.
  2181  // The 2nd arg is an XMM, and this function merely checks that.
  2182  // Example instruction: VPSLLQ Z1, X1, K1, Z2
  2183  func simdVfpkv(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2184  	p := s.Prog(v.Op.Asm())
  2185  	p.From.Type = obj.TYPE_REG
  2186  	p.From.Reg = v.Args[1].Reg()
  2187  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2188  	p.AddRestSourceReg(maskReg(v.Args[2]))
  2189  	p.To.Type = obj.TYPE_REG
  2190  	p.To.Reg = simdReg(v)
  2191  	return p
  2192  }
  2193  
  2194  // Example instruction: VPCMPEQW Z26, Z30, K1, K4
  2195  func simdV2kk(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2196  	p := s.Prog(v.Op.Asm())
  2197  	p.From.Type = obj.TYPE_REG
  2198  	p.From.Reg = simdReg(v.Args[1])
  2199  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2200  	p.AddRestSourceReg(maskReg(v.Args[2]))
  2201  	p.To.Type = obj.TYPE_REG
  2202  	p.To.Reg = maskReg(v)
  2203  	return p
  2204  }
  2205  
  2206  // Example instruction: VPOPCNTB X14, K4, X16
  2207  func simdVkv(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2208  	p := s.Prog(v.Op.Asm())
  2209  	p.From.Type = obj.TYPE_REG
  2210  	p.From.Reg = simdReg(v.Args[0])
  2211  	p.AddRestSourceReg(maskReg(v.Args[1]))
  2212  	p.To.Type = obj.TYPE_REG
  2213  	p.To.Reg = simdReg(v)
  2214  	return p
  2215  }
  2216  
  2217  // Example instruction: VROUNDPD $7, X2, X2
  2218  func simdV11Imm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2219  	p := s.Prog(v.Op.Asm())
  2220  	p.From.Offset = int64(v.AuxUInt8())
  2221  	p.From.Type = obj.TYPE_CONST
  2222  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2223  	p.To.Type = obj.TYPE_REG
  2224  	p.To.Reg = simdReg(v)
  2225  	return p
  2226  }
  2227  
  2228  // Example instruction: VREDUCEPD $126, X1, K3, X31
  2229  func simdVkvImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2230  	p := s.Prog(v.Op.Asm())
  2231  	p.From.Offset = int64(v.AuxUInt8())
  2232  	p.From.Type = obj.TYPE_CONST
  2233  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2234  	p.AddRestSourceReg(maskReg(v.Args[1]))
  2235  	p.To.Type = obj.TYPE_REG
  2236  	p.To.Reg = simdReg(v)
  2237  	return p
  2238  }
  2239  
  2240  // Example instruction: VCMPPS $7, X2, X9, X2
  2241  func simdV21Imm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2242  	p := s.Prog(v.Op.Asm())
  2243  	p.From.Offset = int64(v.AuxUInt8())
  2244  	p.From.Type = obj.TYPE_CONST
  2245  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2246  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2247  	p.To.Type = obj.TYPE_REG
  2248  	p.To.Reg = simdReg(v)
  2249  	return p
  2250  }
  2251  
  2252  // Example instruction: VPINSRB $3, DX, X0, X0
  2253  func simdVgpvImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2254  	p := s.Prog(v.Op.Asm())
  2255  	p.From.Offset = int64(v.AuxUInt8())
  2256  	p.From.Type = obj.TYPE_CONST
  2257  	p.AddRestSourceReg(v.Args[1].Reg())
  2258  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2259  	p.To.Type = obj.TYPE_REG
  2260  	p.To.Reg = simdReg(v)
  2261  	return p
  2262  }
  2263  func simdVgpvImm(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2264  	// within simdgen, the choice of intrinsic shape and the output
  2265  	// instruction format are linked.  In the case of VgpImm, there is
  2266  	// a difference in the intrinsic, but no difference in the
  2267  	// instruction, it is just like VgpvImm8.
  2268  	//
  2269  	// See also, simdVgpImm and simdVgpImm8
  2270  	return simdVgpvImm8(s, v)
  2271  }
  2272  
  2273  // Example instruction: VPCMPD $1, Z1, Z2, K1
  2274  func simdV2kImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2275  	p := s.Prog(v.Op.Asm())
  2276  	p.From.Offset = int64(v.AuxUInt8())
  2277  	p.From.Type = obj.TYPE_CONST
  2278  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2279  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2280  	p.To.Type = obj.TYPE_REG
  2281  	p.To.Reg = maskReg(v)
  2282  	return p
  2283  }
  2284  
  2285  // Example instruction: VPCMPD $1, Z1, Z2, K2, K1
  2286  func simdV2kkImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2287  	p := s.Prog(v.Op.Asm())
  2288  	p.From.Offset = int64(v.AuxUInt8())
  2289  	p.From.Type = obj.TYPE_CONST
  2290  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2291  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2292  	p.AddRestSourceReg(maskReg(v.Args[2]))
  2293  	p.To.Type = obj.TYPE_REG
  2294  	p.To.Reg = maskReg(v)
  2295  	return p
  2296  }
  2297  
  2298  func simdV2kvImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2299  	p := s.Prog(v.Op.Asm())
  2300  	p.From.Offset = int64(v.AuxUInt8())
  2301  	p.From.Type = obj.TYPE_CONST
  2302  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2303  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2304  	p.AddRestSourceReg(maskReg(v.Args[2]))
  2305  	p.To.Type = obj.TYPE_REG
  2306  	p.To.Reg = simdReg(v)
  2307  	return p
  2308  }
  2309  
  2310  // Example instruction: VFMADD213PD Z2, Z1, Z0
  2311  func simdV31ResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2312  	p := s.Prog(v.Op.Asm())
  2313  	p.From.Type = obj.TYPE_REG
  2314  	p.From.Reg = simdReg(v.Args[2])
  2315  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2316  	p.To.Type = obj.TYPE_REG
  2317  	p.To.Reg = simdReg(v)
  2318  	return p
  2319  }
  2320  
  2321  func simdV31ResultInArg0Imm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2322  	p := s.Prog(v.Op.Asm())
  2323  	p.From.Offset = int64(v.AuxUInt8())
  2324  	p.From.Type = obj.TYPE_CONST
  2325  
  2326  	p.AddRestSourceReg(simdReg(v.Args[2]))
  2327  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2328  	// p.AddRestSourceReg(x86.REG_K0)
  2329  	p.To.Type = obj.TYPE_REG
  2330  	p.To.Reg = simdReg(v)
  2331  	return p
  2332  }
  2333  
  2334  // v31loadResultInArg0Imm8
  2335  // Example instruction:
  2336  // for (VPTERNLOGD128load {sym} [makeValAndOff(int32(int8(c)),off)]  x y ptr mem)
  2337  func simdV31loadResultInArg0Imm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2338  	sc := v.AuxValAndOff()
  2339  	p := s.Prog(v.Op.Asm())
  2340  
  2341  	p.From.Type = obj.TYPE_CONST
  2342  	p.From.Offset = sc.Val64()
  2343  
  2344  	m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[2].Reg()}
  2345  	ssagen.AddAux2(&m, v, sc.Off64())
  2346  	p.AddRestSource(m)
  2347  
  2348  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2349  	return p
  2350  }
  2351  
  2352  // Example instruction: VFMADD213PD Z2, Z1, K1, Z0
  2353  func simdV3kvResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2354  	p := s.Prog(v.Op.Asm())
  2355  	p.From.Type = obj.TYPE_REG
  2356  	p.From.Reg = simdReg(v.Args[2])
  2357  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2358  	p.AddRestSourceReg(maskReg(v.Args[3]))
  2359  	p.To.Type = obj.TYPE_REG
  2360  	p.To.Reg = simdReg(v)
  2361  	return p
  2362  }
  2363  
  2364  func simdVgpImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2365  	p := s.Prog(v.Op.Asm())
  2366  	p.From.Offset = int64(v.AuxUInt8())
  2367  	p.From.Type = obj.TYPE_CONST
  2368  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2369  	p.To.Type = obj.TYPE_REG
  2370  	p.To.Reg = v.Reg()
  2371  	return p
  2372  }
  2373  
  2374  func simdVgpImm(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2375  	// within simdgen, the choice of intrinsic shape and the output
  2376  	// instruction format are linked.  In the case of VgpImm, there is
  2377  	// a difference in the intrinsic, but no difference in the
  2378  	// instruction, it is just like VgpImm8.
  2379  	return simdVgpImm8(s, v)
  2380  }
  2381  
  2382  // Currently unused
  2383  func simdV31(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2384  	p := s.Prog(v.Op.Asm())
  2385  	p.From.Type = obj.TYPE_REG
  2386  	p.From.Reg = simdReg(v.Args[2])
  2387  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2388  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2389  	p.To.Type = obj.TYPE_REG
  2390  	p.To.Reg = simdReg(v)
  2391  	return p
  2392  }
  2393  
  2394  // Currently unused
  2395  func simdV3kv(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2396  	p := s.Prog(v.Op.Asm())
  2397  	p.From.Type = obj.TYPE_REG
  2398  	p.From.Reg = simdReg(v.Args[2])
  2399  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2400  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2401  	p.AddRestSourceReg(maskReg(v.Args[3]))
  2402  	p.To.Type = obj.TYPE_REG
  2403  	p.To.Reg = simdReg(v)
  2404  	return p
  2405  }
  2406  
  2407  // Example instruction: VRCP14PS (DI), K6, X22
  2408  func simdVkvload(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2409  	p := s.Prog(v.Op.Asm())
  2410  	p.From.Type = obj.TYPE_MEM
  2411  	p.From.Reg = v.Args[0].Reg()
  2412  	ssagen.AddAux(&p.From, v)
  2413  	p.AddRestSourceReg(maskReg(v.Args[1]))
  2414  	p.To.Type = obj.TYPE_REG
  2415  	p.To.Reg = simdReg(v)
  2416  	return p
  2417  }
  2418  
  2419  // Example instruction: VPSLLVD (DX), X7, X18
  2420  func simdV21load(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2421  	p := s.Prog(v.Op.Asm())
  2422  	p.From.Type = obj.TYPE_MEM
  2423  	p.From.Reg = v.Args[1].Reg()
  2424  	ssagen.AddAux(&p.From, v)
  2425  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2426  	p.To.Type = obj.TYPE_REG
  2427  	p.To.Reg = simdReg(v)
  2428  	return p
  2429  }
  2430  
  2431  // Example instruction: VPDPWSSD (SI), X24, X18
  2432  func simdV31loadResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2433  	p := s.Prog(v.Op.Asm())
  2434  	p.From.Type = obj.TYPE_MEM
  2435  	p.From.Reg = v.Args[2].Reg()
  2436  	ssagen.AddAux(&p.From, v)
  2437  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2438  	p.To.Type = obj.TYPE_REG
  2439  	p.To.Reg = simdReg(v)
  2440  	return p
  2441  }
  2442  
  2443  // Example instruction: VPDPWSSD (SI), X24, K1, X18
  2444  func simdV3kvloadResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2445  	p := s.Prog(v.Op.Asm())
  2446  	p.From.Type = obj.TYPE_MEM
  2447  	p.From.Reg = v.Args[2].Reg()
  2448  	ssagen.AddAux(&p.From, v)
  2449  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2450  	p.AddRestSourceReg(maskReg(v.Args[3]))
  2451  	p.To.Type = obj.TYPE_REG
  2452  	p.To.Reg = simdReg(v)
  2453  	return p
  2454  }
  2455  
  2456  // Example instruction: VPSLLVD (SI), X1, K1, X2
  2457  func simdV2kvload(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2458  	p := s.Prog(v.Op.Asm())
  2459  	p.From.Type = obj.TYPE_MEM
  2460  	p.From.Reg = v.Args[1].Reg()
  2461  	ssagen.AddAux(&p.From, v)
  2462  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2463  	p.AddRestSourceReg(maskReg(v.Args[2]))
  2464  	p.To.Type = obj.TYPE_REG
  2465  	p.To.Reg = simdReg(v)
  2466  	return p
  2467  }
  2468  
  2469  // Example instruction: VPCMPEQD (SI), X1, K1
  2470  func simdV2kload(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2471  	p := s.Prog(v.Op.Asm())
  2472  	p.From.Type = obj.TYPE_MEM
  2473  	p.From.Reg = v.Args[1].Reg()
  2474  	ssagen.AddAux(&p.From, v)
  2475  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2476  	p.To.Type = obj.TYPE_REG
  2477  	p.To.Reg = maskReg(v)
  2478  	return p
  2479  }
  2480  
  2481  // Example instruction: VCVTTPS2DQ (BX), X2
  2482  func simdV11load(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2483  	p := s.Prog(v.Op.Asm())
  2484  	p.From.Type = obj.TYPE_MEM
  2485  	p.From.Reg = v.Args[0].Reg()
  2486  	ssagen.AddAux(&p.From, v)
  2487  	p.To.Type = obj.TYPE_REG
  2488  	p.To.Reg = simdReg(v)
  2489  	return p
  2490  }
  2491  
  2492  // Example instruction: VPSHUFD $7, (BX), X11
  2493  func simdV11loadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2494  	sc := v.AuxValAndOff()
  2495  	p := s.Prog(v.Op.Asm())
  2496  	p.From.Type = obj.TYPE_CONST
  2497  	p.From.Offset = sc.Val64()
  2498  	m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[0].Reg()}
  2499  	ssagen.AddAux2(&m, v, sc.Off64())
  2500  	p.AddRestSource(m)
  2501  	p.To.Type = obj.TYPE_REG
  2502  	p.To.Reg = simdReg(v)
  2503  	return p
  2504  }
  2505  
  2506  // Example instruction: VPRORD $81, -15(R14), K7, Y1
  2507  func simdVkvloadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2508  	sc := v.AuxValAndOff()
  2509  	p := s.Prog(v.Op.Asm())
  2510  	p.From.Type = obj.TYPE_CONST
  2511  	p.From.Offset = sc.Val64()
  2512  	m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[0].Reg()}
  2513  	ssagen.AddAux2(&m, v, sc.Off64())
  2514  	p.AddRestSource(m)
  2515  	p.AddRestSourceReg(maskReg(v.Args[1]))
  2516  	p.To.Type = obj.TYPE_REG
  2517  	p.To.Reg = simdReg(v)
  2518  	return p
  2519  }
  2520  
  2521  // Example instruction: VPSHLDD $82, 7(SI), Y21, Y3
  2522  func simdV21loadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2523  	sc := v.AuxValAndOff()
  2524  	p := s.Prog(v.Op.Asm())
  2525  	p.From.Type = obj.TYPE_CONST
  2526  	p.From.Offset = sc.Val64()
  2527  	m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[1].Reg()}
  2528  	ssagen.AddAux2(&m, v, sc.Off64())
  2529  	p.AddRestSource(m)
  2530  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2531  	p.To.Type = obj.TYPE_REG
  2532  	p.To.Reg = simdReg(v)
  2533  	return p
  2534  }
  2535  
  2536  // Example instruction: VCMPPS $81, -7(DI), Y16, K3
  2537  func simdV2kloadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2538  	sc := v.AuxValAndOff()
  2539  	p := s.Prog(v.Op.Asm())
  2540  	p.From.Type = obj.TYPE_CONST
  2541  	p.From.Offset = sc.Val64()
  2542  	m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[1].Reg()}
  2543  	ssagen.AddAux2(&m, v, sc.Off64())
  2544  	p.AddRestSource(m)
  2545  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2546  	p.To.Type = obj.TYPE_REG
  2547  	p.To.Reg = maskReg(v)
  2548  	return p
  2549  }
  2550  
  2551  // Example instruction: VCMPPS $81, -7(DI), Y16, K1, K3
  2552  func simdV2kkloadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2553  	sc := v.AuxValAndOff()
  2554  	p := s.Prog(v.Op.Asm())
  2555  	p.From.Type = obj.TYPE_CONST
  2556  	p.From.Offset = sc.Val64()
  2557  	m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[1].Reg()}
  2558  	ssagen.AddAux2(&m, v, sc.Off64())
  2559  	p.AddRestSource(m)
  2560  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2561  	p.AddRestSourceReg(maskReg(v.Args[2]))
  2562  	p.To.Type = obj.TYPE_REG
  2563  	p.To.Reg = maskReg(v)
  2564  	return p
  2565  }
  2566  
  2567  // Example instruction: VGF2P8AFFINEINVQB $64, -17(BP), X31, K3, X26
  2568  func simdV2kvloadImm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2569  	sc := v.AuxValAndOff()
  2570  	p := s.Prog(v.Op.Asm())
  2571  	p.From.Type = obj.TYPE_CONST
  2572  	p.From.Offset = sc.Val64()
  2573  	m := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[1].Reg()}
  2574  	ssagen.AddAux2(&m, v, sc.Off64())
  2575  	p.AddRestSource(m)
  2576  	p.AddRestSourceReg(simdReg(v.Args[0]))
  2577  	p.AddRestSourceReg(maskReg(v.Args[2]))
  2578  	p.To.Type = obj.TYPE_REG
  2579  	p.To.Reg = simdReg(v)
  2580  	return p
  2581  }
  2582  
  2583  // Example instruction: SHA1NEXTE X2, X2
  2584  func simdV21ResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2585  	p := s.Prog(v.Op.Asm())
  2586  	p.From.Type = obj.TYPE_REG
  2587  	p.From.Reg = simdReg(v.Args[1])
  2588  	p.To.Type = obj.TYPE_REG
  2589  	p.To.Reg = simdReg(v)
  2590  	return p
  2591  }
  2592  
  2593  // Example instruction: SHA1RNDS4 $1, X2, X2
  2594  func simdV21ResultInArg0Imm8(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2595  	p := s.Prog(v.Op.Asm())
  2596  	p.From.Offset = int64(v.AuxUInt8())
  2597  	p.From.Type = obj.TYPE_CONST
  2598  	p.AddRestSourceReg(simdReg(v.Args[1]))
  2599  	p.To.Type = obj.TYPE_REG
  2600  	p.To.Reg = simdReg(v)
  2601  	return p
  2602  }
  2603  
  2604  // Example instruction: SHA256RNDS2 X0, X11, X2
  2605  func simdV31x0AtIn2ResultInArg0(s *ssagen.State, v *ssa.Value) *obj.Prog {
  2606  	return simdV31ResultInArg0(s, v)
  2607  }
  2608  
  2609  var blockJump = [...]struct {
  2610  	asm, invasm obj.As
  2611  }{
  2612  	block.BlockAMD64EQ:  {x86.AJEQ, x86.AJNE},
  2613  	block.BlockAMD64NE:  {x86.AJNE, x86.AJEQ},
  2614  	block.BlockAMD64LT:  {x86.AJLT, x86.AJGE},
  2615  	block.BlockAMD64GE:  {x86.AJGE, x86.AJLT},
  2616  	block.BlockAMD64LE:  {x86.AJLE, x86.AJGT},
  2617  	block.BlockAMD64GT:  {x86.AJGT, x86.AJLE},
  2618  	block.BlockAMD64OS:  {x86.AJOS, x86.AJOC},
  2619  	block.BlockAMD64OC:  {x86.AJOC, x86.AJOS},
  2620  	block.BlockAMD64ULT: {x86.AJCS, x86.AJCC},
  2621  	block.BlockAMD64UGE: {x86.AJCC, x86.AJCS},
  2622  	block.BlockAMD64UGT: {x86.AJHI, x86.AJLS},
  2623  	block.BlockAMD64ULE: {x86.AJLS, x86.AJHI},
  2624  	block.BlockAMD64ORD: {x86.AJPC, x86.AJPS},
  2625  	block.BlockAMD64NAN: {x86.AJPS, x86.AJPC},
  2626  }
  2627  
  2628  var eqfJumps = [2][2]ssagen.IndexJump{
  2629  	{{Jump: x86.AJNE, Index: 1}, {Jump: x86.AJPS, Index: 1}}, // next == b.Succs[0]
  2630  	{{Jump: x86.AJNE, Index: 1}, {Jump: x86.AJPC, Index: 0}}, // next == b.Succs[1]
  2631  }
  2632  var nefJumps = [2][2]ssagen.IndexJump{
  2633  	{{Jump: x86.AJNE, Index: 0}, {Jump: x86.AJPC, Index: 1}}, // next == b.Succs[0]
  2634  	{{Jump: x86.AJNE, Index: 0}, {Jump: x86.AJPS, Index: 0}}, // next == b.Succs[1]
  2635  }
  2636  
  2637  func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
  2638  	switch b.Kind {
  2639  	case block.BlockPlain, block.BlockDefer:
  2640  		if b.Succs[0].Block() != next {
  2641  			p := s.Prog(obj.AJMP)
  2642  			p.To.Type = obj.TYPE_BRANCH
  2643  			s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
  2644  		}
  2645  	case block.BlockExit, block.BlockRetJmp:
  2646  	case block.BlockRet:
  2647  		s.Prog(obj.ARET)
  2648  
  2649  	case block.BlockAMD64EQF:
  2650  		s.CombJump(b, next, &eqfJumps)
  2651  
  2652  	case block.BlockAMD64NEF:
  2653  		s.CombJump(b, next, &nefJumps)
  2654  
  2655  	case block.BlockAMD64EQ, block.BlockAMD64NE,
  2656  		block.BlockAMD64LT, block.BlockAMD64GE,
  2657  		block.BlockAMD64LE, block.BlockAMD64GT,
  2658  		block.BlockAMD64OS, block.BlockAMD64OC,
  2659  		block.BlockAMD64ULT, block.BlockAMD64UGT,
  2660  		block.BlockAMD64ULE, block.BlockAMD64UGE:
  2661  		jmp := blockJump[b.Kind]
  2662  		switch next {
  2663  		case b.Succs[0].Block():
  2664  			s.Br(jmp.invasm, b.Succs[1].Block())
  2665  		case b.Succs[1].Block():
  2666  			s.Br(jmp.asm, b.Succs[0].Block())
  2667  		default:
  2668  			if b.Likely != ssa.BranchUnlikely {
  2669  				s.Br(jmp.asm, b.Succs[0].Block())
  2670  				s.Br(obj.AJMP, b.Succs[1].Block())
  2671  			} else {
  2672  				s.Br(jmp.invasm, b.Succs[1].Block())
  2673  				s.Br(obj.AJMP, b.Succs[0].Block())
  2674  			}
  2675  		}
  2676  
  2677  	case block.BlockAMD64JUMPTABLE:
  2678  		// JMP      *(TABLE)(INDEX*8)
  2679  		p := s.Prog(obj.AJMP)
  2680  		p.To.Type = obj.TYPE_MEM
  2681  		p.To.Reg = b.Controls[1].Reg()
  2682  		p.To.Index = b.Controls[0].Reg()
  2683  		p.To.Scale = 8
  2684  		// Save jump tables for later resolution of the target blocks.
  2685  		s.JumpTables = append(s.JumpTables, b)
  2686  
  2687  	default:
  2688  		b.Fatalf("branch not implemented: %s", b.LongString())
  2689  	}
  2690  }
  2691  
  2692  func loadRegResult(s *ssagen.State, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  2693  	// The entry block's CPU features hold on every path through the
  2694  	// function, so they can be used regardless of where this load is
  2695  	// placed.
  2696  	p := s.Prog(loadByRegWidth(reg, t.Size(), f.Entry.CPUfeatures.HasFeature(ssa.CPUavx)))
  2697  	p.From.Type = obj.TYPE_MEM
  2698  	p.From.Name = obj.NAME_AUTO
  2699  	p.From.Sym = n.Linksym()
  2700  	p.From.Offset = n.FrameOffset() + off
  2701  	p.To.Type = obj.TYPE_REG
  2702  	p.To.Reg = reg
  2703  	return p
  2704  }
  2705  
  2706  func spillArgReg(pp *objw.Progs, p *obj.Prog, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  2707  	// See loadRegResult for why the entry block's features apply.
  2708  	p = pp.Append(p, storeByRegWidth(reg, t.Size(), f.Entry.CPUfeatures.HasFeature(ssa.CPUavx)), obj.TYPE_REG, reg, 0, obj.TYPE_MEM, 0, n.FrameOffset()+off)
  2709  	p.To.Name = obj.NAME_PARAM
  2710  	p.To.Sym = n.Linksym()
  2711  	p.Pos = p.Pos.WithNotStmt()
  2712  	return p
  2713  }
  2714  
  2715  // hasAVX reports whether the cpufeatures pass proved that AVX must be
  2716  // present whenever v executes. In that case VEX encodings are used for
  2717  // otherwise legacy-SSE instructions, to avoid AVX-SSE transition
  2718  // penalties. See issue #80835.
  2719  func hasAVX(v *ssa.Value) bool {
  2720  	return v.Block.CPUfeatures.HasFeature(ssa.CPUavx)
  2721  }
  2722  
  2723  // zero 16 bytes at reg+off.
  2724  func zero16(s *ssagen.State, reg int16, off int64, avx bool) {
  2725  	//   MOVUPS  X15, off(ptrReg)
  2726  	op := x86.AMOVUPS
  2727  	if avx {
  2728  		op = x86.AVMOVUPS
  2729  	}
  2730  	p := s.Prog(op)
  2731  	p.From.Type = obj.TYPE_REG
  2732  	p.From.Reg = x86.REG_X15
  2733  	p.To.Type = obj.TYPE_MEM
  2734  	p.To.Reg = reg
  2735  	p.To.Offset = off
  2736  }
  2737  
  2738  // move 16 bytes from src+off to dst+off using temporary register tmp.
  2739  func move16(s *ssagen.State, src, dst, tmp int16, off int64, avx bool) {
  2740  	//   MOVUPS  off(srcReg), tmpReg
  2741  	//   MOVUPS  tmpReg, off(dstReg)
  2742  	op := x86.AMOVUPS
  2743  	if avx {
  2744  		op = x86.AVMOVUPS
  2745  	}
  2746  	p := s.Prog(op)
  2747  	p.From.Type = obj.TYPE_MEM
  2748  	p.From.Reg = src
  2749  	p.From.Offset = off
  2750  	p.To.Type = obj.TYPE_REG
  2751  	p.To.Reg = tmp
  2752  	p = s.Prog(op)
  2753  	p.From.Type = obj.TYPE_REG
  2754  	p.From.Reg = tmp
  2755  	p.To.Type = obj.TYPE_MEM
  2756  	p.To.Reg = dst
  2757  	p.To.Offset = off
  2758  }
  2759  
  2760  // XXX maybe make this part of v.Reg?
  2761  // On the other hand, it is architecture-specific.
  2762  func simdReg(v *ssa.Value) int16 {
  2763  	t := v.Type
  2764  	if !t.IsSIMD() {
  2765  		base.Fatalf("simdReg: not a simd type; v=%s, b=b%d, f=%s", v.LongString(), v.Block.ID, v.Block.Func.Name)
  2766  	}
  2767  	return simdRegBySize(v.Reg(), t.Size())
  2768  }
  2769  
  2770  func simdRegBySize(reg int16, size int64) int16 {
  2771  	switch size {
  2772  	case 16:
  2773  		return reg
  2774  	case 32:
  2775  		return reg + (x86.REG_Y0 - x86.REG_X0)
  2776  	case 64:
  2777  		return reg + (x86.REG_Z0 - x86.REG_X0)
  2778  	}
  2779  	panic("simdRegBySize: bad size")
  2780  }
  2781  
  2782  // XXX k mask
  2783  func maskReg(v *ssa.Value) int16 {
  2784  	t := v.Type
  2785  	if !t.IsSIMD() {
  2786  		base.Fatalf("maskReg: not a simd type; v=%s, b=b%d, f=%s", v.LongString(), v.Block.ID, v.Block.Func.Name)
  2787  	}
  2788  	switch t.Size() {
  2789  	case 8:
  2790  		return v.Reg()
  2791  	}
  2792  	panic("unreachable")
  2793  }
  2794  
  2795  // XXX k mask + vec
  2796  func simdOrMaskReg(v *ssa.Value) int16 {
  2797  	t := v.Type
  2798  	if t.Size() <= 8 {
  2799  		return maskReg(v)
  2800  	}
  2801  	return simdReg(v)
  2802  }
  2803  
  2804  // XXX this is used for shift operations only.
  2805  // regalloc will issue OpCopy with incorrect type, but the assigned
  2806  // register should be correct, and this function is merely checking
  2807  // the sanity of this part.
  2808  func simdCheckRegOnly(v *ssa.Value, regStart, regEnd int16) int16 {
  2809  	if v.Reg() > regEnd || v.Reg() < regStart {
  2810  		panic("simdCheckRegOnly: not the desired register")
  2811  	}
  2812  	return v.Reg()
  2813  }
  2814  

View as plain text