Source file src/cmd/compile/internal/arm64/ssa.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package arm64
     6  
     7  import (
     8  	"math"
     9  
    10  	"cmd/compile/internal/base"
    11  	"cmd/compile/internal/ir"
    12  	"cmd/compile/internal/logopt"
    13  	"cmd/compile/internal/objw"
    14  	"cmd/compile/internal/ssa"
    15  	"cmd/compile/internal/ssa/block"
    16  	"cmd/compile/internal/ssa/ssaop"
    17  	"cmd/compile/internal/ssagen"
    18  	"cmd/compile/internal/types"
    19  	"cmd/internal/obj"
    20  	"cmd/internal/obj/arm64"
    21  	"internal/abi"
    22  )
    23  
    24  // loadByType returns the load instruction of the given type.
    25  func loadByType(t *types.Type) obj.As {
    26  	if t.IsSIMD() {
    27  		switch t.Size() {
    28  		case 16:
    29  			return arm64.AFMOVQ // Use FMOVQ (LDR Q) for 128-bit SIMD loads
    30  		case 8:
    31  			return arm64.APLDR
    32  		case 32:
    33  			return arm64.AZLDR
    34  		}
    35  	} else if t.IsFloat() {
    36  		switch t.Size() {
    37  		case 4:
    38  			return arm64.AFMOVS
    39  		case 8:
    40  			return arm64.AFMOVD
    41  		}
    42  	} else {
    43  		switch t.Size() {
    44  		case 1:
    45  			if t.IsSigned() {
    46  				return arm64.AMOVB
    47  			} else {
    48  				return arm64.AMOVBU
    49  			}
    50  		case 2:
    51  			if t.IsSigned() {
    52  				return arm64.AMOVH
    53  			} else {
    54  				return arm64.AMOVHU
    55  			}
    56  		case 4:
    57  			if t.IsSigned() {
    58  				return arm64.AMOVW
    59  			} else {
    60  				return arm64.AMOVWU
    61  			}
    62  		case 8:
    63  			return arm64.AMOVD
    64  		}
    65  	}
    66  	panic("bad load type")
    67  }
    68  
    69  // storeByType returns the store instruction of the given type.
    70  func storeByType(t *types.Type) obj.As {
    71  	if t.IsSIMD() {
    72  		switch t.Size() {
    73  		case 16:
    74  			return arm64.AFMOVQ // Use FMOVQ (STR Q) for 128-bit SIMD stores
    75  		case 8:
    76  			return arm64.APSTR
    77  		case 32:
    78  			return arm64.AZSTR
    79  		}
    80  	} else if t.IsFloat() {
    81  		switch t.Size() {
    82  		case 4:
    83  			return arm64.AFMOVS
    84  		case 8:
    85  			return arm64.AFMOVD
    86  		}
    87  	} else {
    88  		switch t.Size() {
    89  		case 1:
    90  			return arm64.AMOVB
    91  		case 2:
    92  			return arm64.AMOVH
    93  		case 4:
    94  			return arm64.AMOVW
    95  		case 8:
    96  			return arm64.AMOVD
    97  		}
    98  	}
    99  	panic("bad store type")
   100  }
   101  
   102  // loadByType2 returns an opcode that can load consecutive memory locations into 2 registers with type t.
   103  // returns obj.AXXX if no such opcode exists.
   104  func loadByType2(t *types.Type) obj.As {
   105  	if t.IsFloat() {
   106  		switch t.Size() {
   107  		case 4:
   108  			return arm64.AFLDPS
   109  		case 8:
   110  			return arm64.AFLDPD
   111  		}
   112  	} else {
   113  		switch t.Size() {
   114  		case 4:
   115  			return arm64.ALDPW
   116  		case 8:
   117  			return arm64.ALDP
   118  		}
   119  	}
   120  	return obj.AXXX
   121  }
   122  
   123  // storeByType2 returns an opcode that can store registers with type t into 2 consecutive memory locations.
   124  // returns obj.AXXX if no such opcode exists.
   125  func storeByType2(t *types.Type) obj.As {
   126  	if t.IsFloat() {
   127  		switch t.Size() {
   128  		case 4:
   129  			return arm64.AFSTPS
   130  		case 8:
   131  			return arm64.AFSTPD
   132  		}
   133  	} else {
   134  		switch t.Size() {
   135  		case 4:
   136  			return arm64.ASTPW
   137  		case 8:
   138  			return arm64.ASTP
   139  		}
   140  	}
   141  	return obj.AXXX
   142  }
   143  
   144  // makeshift encodes a register shifted by a constant, used as an Offset in Prog.
   145  func makeshift(v *ssa.Value, reg int16, typ int64, s int64) int64 {
   146  	if s < 0 || s >= 64 {
   147  		v.Fatalf("shift out of range: %d", s)
   148  	}
   149  	return int64(reg&31)<<16 | typ | (s&63)<<10
   150  }
   151  
   152  // genshift generates a Prog for r = r0 op (r1 shifted by n).
   153  func genshift(s *ssagen.State, v *ssa.Value, as obj.As, r0, r1, r int16, typ int64, n int64) *obj.Prog {
   154  	p := s.Prog(as)
   155  	p.From.Type = obj.TYPE_SHIFT
   156  	p.From.Offset = makeshift(v, r1, typ, n)
   157  	p.Reg = r0
   158  	if r != 0 {
   159  		p.To.Type = obj.TYPE_REG
   160  		p.To.Reg = r
   161  	}
   162  	return p
   163  }
   164  
   165  // generate the memory operand for the indexed load/store instructions.
   166  // base and idx are registers.
   167  func genIndexedOperand(op ssaop.Op, base, idx int16) obj.Addr {
   168  	// Reg: base register, Index: (shifted) index register
   169  	mop := obj.Addr{Type: obj.TYPE_MEM, Reg: base}
   170  	switch op {
   171  	case ssaop.OpARM64MOVDloadidx8, ssaop.OpARM64MOVDstoreidx8,
   172  		ssaop.OpARM64FMOVDloadidx8, ssaop.OpARM64FMOVDstoreidx8:
   173  		mop.Index = arm64.REG_LSL | 3<<5 | idx&31
   174  	case ssaop.OpARM64MOVWloadidx4, ssaop.OpARM64MOVWUloadidx4, ssaop.OpARM64MOVWstoreidx4,
   175  		ssaop.OpARM64FMOVSloadidx4, ssaop.OpARM64FMOVSstoreidx4:
   176  		mop.Index = arm64.REG_LSL | 2<<5 | idx&31
   177  	case ssaop.OpARM64MOVHloadidx2, ssaop.OpARM64MOVHUloadidx2, ssaop.OpARM64MOVHstoreidx2:
   178  		mop.Index = arm64.REG_LSL | 1<<5 | idx&31
   179  	default: // not shifted
   180  		mop.Index = idx
   181  	}
   182  	return mop
   183  }
   184  
   185  // simdSVEVectorLengthScaled marks a VL-scaled displacement ("#imm, mul vl").
   186  const simdSVEVectorLengthScaled = arm64.SIMDSVEVectorLengthScaled
   187  
   188  // simdRegArng encodes ssa value's register with specified simd arrangement
   189  func simdRegArng(reg int16, arng int16) int16 {
   190  	if reg < arm64.REG_F0 || arm64.REG_F31 < reg {
   191  		base.Fatalf("expected fp register: r%d", reg)
   192  	}
   193  	var err error
   194  	if reg, err = arm64.RegisterArrangement(reg, arng, false); err != nil {
   195  		base.Fatalf("bad simd register arrangement: %v", err)
   196  	}
   197  	return reg
   198  }
   199  
   200  // simdRegElem encodes ssa value's reference to a vector register element
   201  func simdRegElem(reg int16, arng int16, idx int16) (res obj.Addr) {
   202  	if reg < arm64.REG_F0 || arm64.REG_F31 < reg {
   203  		base.Fatalf("expected fp register: r%d", reg)
   204  	}
   205  	elem, err := arm64.RegisterArrangement(reg, arng, true /*indexing*/)
   206  	if err != nil {
   207  		base.Fatalf("bad simd register indexing arrangement: %v", err)
   208  	}
   209  	res.Type = obj.TYPE_REG
   210  	res.Class = arm64.C_ELEM
   211  	res.Index = idx
   212  	res.Reg = elem
   213  	return
   214  }
   215  
   216  // allLanes converts an element arrangement to its 128-bit vector arrangement.
   217  // e.g., ARNG_B -> ARNG_16B, ARNG_S -> ARNG_4S
   218  func allLanes(arng int16) int16 {
   219  	switch arng {
   220  	case arm64.ARNG_B:
   221  		return arm64.ARNG_16B
   222  	case arm64.ARNG_H:
   223  		return arm64.ARNG_8H
   224  	case arm64.ARNG_S:
   225  		return arm64.ARNG_4S
   226  	case arm64.ARNG_D:
   227  		return arm64.ARNG_2D
   228  	default:
   229  		base.Fatalf("unsupported element arrangement: %d", arng)
   230  		return 0
   231  	}
   232  }
   233  
   234  // arngNarrow converts arng to its narrow (halved element width and vector width) arrangement.
   235  func arngNarrow(arng int16) int16 {
   236  	switch arng {
   237  	case arm64.ARNG_8H:
   238  		return arm64.ARNG_8B
   239  	case arm64.ARNG_4S:
   240  		return arm64.ARNG_4H
   241  	case arm64.ARNG_2D:
   242  		return arm64.ARNG_2S
   243  	default:
   244  		base.Fatalf("unsupported narrow input arrangement: %d", arng)
   245  		return 0
   246  	}
   247  }
   248  
   249  // arngLong converts a half-lane arrangement to its long (doubled element width and vector width) arrangement.
   250  func arngLong(arng int16) int16 {
   251  	switch arng {
   252  	case arm64.ARNG_8B:
   253  		return arm64.ARNG_8H
   254  	case arm64.ARNG_4H:
   255  		return arm64.ARNG_4S
   256  	case arm64.ARNG_2S:
   257  		return arm64.ARNG_2D
   258  	case arm64.ARNG_1D:
   259  		return arm64.ARNG_1Q
   260  	default:
   261  		base.Fatalf("unsupported long input arrangement: %d", arng)
   262  		return 0
   263  	}
   264  }
   265  
   266  // arngHalfLanes converts a full-width arrangement to its half-lane (64-bit) arrangement.
   267  // Same element width, half the lanes. Used for long base variant sources.
   268  func arngHalfLanes(arng int16) int16 {
   269  	switch arng {
   270  	case arm64.ARNG_16B:
   271  		return arm64.ARNG_8B
   272  	case arm64.ARNG_8H:
   273  		return arm64.ARNG_4H
   274  	case arm64.ARNG_4S:
   275  		return arm64.ARNG_2S
   276  	case arm64.ARNG_2D:
   277  		return arm64.ARNG_1D
   278  	default:
   279  		base.Fatalf("unsupported halfLanes input arrangement: %d", arng)
   280  		return 0
   281  	}
   282  }
   283  
   284  // arngTwiceLanes converts a half-lane (64-bit) arrangement to its full-width arrangement.
   285  // Same element width, double the lanes. Inverse of arngHalfLanes.
   286  func arngTwiceLanes(arng int16) int16 {
   287  	switch arng {
   288  	case arm64.ARNG_8B:
   289  		return arm64.ARNG_16B
   290  	case arm64.ARNG_4H:
   291  		return arm64.ARNG_8H
   292  	case arm64.ARNG_2S:
   293  		return arm64.ARNG_4S
   294  	default:
   295  		base.Fatalf("unsupported twiceLanes input arrangement: %d", arng)
   296  		return 0
   297  	}
   298  }
   299  
   300  // simdV01Imm generates a VMOVI-like instruction, e.g. VMOVI $0, V0.B16
   301  func simdV01Imm(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   302  	p := s.Prog(v.Op.Asm())
   303  	p.From.Type = obj.TYPE_CONST
   304  	p.From.Offset = int64(v.AuxUInt8())
   305  	p.To.Type = obj.TYPE_REG
   306  	p.To.Reg = simdRegArng(v.Reg(), arrangement)
   307  	return p
   308  }
   309  
   310  // simdV11Asm generates element-wise unary vector operations with explicit asm, e.g. VMOV V1.B16, V0.B16
   311  func simdV11Asm(s *ssagen.State, asm obj.As, src, dst int16, arrangement int16) *obj.Prog {
   312  	p := s.Prog(asm)
   313  	p.From.Type = obj.TYPE_REG
   314  	p.From.Reg = simdRegArng(src, arrangement)
   315  	p.To.Type = obj.TYPE_REG
   316  	p.To.Reg = simdRegArng(dst, arrangement)
   317  	return p
   318  }
   319  
   320  // simdV11 generates element-wise unary vector operations, e.g. VCNT V1.B8, V0.B8
   321  func simdV11(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   322  	return simdV11Asm(s, v.Op.Asm(), v.Args[0].Reg(), v.Reg(), arrangement)
   323  }
   324  
   325  // simdV11Imm generates a unary vector operation with immediate constant,
   326  // e.g. VUSHR $3, V1.B16, V0.B16
   327  func simdV11Imm(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   328  	p := s.Prog(v.Op.Asm())
   329  	p.From.Type = obj.TYPE_CONST
   330  	p.From.Offset = int64(v.AuxUInt8())
   331  	p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   332  	p.To.Type = obj.TYPE_REG
   333  	p.To.Reg = simdRegArng(v.Reg(), arrangement)
   334  	return p
   335  }
   336  
   337  // simdV11ImmIn1 generates a broadcast1ToN instruction,
   338  // e.g. VDUP V1.S[0], V0.S4 (duplicate element 0 to all lanes)
   339  // The arrangement parameter specifies the element arrangement (e.g., ARNG_S, ARNG_D)
   340  func simdV11ImmIn1(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   341  	p := s.Prog(v.Op.Asm())
   342  	p.From = simdRegElem(v.Args[0].Reg(), arrangement, int16(v.AuxUInt8()))
   343  	p.To.Type = obj.TYPE_REG
   344  	p.To.Reg = simdRegArng(v.Reg(), allLanes(arrangement))
   345  	return p
   346  }
   347  
   348  // simdV11Scalar generates vector-to-scalar reduction operations, e.g. VUADDLV V1.B8, V0
   349  func simdV11Scalar(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   350  	p := s.Prog(v.Op.Asm())
   351  	p.From.Type = obj.TYPE_REG
   352  	p.From.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   353  	p.To.Type = obj.TYPE_REG
   354  	p.To.Reg = v.Reg() - arm64.REG_F0 + arm64.REG_V0
   355  	return p
   356  }
   357  
   358  // simdV11ScalarImmIn1 generates a SIMD instruction with indexed input and
   359  // scalar-in-vector-register output, e.g. VDUP V1.S[1], V0
   360  // The arrangement parameter specifies the source arrangement (e.g., S, D)
   361  func simdV11ScalarImmIn1(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   362  	p := s.Prog(v.Op.Asm())
   363  	p.From = simdRegElem(v.Args[0].Reg(), arrangement, int16(v.AuxUInt8()))
   364  	p.To.Type = obj.TYPE_REG
   365  	p.To.Reg = v.Reg() - arm64.REG_F0 + arm64.REG_V0
   366  	p.To.Class = arm64.C_VREG
   367  	return p
   368  }
   369  
   370  // simdV21 generates element-wise binary vector operations, e.g. VFADD V1.S4, V2.S4, V0.S4
   371  func simdV21(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   372  	p := s.Prog(v.Op.Asm())
   373  	p.From.Type = obj.TYPE_REG
   374  	p.From.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
   375  	p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   376  	p.To.Type = obj.TYPE_REG
   377  	p.To.Reg = simdRegArng(v.Reg(), arrangement)
   378  	return p
   379  }
   380  
   381  // simdV21Imm generates a binary instruction with immediate, e.g. EXT $imm, Vm.16B, Vn.16B, Vd.16B
   382  func simdV21Imm(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   383  	p := s.Prog(v.Op.Asm())
   384  	p.From.Type = obj.TYPE_CONST
   385  	p.From.Offset = int64(v.AuxUInt8())
   386  	p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   387  	p.To.Type = obj.TYPE_REG
   388  	p.To.Reg = simdRegArng(v.Reg(), arrangement)
   389  	p.AddRestSource(obj.Addr{Type: obj.TYPE_REG, Reg: simdRegArng(v.Args[1].Reg(), arrangement)})
   390  	return p
   391  }
   392  
   393  // simdV31ResultInArg0 generates a destructive 3-register instruction,
   394  // e.g. VBIT Vm.16B, Vn.16B, Vd.16B.
   395  func simdV31ResultInArg0(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   396  	p := s.Prog(v.Op.Asm())
   397  	p.From.Type = obj.TYPE_REG
   398  	p.From.Reg = simdRegArng(v.Args[2].Reg(), arrangement)
   399  	p.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
   400  	p.To.Type = obj.TYPE_REG
   401  	p.To.Reg = simdRegArng(v.Reg(), arrangement)
   402  	return p
   403  }
   404  
   405  // simdV21List generates a binary instruction with register list, e.g. TBL Vm.Ta, {Vn.B16}, Vd.Ta.
   406  func simdV21List(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   407  	if v.Op.Asm() != arm64.AVTBL { // TODO: support other instructions as needed.
   408  		panic("simdV21List: expected VTBL")
   409  	}
   410  	p := s.Prog(v.Op.Asm())
   411  	p.From.Type = obj.TYPE_REG
   412  	p.From.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
   413  	// TBL requires B16 table arrangement.
   414  	// Also, multi-element register lists are not supported by regalloc.
   415  	const listB16 = int64(1 << 30)
   416  	regList, _ := arm64.RegisterListOffset(int(v.Args[0].Reg()&31), 1, listB16, 0)
   417  	p.AddRestSource(obj.Addr{Type: obj.TYPE_REGLIST, Offset: regList})
   418  	p.To.Type = obj.TYPE_REG
   419  	p.To.Reg = simdRegArng(v.Reg(), arrangement)
   420  	return p
   421  }
   422  
   423  // simdV31ResultInArg0List generates a destructive 3-register instruction
   424  // with register list, e.g. TBX Vm.Ta, {Vn.B16}, Vd.Ta.
   425  func simdV31ResultInArg0List(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   426  	if v.Op.Asm() != arm64.AVTBX { // TODO: support other instructions as needed.
   427  		panic("simdV31ResultInArg0List: expected VTBX")
   428  	}
   429  	p := s.Prog(v.Op.Asm())
   430  	p.From.Type = obj.TYPE_REG
   431  	p.From.Reg = simdRegArng(v.Args[2].Reg(), arrangement)
   432  	// TBX requires B16 table arrangement.
   433  	// Also, multi-element register lists are not supported by regalloc.
   434  	const listB16 = int64(1 << 30)
   435  	regList, _ := arm64.RegisterListOffset(int(v.Args[1].Reg()&31), 1, listB16, 0)
   436  	p.AddRestSource(obj.Addr{Type: obj.TYPE_REGLIST, Offset: regList})
   437  	p.To.Type = obj.TYPE_REG
   438  	p.To.Reg = simdRegArng(v.Reg(), arrangement)
   439  	return p
   440  }
   441  
   442  // simdVfpvResultInArg0ImmOutIn1 generates vector floating-point SetElem,
   443  // e.g. VMOV V2.S[0], V1.S[3] (INS element instruction)
   444  // The arrangement parameter specifies the vector element arrangement (e.g., S, D)
   445  func simdVfpvResultInArg0ImmOutIn1(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   446  	p := s.Prog(v.Op.Asm())
   447  	p.To = simdRegElem(v.Reg(), arrangement, int16(v.AuxUInt8()))
   448  	p.From = simdRegElem(v.Args[1].Reg(), arrangement, 0)
   449  	return p
   450  }
   451  
   452  // simdVgpImmIn1 generates vector Elem instruction VMOV V1.S[2], R0
   453  // The arrangement parameter specifies the vector element arrangement (e.g., S, D)
   454  func simdVgpImmIn1(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   455  	p := s.Prog(v.Op.Asm())
   456  	p.From = simdRegElem(v.Args[0].Reg(), arrangement, int16(v.AuxUInt8()))
   457  	p.To.Reg = v.Reg()
   458  	p.To.Type = obj.TYPE_REG
   459  	return p
   460  }
   461  
   462  // simdVgpvResultInArg0ImmOutIn0 generates vector SetElem, e.g. VMOV R0, V1.S[2] (INS general instruction)
   463  // The arrangement parameter specifies the vector element arrangement (e.g., S, D)
   464  func simdVgpvResultInArg0ImmOutIn0(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   465  	p := s.Prog(v.Op.Asm())
   466  	p.To = simdRegElem(v.Reg(), arrangement, int16(v.AuxUInt8()))
   467  	p.From.Reg = v.Args[1].Reg()
   468  	p.From.Type = obj.TYPE_REG
   469  	return p
   470  }
   471  
   472  // Narrow and long lowering helpers
   473  
   474  // simdV11Narrow generates a pure narrowing instruction, e.g. XTN Vn.8H, Vd.8B
   475  func simdV11Narrow(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   476  	p := s.Prog(v.Op.Asm())
   477  	p.From.Type = obj.TYPE_REG
   478  	p.From.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   479  	p.To.Type = obj.TYPE_REG
   480  	p.To.Reg = simdRegArng(v.Reg(), arngNarrow(arrangement))
   481  	return p
   482  }
   483  
   484  // simdV21Narrow2 generates a destructive (updating upper half only) narrow "2" instruction,
   485  // e.g. XTN2 V1.4S, V0.8H. The arrangement parameter specifies the source arrangement.
   486  func simdV21Narrow2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   487  	p := s.Prog(v.Op.Asm())
   488  	p.From.Type = obj.TYPE_REG
   489  	p.From.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
   490  	p.To.Type = obj.TYPE_REG
   491  	p.To.Reg = simdRegArng(v.Reg(), arngTwiceLanes(arngNarrow(arrangement)))
   492  	return p
   493  }
   494  
   495  // simdV11ImmNarrow generates a pure narrowing instruction with immediate, e.g. SHRN $imm, V1.4S, V0.8B
   496  // The arrangement parameter specifies the source arrangement.
   497  func simdV11ImmNarrow(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   498  	p := s.Prog(v.Op.Asm())
   499  	p.From.Type = obj.TYPE_CONST
   500  	p.From.Offset = int64(v.AuxUInt8())
   501  	p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   502  	p.To.Type = obj.TYPE_REG
   503  	p.To.Reg = simdRegArng(v.Reg(), arngNarrow(arrangement))
   504  	return p
   505  }
   506  
   507  // simdV21ImmNarrow2 generates a destructive (updating upper half only) narrow "2" instruction
   508  // with immediate, e.g. SHRN2 $imm, V1.4S, V0.16B. The arrangement parameter specifies the source arrangement.
   509  func simdV21ImmNarrow2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   510  	p := s.Prog(v.Op.Asm())
   511  	p.From.Type = obj.TYPE_CONST
   512  	p.From.Offset = int64(v.AuxUInt8())
   513  	p.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
   514  	p.To.Type = obj.TYPE_REG
   515  	p.To.Reg = simdRegArng(v.Reg(), arngTwiceLanes(arngNarrow(arrangement)))
   516  	return p
   517  }
   518  
   519  // simdV11Long generates a unary long instruction, e.g. SXTL V1.4H, V0.8H
   520  // The instruction reads the lower half of the source, the destination has 2x element size.
   521  func simdV11Long(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   522  	src := arngHalfLanes(arrangement)
   523  	p := s.Prog(v.Op.Asm())
   524  	p.From.Type = obj.TYPE_REG
   525  	p.From.Reg = simdRegArng(v.Args[0].Reg(), src)
   526  	p.To.Type = obj.TYPE_REG
   527  	p.To.Reg = simdRegArng(v.Reg(), arngLong(src))
   528  	return p
   529  }
   530  
   531  // simdV11Long2 generates a unary long "2" instruction, e.g. SXTL2 V1.4S, V0.2D
   532  // The instruction reads the upper half of the source, the destination has 2x element size.
   533  func simdV11Long2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   534  	p := s.Prog(v.Op.Asm())
   535  	p.From.Type = obj.TYPE_REG
   536  	p.From.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   537  	p.To.Type = obj.TYPE_REG
   538  	p.To.Reg = simdRegArng(v.Reg(), arngLong(arngHalfLanes(arrangement)))
   539  	return p
   540  }
   541  
   542  // simdV11ImmLong generates a long instruction with immediate, e.g. USHLL $imm, V1.4H, V0.8H
   543  // The instruction reads the lower half of the source, the destination has 2x element size.
   544  func simdV11ImmLong(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   545  	src := arngHalfLanes(arrangement)
   546  	p := s.Prog(v.Op.Asm())
   547  	p.From.Type = obj.TYPE_CONST
   548  	p.From.Offset = int64(v.AuxUInt8())
   549  	p.Reg = simdRegArng(v.Args[0].Reg(), src)
   550  	p.To.Type = obj.TYPE_REG
   551  	p.To.Reg = simdRegArng(v.Reg(), arngLong(src))
   552  	return p
   553  }
   554  
   555  // simdV11ImmLong2 generates a long "2" instruction with immediate, e.g. USHLL2 $imm, V1.4S, V0.2D
   556  // The instruction reads the upper half of the source, the destination has 2x element size.
   557  func simdV11ImmLong2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   558  	p := s.Prog(v.Op.Asm())
   559  	p.From.Type = obj.TYPE_CONST
   560  	p.From.Offset = int64(v.AuxUInt8())
   561  	p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   562  	p.To.Type = obj.TYPE_REG
   563  	p.To.Reg = simdRegArng(v.Reg(), arngLong(arngHalfLanes(arrangement)))
   564  	return p
   565  }
   566  
   567  // simdV21Long generates a binary long instruction, e.g. UMULL V1.4H, V2.4H, V0.8H
   568  // The instruction reads lower halves of its sources, the destination has 2x element size.
   569  func simdV21Long(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   570  	src := arngHalfLanes(arrangement)
   571  	p := s.Prog(v.Op.Asm())
   572  	p.From.Type = obj.TYPE_REG
   573  	p.From.Reg = simdRegArng(v.Args[1].Reg(), src)
   574  	p.Reg = simdRegArng(v.Args[0].Reg(), src)
   575  	p.To.Type = obj.TYPE_REG
   576  	p.To.Reg = simdRegArng(v.Reg(), arngLong(src))
   577  	return p
   578  }
   579  
   580  // simdV21Long2 generates a binary long "2" instruction, e.g. UMULL2 V1.4S, V2.4S, V0.2D
   581  // The instruction reads upper halves of its sources, the destination has 2x element size.
   582  func simdV21Long2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
   583  	p := s.Prog(v.Op.Asm())
   584  	p.From.Type = obj.TYPE_REG
   585  	p.From.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
   586  	p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
   587  	p.To.Type = obj.TYPE_REG
   588  	p.To.Reg = simdRegArng(v.Reg(), arngLong(arngHalfLanes(arrangement)))
   589  	return p
   590  }
   591  
   592  func ssaGenValue(s *ssagen.State, v *ssa.Value) {
   593  	switch v.Op {
   594  	case ssaop.OpCopy, ssaop.OpARM64MOVDreg:
   595  		if v.Type.IsMemory() {
   596  			return
   597  		}
   598  		x := v.Args[0].Reg()
   599  		y := v.Reg()
   600  		if x == y {
   601  			return
   602  		}
   603  		as := arm64.AMOVD
   604  		if v.Type.IsFloat() {
   605  			switch v.Type.Size() {
   606  			case 4:
   607  				as = arm64.AFMOVS
   608  			case 8:
   609  				as = arm64.AFMOVD
   610  			default:
   611  				panic("bad float size")
   612  			}
   613  		} else if v.Type.IsSIMD() {
   614  			if v.Type.Size() == 16 {
   615  				simdV11Asm(s, arm64.AVMOV, x, y, arm64.ARNG_16B)
   616  				return
   617  			} else if v.Type.Size() == 32 {
   618  				// Z->Z
   619  				p := s.Prog(arm64.AZORR)
   620  				p.From.Type = obj.TYPE_REG
   621  				p.From.Reg = zregArng(x, arm64.ARNG_D)
   622  				p.AddRestSourceReg(zregArng(x, arm64.ARNG_D))
   623  				p.To.Type = obj.TYPE_REG
   624  				p.To.Reg = zregArng(y, arm64.ARNG_D)
   625  				return
   626  			} else if v.Type.Size() == 8 {
   627  				// P->P
   628  				p := s.Prog(arm64.APORR)
   629  				p.From.Type = obj.TYPE_REG
   630  				if x < arm64.REG_P0 || x > arm64.REG_P15 {
   631  					panic("bad P reg")
   632  				}
   633  				p.From.Reg = pregArng(x, arm64.ARNG_B)
   634  				p.AddRestSourceReg(pregArng(x, arm64.ARNG_B))
   635  				p.AddRestSourceReg(pregMask(x, arm64.PRED_Z))
   636  				p.To.Type = obj.TYPE_REG
   637  				p.To.Reg = pregArng(y, arm64.ARNG_B)
   638  				return
   639  			} else {
   640  				panic("bad simd size")
   641  			}
   642  		}
   643  		p := s.Prog(as)
   644  		p.From.Type = obj.TYPE_REG
   645  		p.From.Reg = x
   646  		p.To.Type = obj.TYPE_REG
   647  		p.To.Reg = y
   648  	case ssaop.OpARM64MOVDnop, ssaop.OpARM64ZERO:
   649  		// nothing to do
   650  	case ssaop.OpARM64VMOVI16B:
   651  		simdV01Imm(s, v, arm64.ARNG_16B)
   652  	case ssaop.OpLoadReg:
   653  		if v.Type.IsFlags() {
   654  			v.Fatalf("load flags not implemented: %v", v.LongString())
   655  			return
   656  		}
   657  		p := s.Prog(loadByType(v.Type))
   658  		ssagen.AddrAuto(&p.From, v.Args[0])
   659  		p.To.Type = obj.TYPE_REG
   660  		if v.Type.IsSIMD() && (v.Type.Size() == 32 || v.Type.Size() == 8) {
   661  			p.To.Reg = pzreg(v.Reg())
   662  		} else {
   663  			p.To.Reg = v.Reg()
   664  		}
   665  	case ssaop.OpStoreReg:
   666  		if v.Type.IsFlags() {
   667  			v.Fatalf("store flags not implemented: %v", v.LongString())
   668  			return
   669  		}
   670  		p := s.Prog(storeByType(v.Type))
   671  		p.From.Type = obj.TYPE_REG
   672  		if v.Type.IsSIMD() && (v.Type.Size() == 32 || v.Type.Size() == 8) {
   673  			p.From.Reg = pzreg(v.Args[0].Reg())
   674  		} else {
   675  			p.From.Reg = v.Args[0].Reg()
   676  		}
   677  		ssagen.AddrAuto(&p.To, v)
   678  	case ssaop.OpArgIntReg, ssaop.OpArgFloatReg:
   679  		ssagen.CheckArgReg(v)
   680  		// The assembler needs to wrap the entry safepoint/stack growth code with spill/unspill
   681  		// The loop only runs once.
   682  		args := v.Block.Func.RegArgs
   683  		if len(args) == 0 {
   684  			break
   685  		}
   686  		v.Block.Func.RegArgs = nil // prevent from running again
   687  
   688  		for i := 0; i < len(args); i++ {
   689  			a := args[i]
   690  			// Offset by size of the saved LR slot.
   691  			addr := ssagen.SpillSlotAddr(a, arm64.REGSP, base.Ctxt.Arch.FixedFrameSize)
   692  			// Look for double-register operations if we can.
   693  			if i < len(args)-1 {
   694  				b := args[i+1]
   695  				if a.Type.Size() == b.Type.Size() &&
   696  					a.Type.IsFloat() == b.Type.IsFloat() &&
   697  					b.Offset == a.Offset+a.Type.Size() {
   698  					ld := loadByType2(a.Type)
   699  					st := storeByType2(a.Type)
   700  					if ld != obj.AXXX && st != obj.AXXX {
   701  						s.FuncInfo().AddSpill(obj.RegSpill{Reg: a.Reg, Reg2: b.Reg, Addr: addr, Unspill: ld, Spill: st})
   702  						i++ // b is done also, skip it.
   703  						continue
   704  					}
   705  				}
   706  			}
   707  			reg := a.Reg
   708  			if a.Type.IsSIMD() && a.Type.Size() == 32 {
   709  				// SVE argument: the assembler materializes the slot through REGTMP.
   710  				reg = pzreg(reg)
   711  			}
   712  			// Pass the spill/unspill information along to the assembler.
   713  			s.FuncInfo().AddSpill(obj.RegSpill{Reg: reg, Addr: addr, Unspill: loadByType(a.Type), Spill: storeByType(a.Type)})
   714  		}
   715  
   716  	case ssaop.OpARM64ADD,
   717  		ssaop.OpARM64SUB,
   718  		ssaop.OpARM64AND,
   719  		ssaop.OpARM64OR,
   720  		ssaop.OpARM64XOR,
   721  		ssaop.OpARM64BIC,
   722  		ssaop.OpARM64EON,
   723  		ssaop.OpARM64ORN,
   724  		ssaop.OpARM64MUL,
   725  		ssaop.OpARM64MULW,
   726  		ssaop.OpARM64MNEG,
   727  		ssaop.OpARM64MNEGW,
   728  		ssaop.OpARM64MULH,
   729  		ssaop.OpARM64UMULH,
   730  		ssaop.OpARM64MULL,
   731  		ssaop.OpARM64UMULL,
   732  		ssaop.OpARM64DIV,
   733  		ssaop.OpARM64UDIV,
   734  		ssaop.OpARM64DIVW,
   735  		ssaop.OpARM64UDIVW,
   736  		ssaop.OpARM64MOD,
   737  		ssaop.OpARM64UMOD,
   738  		ssaop.OpARM64MODW,
   739  		ssaop.OpARM64UMODW,
   740  		ssaop.OpARM64SLL,
   741  		ssaop.OpARM64SRL,
   742  		ssaop.OpARM64SRA,
   743  		ssaop.OpARM64FADDS,
   744  		ssaop.OpARM64FADDD,
   745  		ssaop.OpARM64FSUBS,
   746  		ssaop.OpARM64FSUBD,
   747  		ssaop.OpARM64FMULS,
   748  		ssaop.OpARM64FMULD,
   749  		ssaop.OpARM64FNMULS,
   750  		ssaop.OpARM64FNMULD,
   751  		ssaop.OpARM64FDIVS,
   752  		ssaop.OpARM64FDIVD,
   753  		ssaop.OpARM64FMINS,
   754  		ssaop.OpARM64FMIND,
   755  		ssaop.OpARM64FMAXS,
   756  		ssaop.OpARM64FMAXD,
   757  		ssaop.OpARM64ROR,
   758  		ssaop.OpARM64RORW:
   759  		r := v.Reg()
   760  		r1 := v.Args[0].Reg()
   761  		r2 := v.Args[1].Reg()
   762  		p := s.Prog(v.Op.Asm())
   763  		p.From.Type = obj.TYPE_REG
   764  		p.From.Reg = r2
   765  		p.Reg = r1
   766  		p.To.Type = obj.TYPE_REG
   767  		p.To.Reg = r
   768  	case ssaop.OpARM64ZLDRload:
   769  		// Whole-register load of a scalable vector, e.g. ZLDR (VL*0)(R0), Z1.
   770  		p := s.Prog(v.Op.Asm())
   771  		p.From.Type = obj.TYPE_MEM
   772  		p.From.Reg = v.Args[0].Reg()
   773  		p.From.Scale = simdSVEVectorLengthScaled
   774  		ssagen.AddAux(&p.From, v)
   775  		p.To.Type = obj.TYPE_REG
   776  		p.To.Reg = pzreg(v.Reg())
   777  	case ssaop.OpARM64ZSTRstore:
   778  		// Whole-register VL-scaled store of a scalable vector: ZSTR Zn, (mem).
   779  		p := s.Prog(v.Op.Asm())
   780  		p.From.Type = obj.TYPE_REG
   781  		p.From.Reg = pzreg(v.Args[1].Reg())
   782  		p.To.Type = obj.TYPE_MEM
   783  		p.To.Reg = v.Args[0].Reg()
   784  		p.To.Scale = simdSVEVectorLengthScaled
   785  		ssagen.AddAux(&p.To, v)
   786  	case ssaop.OpARM64PLDRload:
   787  		// Whole-register VL-scaled load of a predicate, e.g. PLDR (VL*0)(R0), P0.
   788  		p := s.Prog(v.Op.Asm())
   789  		p.From.Type = obj.TYPE_MEM
   790  		p.From.Reg = v.Args[0].Reg()
   791  		p.From.Scale = simdSVEVectorLengthScaled
   792  		ssagen.AddAux(&p.From, v)
   793  		p.To.Type = obj.TYPE_REG
   794  		p.To.Reg = v.Reg()
   795  	case ssaop.OpARM64PSTRstore:
   796  		// Whole-register VL-scaled store of a predicate, e.g. PSTR P0, (VL*0)(R0).
   797  		p := s.Prog(v.Op.Asm())
   798  		p.From.Type = obj.TYPE_REG
   799  		p.From.Reg = v.Args[1].Reg()
   800  		p.To.Type = obj.TYPE_MEM
   801  		p.To.Reg = v.Args[0].Reg()
   802  		p.To.Scale = simdSVEVectorLengthScaled
   803  		ssagen.AddAux(&p.To, v)
   804  	case ssaop.OpARM64PPNEXTB:
   805  		simdPNext(s, v, arm64.ARNG_B)
   806  	case ssaop.OpARM64PPNEXTH:
   807  		simdPNext(s, v, arm64.ARNG_H)
   808  	case ssaop.OpARM64PPNEXTS:
   809  		simdPNext(s, v, arm64.ARNG_S)
   810  	case ssaop.OpARM64PPNEXTD:
   811  		simdPNext(s, v, arm64.ARNG_D)
   812  	case ssaop.OpARM64PBICSB:
   813  		// PBICS P2.B, P1.B, P0.Z, P3.B: P3 = P1 AND NOT P2 on the lanes of P0.
   814  		p := s.Prog(v.Op.Asm())
   815  		p.From.Type = obj.TYPE_REG
   816  		p.From.Reg = pregArng(v.Args[2].Reg(), arm64.ARNG_B)
   817  		p.AddRestSourceReg(pregArng(v.Args[1].Reg(), arm64.ARNG_B))
   818  		p.AddRestSourceReg(pregMask(v.Args[0].Reg(), arm64.PRED_Z))
   819  		p.To.Type = obj.TYPE_REG
   820  		p.To.Reg = pregArng(v.Reg0(), arm64.ARNG_B)
   821  	case ssaop.OpARM64PPTEST:
   822  		// PPTEST P1.B, P0: flags from the lanes of P1 that P0 governs.
   823  		p := s.Prog(v.Op.Asm())
   824  		p.From.Type = obj.TYPE_REG
   825  		p.From.Reg = pregArng(v.Args[1].Reg(), arm64.ARNG_B)
   826  		p.To.Type = obj.TYPE_REG
   827  		p.To.Reg = v.Args[0].Reg()
   828  	case ssaop.OpARM64PPFALSEB:
   829  		// Zero value of a mask: every lane false, e.g. PPFALSE P0.B.
   830  		p := s.Prog(v.Op.Asm())
   831  		p.To.Type = obj.TYPE_REG
   832  		p.To.Reg = pregArng(v.Reg(), arm64.ARNG_B)
   833  	case ssaop.OpARM64ZDUPBconst:
   834  		// Broadcast an 8-bit immediate to every byte lane (ZeroSIMD uses [0]).
   835  		simdZDupConst(s, v, arm64.ARNG_B)
   836  	case ssaop.OpARM64ZDUPHconst:
   837  		simdZDupConst(s, v, arm64.ARNG_H)
   838  	case ssaop.OpARM64ZDUPSconst:
   839  		simdZDupConst(s, v, arm64.ARNG_S)
   840  	case ssaop.OpARM64ZDUPDconst:
   841  		simdZDupConst(s, v, arm64.ARNG_D)
   842  	case ssaop.OpARM64RDVL:
   843  		// Read the vector length in bytes into a GP register, e.g. RDVL $1, R0.
   844  		p := s.Prog(v.Op.Asm())
   845  		p.From.Type = obj.TYPE_CONST
   846  		p.From.Offset = v.AuxInt
   847  		p.To.Type = obj.TYPE_REG
   848  		p.To.Reg = v.Reg()
   849  	case ssaop.OpARM64ZDUPB:
   850  		simdZDupGp(s, v, arm64.ARNG_B)
   851  	case ssaop.OpARM64ZDUPH:
   852  		simdZDupGp(s, v, arm64.ARNG_H)
   853  	case ssaop.OpARM64ZDUPS:
   854  		simdZDupGp(s, v, arm64.ARNG_S)
   855  	case ssaop.OpARM64ZDUPD:
   856  		simdZDupGp(s, v, arm64.ARNG_D)
   857  	case ssaop.OpARM64ZDUPIB:
   858  		simdZDupIndexed(s, v, arm64.ARNG_B)
   859  	case ssaop.OpARM64ZDUPIH:
   860  		simdZDupIndexed(s, v, arm64.ARNG_H)
   861  	case ssaop.OpARM64ZDUPIS:
   862  		simdZDupIndexed(s, v, arm64.ARNG_S)
   863  	case ssaop.OpARM64ZDUPID:
   864  		simdZDupIndexed(s, v, arm64.ARNG_D)
   865  	case ssaop.OpARM64ZCPYBconst:
   866  		simdZCpyConst(s, v, arm64.ARNG_B)
   867  	case ssaop.OpARM64ZCPYHconst:
   868  		simdZCpyConst(s, v, arm64.ARNG_H)
   869  	case ssaop.OpARM64ZCPYSconst:
   870  		simdZCpyConst(s, v, arm64.ARNG_S)
   871  	case ssaop.OpARM64ZCPYDconst:
   872  		simdZCpyConst(s, v, arm64.ARNG_D)
   873  	case ssaop.OpARM64ZCPYMergingBconst:
   874  		simdZCpyMergingConst(s, v, arm64.ARNG_B)
   875  	case ssaop.OpARM64ZCPYMergingHconst:
   876  		simdZCpyMergingConst(s, v, arm64.ARNG_H)
   877  	case ssaop.OpARM64ZCPYMergingSconst:
   878  		simdZCpyMergingConst(s, v, arm64.ARNG_S)
   879  	case ssaop.OpARM64ZCPYMergingDconst:
   880  		simdZCpyMergingConst(s, v, arm64.ARNG_D)
   881  	case ssaop.OpARM64ZCPYMergingB:
   882  		simdZCpyMergingGp(s, v, arm64.ARNG_B)
   883  	case ssaop.OpARM64ZCPYMergingH:
   884  		simdZCpyMergingGp(s, v, arm64.ARNG_H)
   885  	case ssaop.OpARM64ZCPYMergingS:
   886  		simdZCpyMergingGp(s, v, arm64.ARNG_S)
   887  	case ssaop.OpARM64ZCPYMergingD:
   888  		simdZCpyMergingGp(s, v, arm64.ARNG_D)
   889  	case ssaop.OpARM64ZCPYIMergingB:
   890  		simdZCpyMergingVec(s, v, arm64.ARNG_B)
   891  	case ssaop.OpARM64ZCPYIMergingH:
   892  		simdZCpyMergingVec(s, v, arm64.ARNG_H)
   893  	case ssaop.OpARM64ZCPYIMergingS:
   894  		simdZCpyMergingVec(s, v, arm64.ARNG_S)
   895  	case ssaop.OpARM64ZCPYIMergingD:
   896  		simdZCpyMergingVec(s, v, arm64.ARNG_D)
   897  	case ssaop.OpARM64ZSELB:
   898  		simdZ2kv(s, v, arm64.ARNG_B)
   899  	case ssaop.OpARM64ZSELH:
   900  		simdZ2kv(s, v, arm64.ARNG_H)
   901  	case ssaop.OpARM64ZSELS:
   902  		simdZ2kv(s, v, arm64.ARNG_S)
   903  	case ssaop.OpARM64ZSELD:
   904  		simdZ2kv(s, v, arm64.ARNG_D)
   905  	case ssaop.OpARM64PWHILELTB:
   906  		simdPWHILELT(s, v, arm64.ARNG_B)
   907  	case ssaop.OpARM64PWHILELTH:
   908  		simdPWHILELT(s, v, arm64.ARNG_H)
   909  	case ssaop.OpARM64PWHILELTS:
   910  		simdPWHILELT(s, v, arm64.ARNG_S)
   911  	case ssaop.OpARM64PWHILELTD:
   912  		simdPWHILELT(s, v, arm64.ARNG_D)
   913  	case ssaop.OpARM64ZLD1BPredload:
   914  		// Predicated contiguous byte load, e.g. ZLD1B (VL*0)(R0), P0.Z, [Z0.B].
   915  		// arg0=addr, arg1=pred, arg2=mem.
   916  		p := s.Prog(v.Op.Asm())
   917  		p.From.Type = obj.TYPE_MEM
   918  		p.From.Reg = v.Args[0].Reg()
   919  		p.From.Scale = simdSVEVectorLengthScaled
   920  		ssagen.AddAux(&p.From, v)
   921  		p.AddRestSourceReg(pregMask(v.Args[1].Reg(), arm64.PRED_Z))
   922  		p.To.Type = obj.TYPE_REGLIST
   923  		p.To.Offset, _ = arm64.RegisterListOffset(int(pzreg(v.Reg())), 1, regListArr("Z", "B"), 0)
   924  	case ssaop.OpARM64ZST1BPredstore:
   925  		// Predicated contiguous byte store, e.g. ZST1B [Z0.B], P0, (VL*0)(R0).
   926  		// arg0=addr, arg1=Z, arg2=pred, arg3=mem.
   927  		p := s.Prog(v.Op.Asm())
   928  		p.From.Type = obj.TYPE_REGLIST
   929  		p.From.Offset, _ = arm64.RegisterListOffset(int(pzreg(v.Args[1].Reg())), 1, regListArr("Z", "B"), 0)
   930  		p.AddRestSourceReg(v.Args[2].Reg()) // store uses a plain governing predicate (no .Z/.M)
   931  		p.To.Type = obj.TYPE_MEM
   932  		p.To.Reg = v.Args[0].Reg()
   933  		p.To.Scale = simdSVEVectorLengthScaled
   934  		ssagen.AddAux(&p.To, v)
   935  	case ssaop.OpARM64FMADDS,
   936  		ssaop.OpARM64FMADDD,
   937  		ssaop.OpARM64FNMADDS,
   938  		ssaop.OpARM64FNMADDD,
   939  		ssaop.OpARM64FMSUBS,
   940  		ssaop.OpARM64FMSUBD,
   941  		ssaop.OpARM64FNMSUBS,
   942  		ssaop.OpARM64FNMSUBD,
   943  		ssaop.OpARM64MADD,
   944  		ssaop.OpARM64MADDW,
   945  		ssaop.OpARM64MSUB,
   946  		ssaop.OpARM64MSUBW:
   947  		rt := v.Reg()
   948  		ra := v.Args[0].Reg()
   949  		rm := v.Args[1].Reg()
   950  		rn := v.Args[2].Reg()
   951  		p := s.Prog(v.Op.Asm())
   952  		p.Reg = ra
   953  		p.From.Type = obj.TYPE_REG
   954  		p.From.Reg = rm
   955  		p.AddRestSourceReg(rn)
   956  		p.To.Type = obj.TYPE_REG
   957  		p.To.Reg = rt
   958  	case ssaop.OpARM64ADDconst,
   959  		ssaop.OpARM64SUBconst,
   960  		ssaop.OpARM64ANDconst,
   961  		ssaop.OpARM64ORconst,
   962  		ssaop.OpARM64XORconst,
   963  		ssaop.OpARM64SLLconst,
   964  		ssaop.OpARM64SRLconst,
   965  		ssaop.OpARM64SRAconst,
   966  		ssaop.OpARM64RORconst,
   967  		ssaop.OpARM64RORWconst:
   968  		p := s.Prog(v.Op.Asm())
   969  		p.From.Type = obj.TYPE_CONST
   970  		p.From.Offset = v.AuxInt
   971  		p.Reg = v.Args[0].Reg()
   972  		p.To.Type = obj.TYPE_REG
   973  		p.To.Reg = v.Reg()
   974  	case ssaop.OpARM64ADDSconstflags:
   975  		p := s.Prog(v.Op.Asm())
   976  		p.From.Type = obj.TYPE_CONST
   977  		p.From.Offset = v.AuxInt
   978  		p.Reg = v.Args[0].Reg()
   979  		p.To.Type = obj.TYPE_REG
   980  		p.To.Reg = v.Reg0()
   981  	case ssaop.OpARM64ADCzerocarry:
   982  		p := s.Prog(v.Op.Asm())
   983  		p.From.Type = obj.TYPE_REG
   984  		p.From.Reg = arm64.REGZERO
   985  		p.Reg = arm64.REGZERO
   986  		p.To.Type = obj.TYPE_REG
   987  		p.To.Reg = v.Reg()
   988  	case ssaop.OpARM64ADCSflags,
   989  		ssaop.OpARM64ADDSflags,
   990  		ssaop.OpARM64SBCSflags,
   991  		ssaop.OpARM64SUBSflags:
   992  		r := v.Reg0()
   993  		r1 := v.Args[0].Reg()
   994  		r2 := v.Args[1].Reg()
   995  		p := s.Prog(v.Op.Asm())
   996  		p.From.Type = obj.TYPE_REG
   997  		p.From.Reg = r2
   998  		p.Reg = r1
   999  		p.To.Type = obj.TYPE_REG
  1000  		p.To.Reg = r
  1001  	case ssaop.OpARM64NEGSflags:
  1002  		p := s.Prog(v.Op.Asm())
  1003  		p.From.Type = obj.TYPE_REG
  1004  		p.From.Reg = v.Args[0].Reg()
  1005  		p.To.Type = obj.TYPE_REG
  1006  		p.To.Reg = v.Reg0()
  1007  	case ssaop.OpARM64NGCzerocarry:
  1008  		p := s.Prog(v.Op.Asm())
  1009  		p.From.Type = obj.TYPE_REG
  1010  		p.From.Reg = arm64.REGZERO
  1011  		p.To.Type = obj.TYPE_REG
  1012  		p.To.Reg = v.Reg()
  1013  	case ssaop.OpARM64EXTRconst,
  1014  		ssaop.OpARM64EXTRWconst:
  1015  		p := s.Prog(v.Op.Asm())
  1016  		p.From.Type = obj.TYPE_CONST
  1017  		p.From.Offset = v.AuxInt
  1018  		p.AddRestSourceReg(v.Args[0].Reg())
  1019  		p.Reg = v.Args[1].Reg()
  1020  		p.To.Type = obj.TYPE_REG
  1021  		p.To.Reg = v.Reg()
  1022  	case ssaop.OpARM64MVNshiftLL, ssaop.OpARM64NEGshiftLL:
  1023  		genshift(s, v, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_LL, v.AuxInt)
  1024  	case ssaop.OpARM64MVNshiftRL, ssaop.OpARM64NEGshiftRL:
  1025  		genshift(s, v, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_LR, v.AuxInt)
  1026  	case ssaop.OpARM64MVNshiftRA, ssaop.OpARM64NEGshiftRA:
  1027  		genshift(s, v, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_AR, v.AuxInt)
  1028  	case ssaop.OpARM64MVNshiftRO:
  1029  		genshift(s, v, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_ROR, v.AuxInt)
  1030  	case ssaop.OpARM64ADDshiftLL,
  1031  		ssaop.OpARM64SUBshiftLL,
  1032  		ssaop.OpARM64ANDshiftLL,
  1033  		ssaop.OpARM64ORshiftLL,
  1034  		ssaop.OpARM64XORshiftLL,
  1035  		ssaop.OpARM64EONshiftLL,
  1036  		ssaop.OpARM64ORNshiftLL,
  1037  		ssaop.OpARM64BICshiftLL:
  1038  		genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_LL, v.AuxInt)
  1039  	case ssaop.OpARM64ADDshiftRL,
  1040  		ssaop.OpARM64SUBshiftRL,
  1041  		ssaop.OpARM64ANDshiftRL,
  1042  		ssaop.OpARM64ORshiftRL,
  1043  		ssaop.OpARM64XORshiftRL,
  1044  		ssaop.OpARM64EONshiftRL,
  1045  		ssaop.OpARM64ORNshiftRL,
  1046  		ssaop.OpARM64BICshiftRL:
  1047  		genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_LR, v.AuxInt)
  1048  	case ssaop.OpARM64ADDshiftRA,
  1049  		ssaop.OpARM64SUBshiftRA,
  1050  		ssaop.OpARM64ANDshiftRA,
  1051  		ssaop.OpARM64ORshiftRA,
  1052  		ssaop.OpARM64XORshiftRA,
  1053  		ssaop.OpARM64EONshiftRA,
  1054  		ssaop.OpARM64ORNshiftRA,
  1055  		ssaop.OpARM64BICshiftRA:
  1056  		genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_AR, v.AuxInt)
  1057  	case ssaop.OpARM64ANDshiftRO,
  1058  		ssaop.OpARM64ORshiftRO,
  1059  		ssaop.OpARM64XORshiftRO,
  1060  		ssaop.OpARM64EONshiftRO,
  1061  		ssaop.OpARM64ORNshiftRO,
  1062  		ssaop.OpARM64BICshiftRO:
  1063  		genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_ROR, v.AuxInt)
  1064  	case ssaop.OpARM64MOVDconst:
  1065  		p := s.Prog(v.Op.Asm())
  1066  		p.From.Type = obj.TYPE_CONST
  1067  		p.From.Offset = v.AuxInt
  1068  		p.To.Type = obj.TYPE_REG
  1069  		p.To.Reg = v.Reg()
  1070  	case ssaop.OpARM64FMOVSconst,
  1071  		ssaop.OpARM64FMOVDconst:
  1072  		p := s.Prog(v.Op.Asm())
  1073  		p.From.Type = obj.TYPE_FCONST
  1074  		p.From.Val = math.Float64frombits(uint64(v.AuxInt))
  1075  		p.To.Type = obj.TYPE_REG
  1076  		p.To.Reg = v.Reg()
  1077  	case ssaop.OpARM64FCMPS0,
  1078  		ssaop.OpARM64FCMPD0:
  1079  		p := s.Prog(v.Op.Asm())
  1080  		p.From.Type = obj.TYPE_FCONST
  1081  		p.From.Val = math.Float64frombits(0)
  1082  		p.Reg = v.Args[0].Reg()
  1083  	case ssaop.OpARM64CMP,
  1084  		ssaop.OpARM64CMPW,
  1085  		ssaop.OpARM64CMN,
  1086  		ssaop.OpARM64CMNW,
  1087  		ssaop.OpARM64TST,
  1088  		ssaop.OpARM64TSTW,
  1089  		ssaop.OpARM64FCMPS,
  1090  		ssaop.OpARM64FCMPD:
  1091  		p := s.Prog(v.Op.Asm())
  1092  		p.From.Type = obj.TYPE_REG
  1093  		p.From.Reg = v.Args[1].Reg()
  1094  		p.Reg = v.Args[0].Reg()
  1095  	case ssaop.OpARM64CMPconst,
  1096  		ssaop.OpARM64CMPWconst,
  1097  		ssaop.OpARM64CMNconst,
  1098  		ssaop.OpARM64CMNWconst,
  1099  		ssaop.OpARM64TSTconst,
  1100  		ssaop.OpARM64TSTWconst:
  1101  		p := s.Prog(v.Op.Asm())
  1102  		p.From.Type = obj.TYPE_CONST
  1103  		p.From.Offset = v.AuxInt
  1104  		p.Reg = v.Args[0].Reg()
  1105  	case ssaop.OpARM64CMPshiftLL, ssaop.OpARM64CMNshiftLL, ssaop.OpARM64TSTshiftLL:
  1106  		genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_LL, v.AuxInt)
  1107  	case ssaop.OpARM64CMPshiftRL, ssaop.OpARM64CMNshiftRL, ssaop.OpARM64TSTshiftRL:
  1108  		genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_LR, v.AuxInt)
  1109  	case ssaop.OpARM64CMPshiftRA, ssaop.OpARM64CMNshiftRA, ssaop.OpARM64TSTshiftRA:
  1110  		genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_AR, v.AuxInt)
  1111  	case ssaop.OpARM64TSTshiftRO:
  1112  		genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_ROR, v.AuxInt)
  1113  	case ssaop.OpARM64MOVDaddr:
  1114  		p := s.Prog(arm64.AMOVD)
  1115  		p.From.Type = obj.TYPE_ADDR
  1116  		p.From.Reg = v.Args[0].Reg()
  1117  		p.To.Type = obj.TYPE_REG
  1118  		p.To.Reg = v.Reg()
  1119  
  1120  		var wantreg string
  1121  		// MOVD $sym+off(base), R
  1122  		// the assembler expands it as the following:
  1123  		// - base is SP: add constant offset to SP (R13)
  1124  		//               when constant is large, tmp register (R11) may be used
  1125  		// - base is SB: load external address from constant pool (use relocation)
  1126  		switch v.Aux.(type) {
  1127  		default:
  1128  			v.Fatalf("aux is of unknown type %T", v.Aux)
  1129  		case *obj.LSym:
  1130  			wantreg = "SB"
  1131  			ssagen.AddAux(&p.From, v)
  1132  		case *ir.Name:
  1133  			wantreg = "SP"
  1134  			ssagen.AddAux(&p.From, v)
  1135  		case nil:
  1136  			// No sym, just MOVD $off(SP), R
  1137  			wantreg = "SP"
  1138  			p.From.Offset = v.AuxInt
  1139  		}
  1140  		if reg := v.Args[0].RegName(); reg != wantreg {
  1141  			v.Fatalf("bad reg %s for symbol type %T, want %s", reg, v.Aux, wantreg)
  1142  		}
  1143  	case ssaop.OpARM64MOVBload,
  1144  		ssaop.OpARM64MOVBUload,
  1145  		ssaop.OpARM64MOVHload,
  1146  		ssaop.OpARM64MOVHUload,
  1147  		ssaop.OpARM64MOVWload,
  1148  		ssaop.OpARM64MOVWUload,
  1149  		ssaop.OpARM64MOVDload,
  1150  		ssaop.OpARM64FMOVSload,
  1151  		ssaop.OpARM64FMOVDload,
  1152  		ssaop.OpARM64FMOVQload:
  1153  		p := s.Prog(v.Op.Asm())
  1154  		p.From.Type = obj.TYPE_MEM
  1155  		p.From.Reg = v.Args[0].Reg()
  1156  		ssagen.AddAux(&p.From, v)
  1157  		p.To.Type = obj.TYPE_REG
  1158  		p.To.Reg = v.Reg()
  1159  	case ssaop.OpARM64LDP, ssaop.OpARM64LDPW, ssaop.OpARM64LDPSW, ssaop.OpARM64FLDPD, ssaop.OpARM64FLDPS, ssaop.OpARM64FLDPQ:
  1160  		p := s.Prog(v.Op.Asm())
  1161  		p.From.Type = obj.TYPE_MEM
  1162  		p.From.Reg = v.Args[0].Reg()
  1163  		ssagen.AddAux(&p.From, v)
  1164  		p.To.Type = obj.TYPE_REGREG
  1165  		p.To.Reg = v.Reg0()
  1166  		p.To.Offset = int64(v.Reg1())
  1167  	case ssaop.OpARM64MOVBloadidx,
  1168  		ssaop.OpARM64MOVBUloadidx,
  1169  		ssaop.OpARM64MOVHloadidx,
  1170  		ssaop.OpARM64MOVHUloadidx,
  1171  		ssaop.OpARM64MOVWloadidx,
  1172  		ssaop.OpARM64MOVWUloadidx,
  1173  		ssaop.OpARM64MOVDloadidx,
  1174  		ssaop.OpARM64FMOVSloadidx,
  1175  		ssaop.OpARM64FMOVDloadidx,
  1176  		ssaop.OpARM64MOVHloadidx2,
  1177  		ssaop.OpARM64MOVHUloadidx2,
  1178  		ssaop.OpARM64MOVWloadidx4,
  1179  		ssaop.OpARM64MOVWUloadidx4,
  1180  		ssaop.OpARM64MOVDloadidx8,
  1181  		ssaop.OpARM64FMOVDloadidx8,
  1182  		ssaop.OpARM64FMOVSloadidx4:
  1183  		p := s.Prog(v.Op.Asm())
  1184  		p.From = genIndexedOperand(v.Op, v.Args[0].Reg(), v.Args[1].Reg())
  1185  		p.To.Type = obj.TYPE_REG
  1186  		p.To.Reg = v.Reg()
  1187  	case ssaop.OpARM64LDAR,
  1188  		ssaop.OpARM64LDARB,
  1189  		ssaop.OpARM64LDARW:
  1190  		p := s.Prog(v.Op.Asm())
  1191  		p.From.Type = obj.TYPE_MEM
  1192  		p.From.Reg = v.Args[0].Reg()
  1193  		ssagen.AddAux(&p.From, v)
  1194  		p.To.Type = obj.TYPE_REG
  1195  		p.To.Reg = v.Reg0()
  1196  	case ssaop.OpARM64MOVBstore,
  1197  		ssaop.OpARM64MOVHstore,
  1198  		ssaop.OpARM64MOVWstore,
  1199  		ssaop.OpARM64MOVDstore,
  1200  		ssaop.OpARM64FMOVSstore,
  1201  		ssaop.OpARM64FMOVDstore,
  1202  		ssaop.OpARM64FMOVQstore,
  1203  		ssaop.OpARM64STLRB,
  1204  		ssaop.OpARM64STLR,
  1205  		ssaop.OpARM64STLRW:
  1206  		p := s.Prog(v.Op.Asm())
  1207  		p.From.Type = obj.TYPE_REG
  1208  		p.From.Reg = v.Args[1].Reg()
  1209  		p.To.Type = obj.TYPE_MEM
  1210  		p.To.Reg = v.Args[0].Reg()
  1211  		ssagen.AddAux(&p.To, v)
  1212  	case ssaop.OpARM64MOVBstoreidx,
  1213  		ssaop.OpARM64MOVHstoreidx,
  1214  		ssaop.OpARM64MOVWstoreidx,
  1215  		ssaop.OpARM64MOVDstoreidx,
  1216  		ssaop.OpARM64FMOVSstoreidx,
  1217  		ssaop.OpARM64FMOVDstoreidx,
  1218  		ssaop.OpARM64MOVHstoreidx2,
  1219  		ssaop.OpARM64MOVWstoreidx4,
  1220  		ssaop.OpARM64FMOVSstoreidx4,
  1221  		ssaop.OpARM64MOVDstoreidx8,
  1222  		ssaop.OpARM64FMOVDstoreidx8:
  1223  		p := s.Prog(v.Op.Asm())
  1224  		p.To = genIndexedOperand(v.Op, v.Args[0].Reg(), v.Args[1].Reg())
  1225  		p.From.Type = obj.TYPE_REG
  1226  		p.From.Reg = v.Args[2].Reg()
  1227  	case ssaop.OpARM64STP, ssaop.OpARM64STPW, ssaop.OpARM64FSTPD, ssaop.OpARM64FSTPS, ssaop.OpARM64FSTPQ:
  1228  		p := s.Prog(v.Op.Asm())
  1229  		p.From.Type = obj.TYPE_REGREG
  1230  		p.From.Reg = v.Args[1].Reg()
  1231  		p.From.Offset = int64(v.Args[2].Reg())
  1232  		p.To.Type = obj.TYPE_MEM
  1233  		p.To.Reg = v.Args[0].Reg()
  1234  		ssagen.AddAux(&p.To, v)
  1235  	case ssaop.OpARM64BFI,
  1236  		ssaop.OpARM64BFXIL:
  1237  		p := s.Prog(v.Op.Asm())
  1238  		p.From.Type = obj.TYPE_CONST
  1239  		p.From.Offset = v.AuxInt >> 8
  1240  		p.AddRestSourceConst(v.AuxInt & 0xff)
  1241  		p.Reg = v.Args[1].Reg()
  1242  		p.To.Type = obj.TYPE_REG
  1243  		p.To.Reg = v.Reg()
  1244  	case ssaop.OpARM64SBFIZ,
  1245  		ssaop.OpARM64SBFX,
  1246  		ssaop.OpARM64UBFIZ,
  1247  		ssaop.OpARM64UBFX:
  1248  		p := s.Prog(v.Op.Asm())
  1249  		p.From.Type = obj.TYPE_CONST
  1250  		p.From.Offset = v.AuxInt >> 8
  1251  		p.AddRestSourceConst(v.AuxInt & 0xff)
  1252  		p.Reg = v.Args[0].Reg()
  1253  		p.To.Type = obj.TYPE_REG
  1254  		p.To.Reg = v.Reg()
  1255  	case ssaop.OpARM64LoweredAtomicExchange64,
  1256  		ssaop.OpARM64LoweredAtomicExchange32,
  1257  		ssaop.OpARM64LoweredAtomicExchange8:
  1258  		// LDAXR	(Rarg0), Rout
  1259  		// STLXR	Rarg1, (Rarg0), Rtmp
  1260  		// CBNZ		Rtmp, -2(PC)
  1261  		//
  1262  		// If the width written to Rout changes, update zeroUpperBits in ARM64Ops.go.
  1263  		var ld, st obj.As
  1264  		switch v.Op {
  1265  		case ssaop.OpARM64LoweredAtomicExchange8:
  1266  			ld = arm64.ALDAXRB
  1267  			st = arm64.ASTLXRB
  1268  		case ssaop.OpARM64LoweredAtomicExchange32:
  1269  			ld = arm64.ALDAXRW
  1270  			st = arm64.ASTLXRW
  1271  		case ssaop.OpARM64LoweredAtomicExchange64:
  1272  			ld = arm64.ALDAXR
  1273  			st = arm64.ASTLXR
  1274  		}
  1275  		r0 := v.Args[0].Reg()
  1276  		r1 := v.Args[1].Reg()
  1277  		out := v.Reg0()
  1278  		p := s.Prog(ld)
  1279  		p.From.Type = obj.TYPE_MEM
  1280  		p.From.Reg = r0
  1281  		p.To.Type = obj.TYPE_REG
  1282  		p.To.Reg = out
  1283  		p1 := s.Prog(st)
  1284  		p1.From.Type = obj.TYPE_REG
  1285  		p1.From.Reg = r1
  1286  		p1.To.Type = obj.TYPE_MEM
  1287  		p1.To.Reg = r0
  1288  		p1.RegTo2 = arm64.REGTMP
  1289  		p2 := s.Prog(arm64.ACBNZ)
  1290  		p2.From.Type = obj.TYPE_REG
  1291  		p2.From.Reg = arm64.REGTMP
  1292  		p2.To.Type = obj.TYPE_BRANCH
  1293  		p2.To.SetTarget(p)
  1294  	case ssaop.OpARM64LoweredAtomicExchange64Variant,
  1295  		ssaop.OpARM64LoweredAtomicExchange32Variant,
  1296  		ssaop.OpARM64LoweredAtomicExchange8Variant:
  1297  		// If the width written to Rout changes, update zeroUpperBits in ARM64Ops.go.
  1298  		var swap obj.As
  1299  		switch v.Op {
  1300  		case ssaop.OpARM64LoweredAtomicExchange8Variant:
  1301  			swap = arm64.ASWPALB
  1302  		case ssaop.OpARM64LoweredAtomicExchange32Variant:
  1303  			swap = arm64.ASWPALW
  1304  		case ssaop.OpARM64LoweredAtomicExchange64Variant:
  1305  			swap = arm64.ASWPALD
  1306  		}
  1307  		r0 := v.Args[0].Reg()
  1308  		r1 := v.Args[1].Reg()
  1309  		out := v.Reg0()
  1310  
  1311  		// SWPALD	Rarg1, (Rarg0), Rout
  1312  		p := s.Prog(swap)
  1313  		p.From.Type = obj.TYPE_REG
  1314  		p.From.Reg = r1
  1315  		p.To.Type = obj.TYPE_MEM
  1316  		p.To.Reg = r0
  1317  		p.RegTo2 = out
  1318  
  1319  	case ssaop.OpARM64LoweredAtomicAdd64,
  1320  		ssaop.OpARM64LoweredAtomicAdd32:
  1321  		// LDAXR	(Rarg0), Rout
  1322  		// ADD		Rarg1, Rout
  1323  		// STLXR	Rout, (Rarg0), Rtmp
  1324  		// CBNZ		Rtmp, -3(PC)
  1325  		ld := arm64.ALDAXR
  1326  		st := arm64.ASTLXR
  1327  		if v.Op == ssaop.OpARM64LoweredAtomicAdd32 {
  1328  			ld = arm64.ALDAXRW
  1329  			st = arm64.ASTLXRW
  1330  		}
  1331  		r0 := v.Args[0].Reg()
  1332  		r1 := v.Args[1].Reg()
  1333  		out := v.Reg0()
  1334  		p := s.Prog(ld)
  1335  		p.From.Type = obj.TYPE_MEM
  1336  		p.From.Reg = r0
  1337  		p.To.Type = obj.TYPE_REG
  1338  		p.To.Reg = out
  1339  		p1 := s.Prog(arm64.AADD)
  1340  		p1.From.Type = obj.TYPE_REG
  1341  		p1.From.Reg = r1
  1342  		p1.To.Type = obj.TYPE_REG
  1343  		p1.To.Reg = out
  1344  		p2 := s.Prog(st)
  1345  		p2.From.Type = obj.TYPE_REG
  1346  		p2.From.Reg = out
  1347  		p2.To.Type = obj.TYPE_MEM
  1348  		p2.To.Reg = r0
  1349  		p2.RegTo2 = arm64.REGTMP
  1350  		p3 := s.Prog(arm64.ACBNZ)
  1351  		p3.From.Type = obj.TYPE_REG
  1352  		p3.From.Reg = arm64.REGTMP
  1353  		p3.To.Type = obj.TYPE_BRANCH
  1354  		p3.To.SetTarget(p)
  1355  	case ssaop.OpARM64LoweredAtomicAdd64Variant,
  1356  		ssaop.OpARM64LoweredAtomicAdd32Variant:
  1357  		// LDADDAL	Rarg1, (Rarg0), Rout
  1358  		// ADD		Rarg1, Rout
  1359  		op := arm64.ALDADDALD
  1360  		if v.Op == ssaop.OpARM64LoweredAtomicAdd32Variant {
  1361  			op = arm64.ALDADDALW
  1362  		}
  1363  		r0 := v.Args[0].Reg()
  1364  		r1 := v.Args[1].Reg()
  1365  		out := v.Reg0()
  1366  		p := s.Prog(op)
  1367  		p.From.Type = obj.TYPE_REG
  1368  		p.From.Reg = r1
  1369  		p.To.Type = obj.TYPE_MEM
  1370  		p.To.Reg = r0
  1371  		p.RegTo2 = out
  1372  		p1 := s.Prog(arm64.AADD)
  1373  		p1.From.Type = obj.TYPE_REG
  1374  		p1.From.Reg = r1
  1375  		p1.To.Type = obj.TYPE_REG
  1376  		p1.To.Reg = out
  1377  	case ssaop.OpARM64LoweredAtomicCas64,
  1378  		ssaop.OpARM64LoweredAtomicCas32:
  1379  		// LDAXR	(Rarg0), Rtmp
  1380  		// CMP		Rarg1, Rtmp
  1381  		// BNE		3(PC)
  1382  		// STLXR	Rarg2, (Rarg0), Rtmp
  1383  		// CBNZ		Rtmp, -4(PC)
  1384  		// CSET		EQ, Rout
  1385  		//
  1386  		// If Rout stops being written only by CSET, update zeroUpperBits in ARM64Ops.go.
  1387  		ld := arm64.ALDAXR
  1388  		st := arm64.ASTLXR
  1389  		cmp := arm64.ACMP
  1390  		if v.Op == ssaop.OpARM64LoweredAtomicCas32 {
  1391  			ld = arm64.ALDAXRW
  1392  			st = arm64.ASTLXRW
  1393  			cmp = arm64.ACMPW
  1394  		}
  1395  		r0 := v.Args[0].Reg()
  1396  		r1 := v.Args[1].Reg()
  1397  		r2 := v.Args[2].Reg()
  1398  		out := v.Reg0()
  1399  		p := s.Prog(ld)
  1400  		p.From.Type = obj.TYPE_MEM
  1401  		p.From.Reg = r0
  1402  		p.To.Type = obj.TYPE_REG
  1403  		p.To.Reg = arm64.REGTMP
  1404  		p1 := s.Prog(cmp)
  1405  		p1.From.Type = obj.TYPE_REG
  1406  		p1.From.Reg = r1
  1407  		p1.Reg = arm64.REGTMP
  1408  		p2 := s.Prog(arm64.ABNE)
  1409  		p2.To.Type = obj.TYPE_BRANCH
  1410  		p3 := s.Prog(st)
  1411  		p3.From.Type = obj.TYPE_REG
  1412  		p3.From.Reg = r2
  1413  		p3.To.Type = obj.TYPE_MEM
  1414  		p3.To.Reg = r0
  1415  		p3.RegTo2 = arm64.REGTMP
  1416  		p4 := s.Prog(arm64.ACBNZ)
  1417  		p4.From.Type = obj.TYPE_REG
  1418  		p4.From.Reg = arm64.REGTMP
  1419  		p4.To.Type = obj.TYPE_BRANCH
  1420  		p4.To.SetTarget(p)
  1421  		p5 := s.Prog(arm64.ACSET)
  1422  		p5.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset
  1423  		p5.From.Offset = int64(arm64.SPOP_EQ)
  1424  		p5.To.Type = obj.TYPE_REG
  1425  		p5.To.Reg = out
  1426  		p2.To.SetTarget(p5)
  1427  	case ssaop.OpARM64LoweredAtomicCas64Variant,
  1428  		ssaop.OpARM64LoweredAtomicCas32Variant:
  1429  		// Rarg0: ptr
  1430  		// Rarg1: old
  1431  		// Rarg2: new
  1432  		// MOV  	Rarg1, Rtmp
  1433  		// CASAL	Rtmp, (Rarg0), Rarg2
  1434  		// CMP  	Rarg1, Rtmp
  1435  		// CSET 	EQ, Rout
  1436  		//
  1437  		// If Rout stops being written only by CSET, update zeroUpperBits in ARM64Ops.go.
  1438  		cas := arm64.ACASALD
  1439  		cmp := arm64.ACMP
  1440  		mov := arm64.AMOVD
  1441  		if v.Op == ssaop.OpARM64LoweredAtomicCas32Variant {
  1442  			cas = arm64.ACASALW
  1443  			cmp = arm64.ACMPW
  1444  			mov = arm64.AMOVW
  1445  		}
  1446  		r0 := v.Args[0].Reg()
  1447  		r1 := v.Args[1].Reg()
  1448  		r2 := v.Args[2].Reg()
  1449  		out := v.Reg0()
  1450  
  1451  		// MOV  	Rarg1, Rtmp
  1452  		p := s.Prog(mov)
  1453  		p.From.Type = obj.TYPE_REG
  1454  		p.From.Reg = r1
  1455  		p.To.Type = obj.TYPE_REG
  1456  		p.To.Reg = arm64.REGTMP
  1457  
  1458  		// CASAL	Rtmp, (Rarg0), Rarg2
  1459  		p1 := s.Prog(cas)
  1460  		p1.From.Type = obj.TYPE_REG
  1461  		p1.From.Reg = arm64.REGTMP
  1462  		p1.To.Type = obj.TYPE_MEM
  1463  		p1.To.Reg = r0
  1464  		p1.RegTo2 = r2
  1465  
  1466  		// CMP  	Rarg1, Rtmp
  1467  		p2 := s.Prog(cmp)
  1468  		p2.From.Type = obj.TYPE_REG
  1469  		p2.From.Reg = r1
  1470  		p2.Reg = arm64.REGTMP
  1471  
  1472  		// CSET 	EQ, Rout
  1473  		p3 := s.Prog(arm64.ACSET)
  1474  		p3.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset
  1475  		p3.From.Offset = int64(arm64.SPOP_EQ)
  1476  		p3.To.Type = obj.TYPE_REG
  1477  		p3.To.Reg = out
  1478  
  1479  	case ssaop.OpARM64LoweredAtomicAnd64,
  1480  		ssaop.OpARM64LoweredAtomicOr64,
  1481  		ssaop.OpARM64LoweredAtomicAnd32,
  1482  		ssaop.OpARM64LoweredAtomicOr32,
  1483  		ssaop.OpARM64LoweredAtomicAnd8,
  1484  		ssaop.OpARM64LoweredAtomicOr8:
  1485  		// LDAXR[BW] (Rarg0), Rout
  1486  		// AND/OR	Rarg1, Rout, tmp1
  1487  		// STLXR[BW] tmp1, (Rarg0), Rtmp
  1488  		// CBNZ		Rtmp, -3(PC)
  1489  		//
  1490  		// If the width written to Rout changes, update zeroUpperBits in ARM64Ops.go.
  1491  		ld := arm64.ALDAXR
  1492  		st := arm64.ASTLXR
  1493  		if v.Op == ssaop.OpARM64LoweredAtomicAnd32 || v.Op == ssaop.OpARM64LoweredAtomicOr32 {
  1494  			ld = arm64.ALDAXRW
  1495  			st = arm64.ASTLXRW
  1496  		}
  1497  		if v.Op == ssaop.OpARM64LoweredAtomicAnd8 || v.Op == ssaop.OpARM64LoweredAtomicOr8 {
  1498  			ld = arm64.ALDAXRB
  1499  			st = arm64.ASTLXRB
  1500  		}
  1501  		r0 := v.Args[0].Reg()
  1502  		r1 := v.Args[1].Reg()
  1503  		out := v.Reg0()
  1504  		tmp := v.RegTmp()
  1505  		p := s.Prog(ld)
  1506  		p.From.Type = obj.TYPE_MEM
  1507  		p.From.Reg = r0
  1508  		p.To.Type = obj.TYPE_REG
  1509  		p.To.Reg = out
  1510  		p1 := s.Prog(v.Op.Asm())
  1511  		p1.From.Type = obj.TYPE_REG
  1512  		p1.From.Reg = r1
  1513  		p1.Reg = out
  1514  		p1.To.Type = obj.TYPE_REG
  1515  		p1.To.Reg = tmp
  1516  		p2 := s.Prog(st)
  1517  		p2.From.Type = obj.TYPE_REG
  1518  		p2.From.Reg = tmp
  1519  		p2.To.Type = obj.TYPE_MEM
  1520  		p2.To.Reg = r0
  1521  		p2.RegTo2 = arm64.REGTMP
  1522  		p3 := s.Prog(arm64.ACBNZ)
  1523  		p3.From.Type = obj.TYPE_REG
  1524  		p3.From.Reg = arm64.REGTMP
  1525  		p3.To.Type = obj.TYPE_BRANCH
  1526  		p3.To.SetTarget(p)
  1527  
  1528  	case ssaop.OpARM64LoweredAtomicAnd8Variant,
  1529  		ssaop.OpARM64LoweredAtomicAnd32Variant,
  1530  		ssaop.OpARM64LoweredAtomicAnd64Variant:
  1531  		// If the width written to Rout changes, update zeroUpperBits in ARM64Ops.go.
  1532  		atomic_clear := arm64.ALDCLRALD
  1533  		if v.Op == ssaop.OpARM64LoweredAtomicAnd32Variant {
  1534  			atomic_clear = arm64.ALDCLRALW
  1535  		}
  1536  		if v.Op == ssaop.OpARM64LoweredAtomicAnd8Variant {
  1537  			atomic_clear = arm64.ALDCLRALB
  1538  		}
  1539  		r0 := v.Args[0].Reg()
  1540  		r1 := v.Args[1].Reg()
  1541  		out := v.Reg0()
  1542  
  1543  		// MNV       Rarg1 Rtemp
  1544  		p := s.Prog(arm64.AMVN)
  1545  		p.From.Type = obj.TYPE_REG
  1546  		p.From.Reg = r1
  1547  		p.To.Type = obj.TYPE_REG
  1548  		p.To.Reg = arm64.REGTMP
  1549  
  1550  		// LDCLRAL[BDW]  Rtemp, (Rarg0), Rout
  1551  		p1 := s.Prog(atomic_clear)
  1552  		p1.From.Type = obj.TYPE_REG
  1553  		p1.From.Reg = arm64.REGTMP
  1554  		p1.To.Type = obj.TYPE_MEM
  1555  		p1.To.Reg = r0
  1556  		p1.RegTo2 = out
  1557  
  1558  	case ssaop.OpARM64LoweredAtomicOr8Variant,
  1559  		ssaop.OpARM64LoweredAtomicOr32Variant,
  1560  		ssaop.OpARM64LoweredAtomicOr64Variant:
  1561  		// If the width written to Rout changes, update zeroUpperBits in ARM64Ops.go.
  1562  		atomic_or := arm64.ALDORALD
  1563  		if v.Op == ssaop.OpARM64LoweredAtomicOr32Variant {
  1564  			atomic_or = arm64.ALDORALW
  1565  		}
  1566  		if v.Op == ssaop.OpARM64LoweredAtomicOr8Variant {
  1567  			atomic_or = arm64.ALDORALB
  1568  		}
  1569  		r0 := v.Args[0].Reg()
  1570  		r1 := v.Args[1].Reg()
  1571  		out := v.Reg0()
  1572  
  1573  		// LDORAL[BDW]  Rarg1, (Rarg0), Rout
  1574  		p := s.Prog(atomic_or)
  1575  		p.From.Type = obj.TYPE_REG
  1576  		p.From.Reg = r1
  1577  		p.To.Type = obj.TYPE_MEM
  1578  		p.To.Reg = r0
  1579  		p.RegTo2 = out
  1580  
  1581  	case ssaop.OpARM64MOVBreg,
  1582  		ssaop.OpARM64MOVBUreg,
  1583  		ssaop.OpARM64MOVHreg,
  1584  		ssaop.OpARM64MOVHUreg,
  1585  		ssaop.OpARM64MOVWreg,
  1586  		ssaop.OpARM64MOVWUreg:
  1587  		a := v.Args[0]
  1588  		for a.Op == ssaop.OpCopy || a.Op == ssaop.OpARM64MOVDreg {
  1589  			a = a.Args[0]
  1590  		}
  1591  		if a.Op == ssaop.OpLoadReg {
  1592  			t := a.Type
  1593  			switch {
  1594  			case v.Op == ssaop.OpARM64MOVBreg && t.Size() == 1 && t.IsSigned(),
  1595  				v.Op == ssaop.OpARM64MOVBUreg && t.Size() == 1 && !t.IsSigned(),
  1596  				v.Op == ssaop.OpARM64MOVHreg && t.Size() == 2 && t.IsSigned(),
  1597  				v.Op == ssaop.OpARM64MOVHUreg && t.Size() == 2 && !t.IsSigned(),
  1598  				v.Op == ssaop.OpARM64MOVWreg && t.Size() == 4 && t.IsSigned(),
  1599  				v.Op == ssaop.OpARM64MOVWUreg && t.Size() == 4 && !t.IsSigned():
  1600  				// arg is a proper-typed load, already zero/sign-extended, don't extend again
  1601  				if v.Reg() == v.Args[0].Reg() {
  1602  					return
  1603  				}
  1604  				p := s.Prog(arm64.AMOVD)
  1605  				p.From.Type = obj.TYPE_REG
  1606  				p.From.Reg = v.Args[0].Reg()
  1607  				p.To.Type = obj.TYPE_REG
  1608  				p.To.Reg = v.Reg()
  1609  				return
  1610  			default:
  1611  			}
  1612  		}
  1613  		fallthrough
  1614  	case ssaop.OpARM64MVN,
  1615  		ssaop.OpARM64NEG,
  1616  		ssaop.OpARM64FABSD,
  1617  		ssaop.OpARM64FABSS,
  1618  		ssaop.OpARM64FMOVDfpgp,
  1619  		ssaop.OpARM64FMOVDgpfp,
  1620  		ssaop.OpARM64FMOVSfpgp,
  1621  		ssaop.OpARM64FMOVSgpfp,
  1622  		ssaop.OpARM64FNEGS,
  1623  		ssaop.OpARM64FNEGD,
  1624  		ssaop.OpARM64FSQRTS,
  1625  		ssaop.OpARM64FSQRTD,
  1626  		ssaop.OpARM64FCVTZSSW,
  1627  		ssaop.OpARM64FCVTZSDW,
  1628  		ssaop.OpARM64FCVTZUSW,
  1629  		ssaop.OpARM64FCVTZUDW,
  1630  		ssaop.OpARM64FCVTZSS,
  1631  		ssaop.OpARM64FCVTZSD,
  1632  		ssaop.OpARM64FCVTZUS,
  1633  		ssaop.OpARM64FCVTZUD,
  1634  		ssaop.OpARM64SCVTFWS,
  1635  		ssaop.OpARM64SCVTFWD,
  1636  		ssaop.OpARM64SCVTFS,
  1637  		ssaop.OpARM64SCVTFD,
  1638  		ssaop.OpARM64UCVTFWS,
  1639  		ssaop.OpARM64UCVTFWD,
  1640  		ssaop.OpARM64UCVTFS,
  1641  		ssaop.OpARM64UCVTFD,
  1642  		ssaop.OpARM64FCVTSD,
  1643  		ssaop.OpARM64FCVTDS,
  1644  		ssaop.OpARM64REV,
  1645  		ssaop.OpARM64REVW,
  1646  		ssaop.OpARM64REV16,
  1647  		ssaop.OpARM64REV16W,
  1648  		ssaop.OpARM64RBIT,
  1649  		ssaop.OpARM64RBITW,
  1650  		ssaop.OpARM64CLZ,
  1651  		ssaop.OpARM64CLZW,
  1652  		ssaop.OpARM64FRINTAD,
  1653  		ssaop.OpARM64FRINTMD,
  1654  		ssaop.OpARM64FRINTND,
  1655  		ssaop.OpARM64FRINTPD,
  1656  		ssaop.OpARM64FRINTZD,
  1657  		ssaop.OpARM64FRINTAS,
  1658  		ssaop.OpARM64FRINTMS,
  1659  		ssaop.OpARM64FRINTNS,
  1660  		ssaop.OpARM64FRINTPS,
  1661  		ssaop.OpARM64FRINTZS:
  1662  		p := s.Prog(v.Op.Asm())
  1663  		p.From.Type = obj.TYPE_REG
  1664  		p.From.Reg = v.Args[0].Reg()
  1665  		p.To.Type = obj.TYPE_REG
  1666  		p.To.Reg = v.Reg()
  1667  	case ssaop.OpARM64LoweredRound32F, ssaop.OpARM64LoweredRound64F:
  1668  		// input is already rounded
  1669  	case ssaop.OpARM64VCNT:
  1670  		simdV11(s, v, arm64.ARNG_8B)
  1671  	case ssaop.OpARM64VUADDLV:
  1672  		simdV11Scalar(s, v, arm64.ARNG_8B)
  1673  	case ssaop.OpARM64CSEL, ssaop.OpARM64CSEL0, ssaop.OpARM64FCSELD, ssaop.OpARM64FCSELS:
  1674  		r1 := int16(arm64.REGZERO)
  1675  		if v.Op != ssaop.OpARM64CSEL0 {
  1676  			r1 = v.Args[1].Reg()
  1677  		}
  1678  		p := s.Prog(v.Op.Asm())
  1679  		p.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset
  1680  		condCode := condBits[ssaop.Op(v.AuxInt)]
  1681  		p.From.Offset = int64(condCode)
  1682  		p.Reg = v.Args[0].Reg()
  1683  		p.AddRestSourceReg(r1)
  1684  		p.To.Type = obj.TYPE_REG
  1685  		p.To.Reg = v.Reg()
  1686  	case ssaop.OpARM64CSINC, ssaop.OpARM64CSINV, ssaop.OpARM64CSNEG:
  1687  		p := s.Prog(v.Op.Asm())
  1688  		p.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset
  1689  		condCode := condBits[ssaop.Op(v.AuxInt)]
  1690  		p.From.Offset = int64(condCode)
  1691  		p.Reg = v.Args[0].Reg()
  1692  		p.AddRestSourceReg(v.Args[1].Reg())
  1693  		p.To.Type = obj.TYPE_REG
  1694  		p.To.Reg = v.Reg()
  1695  	case ssaop.OpARM64CSETM:
  1696  		p := s.Prog(arm64.ACSETM)
  1697  		p.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset
  1698  		condCode := condBits[ssaop.Op(v.AuxInt)]
  1699  		p.From.Offset = int64(condCode)
  1700  		p.To.Type = obj.TYPE_REG
  1701  		p.To.Reg = v.Reg()
  1702  	case ssaop.OpARM64CCMP,
  1703  		ssaop.OpARM64CCMN,
  1704  		ssaop.OpARM64CCMPconst,
  1705  		ssaop.OpARM64CCMNconst,
  1706  		ssaop.OpARM64CCMPW,
  1707  		ssaop.OpARM64CCMNW,
  1708  		ssaop.OpARM64CCMPWconst,
  1709  		ssaop.OpARM64CCMNWconst:
  1710  		p := s.Prog(v.Op.Asm())
  1711  		p.Reg = v.Args[0].Reg()
  1712  		params := v.AuxArm64ConditionalParams()
  1713  		p.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset
  1714  		p.From.Offset = int64(condBits[params.Cond])
  1715  		constValue, ok := params.ConstValue()
  1716  		if ok {
  1717  			p.AddRestSourceConst(constValue)
  1718  		} else {
  1719  			p.AddRestSourceReg(v.Args[1].Reg())
  1720  		}
  1721  		p.To.Type = obj.TYPE_CONST
  1722  		p.To.Offset = params.Nzcv()
  1723  	case ssaop.OpARM64LoweredZero:
  1724  		ptrReg := v.Args[0].Reg()
  1725  		n := v.AuxInt
  1726  		if n < 16 {
  1727  			v.Fatalf("Zero too small %d", n)
  1728  		}
  1729  
  1730  		// Generate zeroing instructions.
  1731  		var off int64
  1732  		for n >= 16 {
  1733  			//  STP     (ZR, ZR), off(ptrReg)
  1734  			zero16(s, ptrReg, off, false)
  1735  			off += 16
  1736  			n -= 16
  1737  		}
  1738  		// Write any fractional portion.
  1739  		// An overlapping 16-byte write can't be used here
  1740  		// because STP's offsets must be a multiple of 8.
  1741  		if n > 8 {
  1742  			//  MOVD    ZR, off(ptrReg)
  1743  			zero8(s, ptrReg, off)
  1744  			off += 8
  1745  			n -= 8
  1746  		}
  1747  		if n != 0 {
  1748  			//  MOVD    ZR, off+n-8(ptrReg)
  1749  			// TODO: for n<=4 we could use a smaller write.
  1750  			zero8(s, ptrReg, off+n-8)
  1751  		}
  1752  	case ssaop.OpARM64LoweredZeroLoop:
  1753  		ptrReg := v.Args[0].Reg()
  1754  		countReg := v.RegTmp()
  1755  		n := v.AuxInt
  1756  		loopSize := int64(64)
  1757  		if n < 3*loopSize {
  1758  			// - a loop count of 0 won't work.
  1759  			// - a loop count of 1 is useless.
  1760  			// - a loop count of 2 is a code size ~tie
  1761  			//     3 instructions to implement the loop
  1762  			//     4 instructions in the loop body
  1763  			//   vs
  1764  			//     8 instructions in the straightline code
  1765  			//   Might as well use straightline code.
  1766  			v.Fatalf("ZeroLoop size too small %d", n)
  1767  		}
  1768  
  1769  		// Put iteration count in a register.
  1770  		//   MOVD    $n, countReg
  1771  		p := s.Prog(arm64.AMOVD)
  1772  		p.From.Type = obj.TYPE_CONST
  1773  		p.From.Offset = n / loopSize
  1774  		p.To.Type = obj.TYPE_REG
  1775  		p.To.Reg = countReg
  1776  		cntInit := p
  1777  
  1778  		// Zero loopSize bytes starting at ptrReg.
  1779  		// Increment ptrReg by loopSize as a side effect.
  1780  		for range loopSize / 16 {
  1781  			//  STP.P   (ZR, ZR), 16(ptrReg)
  1782  			zero16(s, ptrReg, 0, true)
  1783  			// TODO: should we use the postincrement form,
  1784  			// or use a separate += 64 instruction?
  1785  			// postincrement saves an instruction, but maybe
  1786  			// it requires more integer units to do the +=16s.
  1787  		}
  1788  		// Decrement loop count.
  1789  		//   SUB     $1, countReg
  1790  		p = s.Prog(arm64.ASUB)
  1791  		p.From.Type = obj.TYPE_CONST
  1792  		p.From.Offset = 1
  1793  		p.To.Type = obj.TYPE_REG
  1794  		p.To.Reg = countReg
  1795  		// Jump to loop header if we're not done yet.
  1796  		//   CBNZ    head
  1797  		p = s.Prog(arm64.ACBNZ)
  1798  		p.From.Type = obj.TYPE_REG
  1799  		p.From.Reg = countReg
  1800  		p.To.Type = obj.TYPE_BRANCH
  1801  		p.To.SetTarget(cntInit.Link)
  1802  
  1803  		// Multiples of the loop size are now done.
  1804  		n %= loopSize
  1805  
  1806  		// Write any fractional portion.
  1807  		var off int64
  1808  		for n >= 16 {
  1809  			//  STP     (ZR, ZR), off(ptrReg)
  1810  			zero16(s, ptrReg, off, false)
  1811  			off += 16
  1812  			n -= 16
  1813  		}
  1814  		if n > 8 {
  1815  			// Note: an overlapping 16-byte write can't be used
  1816  			// here because STP's offsets must be a multiple of 8.
  1817  			//  MOVD    ZR, off(ptrReg)
  1818  			zero8(s, ptrReg, off)
  1819  			off += 8
  1820  			n -= 8
  1821  		}
  1822  		if n != 0 {
  1823  			//  MOVD    ZR, off+n-8(ptrReg)
  1824  			// TODO: for n<=4 we could use a smaller write.
  1825  			zero8(s, ptrReg, off+n-8)
  1826  		}
  1827  		// TODO: maybe we should use the count register to instead
  1828  		// hold an end pointer and compare against that?
  1829  		//   ADD $n, ptrReg, endReg
  1830  		// then
  1831  		//   CMP ptrReg, endReg
  1832  		//   BNE loop
  1833  		// There's a past-the-end pointer here, any problem with that?
  1834  
  1835  	case ssaop.OpARM64LoweredMove:
  1836  		dstReg := v.Args[0].Reg()
  1837  		srcReg := v.Args[1].Reg()
  1838  		if dstReg == srcReg {
  1839  			break
  1840  		}
  1841  		tmpReg1 := int16(arm64.REG_R25)
  1842  		tmpFReg1 := int16(arm64.REG_F16)
  1843  		tmpFReg2 := int16(arm64.REG_F17)
  1844  		n := v.AuxInt
  1845  		if n < 16 {
  1846  			v.Fatalf("Move too small %d", n)
  1847  		}
  1848  
  1849  		// Generate copying instructions.
  1850  		var off int64
  1851  		for n >= 32 {
  1852  			//  FLDPQ   off(srcReg), (tmpFReg1, tmpFReg2)
  1853  			//  FSTPQ   (tmpFReg1, tmpFReg2), off(dstReg)
  1854  			move32(s, srcReg, dstReg, tmpFReg1, tmpFReg2, off, false)
  1855  			off += 32
  1856  			n -= 32
  1857  		}
  1858  		for n >= 16 {
  1859  			//  FMOVQ   off(src), tmpFReg1
  1860  			//  FMOVQ   tmpFReg1, off(dst)
  1861  			move16(s, srcReg, dstReg, tmpFReg1, off, false)
  1862  			off += 16
  1863  			n -= 16
  1864  		}
  1865  		if n > 8 {
  1866  			//  MOVD    off(srcReg), tmpReg1
  1867  			//  MOVD    tmpReg1, off(dstReg)
  1868  			move8(s, srcReg, dstReg, tmpReg1, off)
  1869  			off += 8
  1870  			n -= 8
  1871  		}
  1872  		if n != 0 {
  1873  			//  MOVD    off+n-8(srcReg), tmpReg1
  1874  			//  MOVD    tmpReg1, off+n-8(dstReg)
  1875  			move8(s, srcReg, dstReg, tmpReg1, off+n-8)
  1876  		}
  1877  	case ssaop.OpARM64LoweredMoveLoop:
  1878  		dstReg := v.Args[0].Reg()
  1879  		srcReg := v.Args[1].Reg()
  1880  		if dstReg == srcReg {
  1881  			break
  1882  		}
  1883  		countReg := int16(arm64.REG_R24)
  1884  		tmpReg1 := int16(arm64.REG_R25)
  1885  		tmpFReg1 := int16(arm64.REG_F16)
  1886  		tmpFReg2 := int16(arm64.REG_F17)
  1887  		n := v.AuxInt
  1888  		loopSize := int64(64)
  1889  		if n < 3*loopSize {
  1890  			// - a loop count of 0 won't work.
  1891  			// - a loop count of 1 is useless.
  1892  			// - a loop count of 2 is a code size ~tie
  1893  			//     3 instructions to implement the loop
  1894  			//     4 instructions in the loop body
  1895  			//   vs
  1896  			//     8 instructions in the straightline code
  1897  			//   Might as well use straightline code.
  1898  			v.Fatalf("ZeroLoop size too small %d", n)
  1899  		}
  1900  
  1901  		// Put iteration count in a register.
  1902  		//   MOVD    $n, countReg
  1903  		p := s.Prog(arm64.AMOVD)
  1904  		p.From.Type = obj.TYPE_CONST
  1905  		p.From.Offset = n / loopSize
  1906  		p.To.Type = obj.TYPE_REG
  1907  		p.To.Reg = countReg
  1908  		cntInit := p
  1909  
  1910  		// Move loopSize bytes starting at srcReg to dstReg.
  1911  		// Increment srcReg and destReg by loopSize as a side effect.
  1912  		for range loopSize / 32 {
  1913  			// FLDPQ.P 32(srcReg), (tmpFReg1, tmpFReg2)
  1914  			// FSTPQ.P (tmpFReg1, tmpFReg2), 32(dstReg)
  1915  			move32(s, srcReg, dstReg, tmpFReg1, tmpFReg2, 0, true)
  1916  		}
  1917  		// Decrement loop count.
  1918  		//   SUB     $1, countReg
  1919  		p = s.Prog(arm64.ASUB)
  1920  		p.From.Type = obj.TYPE_CONST
  1921  		p.From.Offset = 1
  1922  		p.To.Type = obj.TYPE_REG
  1923  		p.To.Reg = countReg
  1924  		// Jump to loop header if we're not done yet.
  1925  		//   CBNZ    head
  1926  		p = s.Prog(arm64.ACBNZ)
  1927  		p.From.Type = obj.TYPE_REG
  1928  		p.From.Reg = countReg
  1929  		p.To.Type = obj.TYPE_BRANCH
  1930  		p.To.SetTarget(cntInit.Link)
  1931  
  1932  		// Multiples of the loop size are now done.
  1933  		n %= loopSize
  1934  
  1935  		// Copy any fractional portion.
  1936  		var off int64
  1937  		for n >= 32 {
  1938  			//  FLDPQ   off(srcReg), (tmpFReg1, tmpFReg2)
  1939  			//  FSTPQ   (tmpFReg1, tmpFReg2), off(dstReg)
  1940  			move32(s, srcReg, dstReg, tmpFReg1, tmpFReg2, off, false)
  1941  			off += 32
  1942  			n -= 32
  1943  		}
  1944  		for n >= 16 {
  1945  			//  FMOVQ   off(src), tmpFReg1
  1946  			//  FMOVQ   tmpFReg1, off(dst)
  1947  			move16(s, srcReg, dstReg, tmpFReg1, off, false)
  1948  			off += 16
  1949  			n -= 16
  1950  		}
  1951  		if n > 8 {
  1952  			//  MOVD    off(srcReg), tmpReg1
  1953  			//  MOVD    tmpReg1, off(dstReg)
  1954  			move8(s, srcReg, dstReg, tmpReg1, off)
  1955  			off += 8
  1956  			n -= 8
  1957  		}
  1958  		if n != 0 {
  1959  			//  MOVD    off+n-8(srcReg), tmpReg1
  1960  			//  MOVD    tmpReg1, off+n-8(dstReg)
  1961  			move8(s, srcReg, dstReg, tmpReg1, off+n-8)
  1962  		}
  1963  
  1964  	case ssaop.OpARM64CALLstatic, ssaop.OpARM64CALLclosure, ssaop.OpARM64CALLinter:
  1965  		s.Call(v)
  1966  	case ssaop.OpARM64CALLtail, ssaop.OpARM64CALLtailinter:
  1967  		s.TailCall(v)
  1968  	case ssaop.OpARM64LoweredWB:
  1969  		p := s.Prog(obj.ACALL)
  1970  		p.To.Type = obj.TYPE_MEM
  1971  		p.To.Name = obj.NAME_EXTERN
  1972  		// AuxInt encodes how many buffer entries we need.
  1973  		p.To.Sym = ir.Syms.GCWriteBarrier[v.AuxInt-1]
  1974  	case ssaop.OpARM64LoweredMemEq:
  1975  		p := s.Prog(obj.ACALL)
  1976  		p.To.Type = obj.TYPE_MEM
  1977  		p.To.Name = obj.NAME_EXTERN
  1978  		p.To.Sym = ir.Syms.Memequal
  1979  
  1980  	case ssaop.OpARM64LoweredPanicBoundsRR, ssaop.OpARM64LoweredPanicBoundsRC, ssaop.OpARM64LoweredPanicBoundsCR, ssaop.OpARM64LoweredPanicBoundsCC:
  1981  		// Compute the constant we put in the PCData entry for this call.
  1982  		code, signed := ssa.BoundsKind(v.AuxInt).Code()
  1983  		xIsReg := false
  1984  		yIsReg := false
  1985  		xVal := 0
  1986  		yVal := 0
  1987  		switch v.Op {
  1988  		case ssaop.OpARM64LoweredPanicBoundsRR:
  1989  			xIsReg = true
  1990  			xVal = int(v.Args[0].Reg() - arm64.REG_R0)
  1991  			yIsReg = true
  1992  			yVal = int(v.Args[1].Reg() - arm64.REG_R0)
  1993  		case ssaop.OpARM64LoweredPanicBoundsRC:
  1994  			xIsReg = true
  1995  			xVal = int(v.Args[0].Reg() - arm64.REG_R0)
  1996  			c := v.Aux.(ssa.PanicBoundsC).C
  1997  			if c >= 0 && c <= abi.BoundsMaxConst {
  1998  				yVal = int(c)
  1999  			} else {
  2000  				// Move constant to a register
  2001  				yIsReg = true
  2002  				if yVal == xVal {
  2003  					yVal = 1
  2004  				}
  2005  				p := s.Prog(arm64.AMOVD)
  2006  				p.From.Type = obj.TYPE_CONST
  2007  				p.From.Offset = c
  2008  				p.To.Type = obj.TYPE_REG
  2009  				p.To.Reg = arm64.REG_R0 + int16(yVal)
  2010  			}
  2011  		case ssaop.OpARM64LoweredPanicBoundsCR:
  2012  			yIsReg = true
  2013  			yVal = int(v.Args[0].Reg() - arm64.REG_R0)
  2014  			c := v.Aux.(ssa.PanicBoundsC).C
  2015  			if c >= 0 && c <= abi.BoundsMaxConst {
  2016  				xVal = int(c)
  2017  			} else {
  2018  				// Move constant to a register
  2019  				xIsReg = true
  2020  				if xVal == yVal {
  2021  					xVal = 1
  2022  				}
  2023  				p := s.Prog(arm64.AMOVD)
  2024  				p.From.Type = obj.TYPE_CONST
  2025  				p.From.Offset = c
  2026  				p.To.Type = obj.TYPE_REG
  2027  				p.To.Reg = arm64.REG_R0 + int16(xVal)
  2028  			}
  2029  		case ssaop.OpARM64LoweredPanicBoundsCC:
  2030  			c := v.Aux.(ssa.PanicBoundsCC).Cx
  2031  			if c >= 0 && c <= abi.BoundsMaxConst {
  2032  				xVal = int(c)
  2033  			} else {
  2034  				// Move constant to a register
  2035  				xIsReg = true
  2036  				p := s.Prog(arm64.AMOVD)
  2037  				p.From.Type = obj.TYPE_CONST
  2038  				p.From.Offset = c
  2039  				p.To.Type = obj.TYPE_REG
  2040  				p.To.Reg = arm64.REG_R0 + int16(xVal)
  2041  			}
  2042  			c = v.Aux.(ssa.PanicBoundsCC).Cy
  2043  			if c >= 0 && c <= abi.BoundsMaxConst {
  2044  				yVal = int(c)
  2045  			} else {
  2046  				// Move constant to a register
  2047  				yIsReg = true
  2048  				yVal = 1
  2049  				p := s.Prog(arm64.AMOVD)
  2050  				p.From.Type = obj.TYPE_CONST
  2051  				p.From.Offset = c
  2052  				p.To.Type = obj.TYPE_REG
  2053  				p.To.Reg = arm64.REG_R0 + int16(yVal)
  2054  			}
  2055  		}
  2056  		c := abi.BoundsEncode(code, signed, xIsReg, yIsReg, xVal, yVal)
  2057  
  2058  		p := s.Prog(obj.APCDATA)
  2059  		p.From.SetConst(abi.PCDATA_PanicBounds)
  2060  		p.To.SetConst(int64(c))
  2061  		p = s.Prog(obj.ACALL)
  2062  		p.To.Type = obj.TYPE_MEM
  2063  		p.To.Name = obj.NAME_EXTERN
  2064  		p.To.Sym = ir.Syms.PanicBounds
  2065  
  2066  	case ssaop.OpARM64LoweredNilCheck:
  2067  		// Issue a load which will fault if arg is nil.
  2068  		p := s.Prog(arm64.AMOVB)
  2069  		p.From.Type = obj.TYPE_MEM
  2070  		p.From.Reg = v.Args[0].Reg()
  2071  		ssagen.AddAux(&p.From, v)
  2072  		p.To.Type = obj.TYPE_REG
  2073  		p.To.Reg = arm64.REGTMP
  2074  		if logopt.Enabled() {
  2075  			logopt.LogOpt(v.Pos, "nilcheck", "genssa", v.Block.Func.Name)
  2076  		}
  2077  		if base.Debug.Nil != 0 && v.Pos.Line() > 1 { // v.Line==1 in generated wrappers
  2078  			base.WarnfAt(v.Pos, "generated nil check")
  2079  		}
  2080  	case ssaop.OpARM64Equal,
  2081  		ssaop.OpARM64NotEqual,
  2082  		ssaop.OpARM64LessThan,
  2083  		ssaop.OpARM64LessEqual,
  2084  		ssaop.OpARM64GreaterThan,
  2085  		ssaop.OpARM64GreaterEqual,
  2086  		ssaop.OpARM64LessThanU,
  2087  		ssaop.OpARM64LessEqualU,
  2088  		ssaop.OpARM64GreaterThanU,
  2089  		ssaop.OpARM64GreaterEqualU,
  2090  		ssaop.OpARM64LessThanF,
  2091  		ssaop.OpARM64LessEqualF,
  2092  		ssaop.OpARM64GreaterThanF,
  2093  		ssaop.OpARM64GreaterEqualF,
  2094  		ssaop.OpARM64NotLessThanF,
  2095  		ssaop.OpARM64NotLessEqualF,
  2096  		ssaop.OpARM64NotGreaterThanF,
  2097  		ssaop.OpARM64NotGreaterEqualF,
  2098  		ssaop.OpARM64LessThanNoov,
  2099  		ssaop.OpARM64GreaterEqualNoov:
  2100  		// generate boolean values using CSET
  2101  		//
  2102  		// If the result stops being a 0/1-producing CSET, update zeroUpperBits in ARM64Ops.go.
  2103  		p := s.Prog(arm64.ACSET)
  2104  		p.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset
  2105  		condCode := condBits[v.Op]
  2106  		p.From.Offset = int64(condCode)
  2107  		p.To.Type = obj.TYPE_REG
  2108  		p.To.Reg = v.Reg()
  2109  	case ssaop.OpARM64PRFM:
  2110  		p := s.Prog(v.Op.Asm())
  2111  		p.From.Type = obj.TYPE_MEM
  2112  		p.From.Reg = v.Args[0].Reg()
  2113  		p.To.Type = obj.TYPE_CONST
  2114  		p.To.Offset = v.AuxInt
  2115  	case ssaop.OpARM64LoweredGetClosurePtr:
  2116  		// Closure pointer is R26 (arm64.REGCTXT).
  2117  		ssagen.CheckLoweredGetClosurePtr(v)
  2118  	case ssaop.OpARM64LoweredGetCallerSP:
  2119  		// caller's SP is FixedFrameSize below the address of the first arg
  2120  		p := s.Prog(arm64.AMOVD)
  2121  		p.From.Type = obj.TYPE_ADDR
  2122  		p.From.Offset = -base.Ctxt.Arch.FixedFrameSize
  2123  		p.From.Name = obj.NAME_PARAM
  2124  		p.To.Type = obj.TYPE_REG
  2125  		p.To.Reg = v.Reg()
  2126  	case ssaop.OpARM64LoweredGetCallerPC:
  2127  		p := s.Prog(obj.AGETCALLERPC)
  2128  		p.To.Type = obj.TYPE_REG
  2129  		p.To.Reg = v.Reg()
  2130  	case ssaop.OpARM64DMB:
  2131  		p := s.Prog(v.Op.Asm())
  2132  		p.From.Type = obj.TYPE_CONST
  2133  		p.From.Offset = v.AuxInt
  2134  	case ssaop.OpARM64FlagConstant:
  2135  		v.Fatalf("FlagConstant op should never make it to codegen %v", v.LongString())
  2136  	case ssaop.OpARM64InvertFlags:
  2137  		v.Fatalf("InvertFlags should never make it to codegen %v", v.LongString())
  2138  	case ssaop.OpClobber:
  2139  		// MOVW	$0xdeaddead, REGTMP
  2140  		// MOVW	REGTMP, (slot)
  2141  		// MOVW	REGTMP, 4(slot)
  2142  		p := s.Prog(arm64.AMOVW)
  2143  		p.From.Type = obj.TYPE_CONST
  2144  		p.From.Offset = 0xdeaddead
  2145  		p.To.Type = obj.TYPE_REG
  2146  		p.To.Reg = arm64.REGTMP
  2147  		p = s.Prog(arm64.AMOVW)
  2148  		p.From.Type = obj.TYPE_REG
  2149  		p.From.Reg = arm64.REGTMP
  2150  		p.To.Type = obj.TYPE_MEM
  2151  		p.To.Reg = arm64.REGSP
  2152  		ssagen.AddAux(&p.To, v)
  2153  		p = s.Prog(arm64.AMOVW)
  2154  		p.From.Type = obj.TYPE_REG
  2155  		p.From.Reg = arm64.REGTMP
  2156  		p.To.Type = obj.TYPE_MEM
  2157  		p.To.Reg = arm64.REGSP
  2158  		ssagen.AddAux2(&p.To, v, v.AuxInt+4)
  2159  	case ssaop.OpClobberReg:
  2160  		x := uint64(0xdeaddeaddeaddead)
  2161  		p := s.Prog(arm64.AMOVD)
  2162  		p.From.Type = obj.TYPE_CONST
  2163  		p.From.Offset = int64(x)
  2164  		p.To.Type = obj.TYPE_REG
  2165  		p.To.Reg = v.Reg()
  2166  	default:
  2167  		if !ssaGenSIMDValue(s, v) && !ssaGenSIMDSVEValue(s, v) {
  2168  			v.Fatalf("genValue not implemented: %s", v.LongString())
  2169  		}
  2170  	}
  2171  }
  2172  
  2173  // simdZ21 emits an unpredicated SVE binary Z-register instruction with the given
  2174  // element arrangement, e.g. ZADD Z2.B, Z0.B, Z1.B.
  2175  func simdZ21(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2176  	p := s.Prog(v.Op.Asm())
  2177  	p.From.Type = obj.TYPE_REG
  2178  	p.From.Reg = zregArng(v.Args[1].Reg(), arng)        // Zm
  2179  	p.AddRestSourceReg(zregArng(v.Args[0].Reg(), arng)) // Zn
  2180  	p.To.Type = obj.TYPE_REG
  2181  	p.To.Reg = zregArng(v.Reg(), arng) // Zd
  2182  	return p
  2183  }
  2184  
  2185  // simdZ21List emits an unpredicated SVE binary Z-register instruction whose
  2186  // first source is a single-register byte list, e.g. ZTBL Z1.B, [Z0.B], Z2.B.
  2187  func simdZ21List(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2188  	if arng != arm64.ARNG_B {
  2189  		base.Fatalf("simdZ21List: unsupported arrangement %d", arng)
  2190  	}
  2191  	p := s.Prog(v.Op.Asm())
  2192  	p.From.Type = obj.TYPE_REG
  2193  	p.From.Reg = zregArng(v.Args[1].Reg(), arng) // Zm
  2194  	// RegisterListOffset wants the register's 0-31 index.
  2195  	regList, _ := arm64.RegisterListOffset(int(v.Args[0].Reg()&31), 1, regListArr("Z", "B"), 0)
  2196  	p.AddRestSource(obj.Addr{Type: obj.TYPE_REGLIST, Offset: regList}) // [Zn.T]
  2197  	p.To.Type = obj.TYPE_REG
  2198  	p.To.Reg = zregArng(v.Reg(), arng) // Zd
  2199  	return p
  2200  }
  2201  
  2202  // simdZDupConst emits an SVE broadcast of a signed 8-bit immediate to every
  2203  // lane with the given element arrangement, e.g. ZDUP $5, Z0.H.
  2204  func simdZDupConst(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2205  	p := s.Prog(v.Op.Asm())
  2206  	p.From.Type = obj.TYPE_CONST
  2207  	p.From.Offset = v.AuxInt
  2208  	p.To.Type = obj.TYPE_REG
  2209  	p.To.Reg = zregArng(v.Reg(), arng)
  2210  	return p
  2211  }
  2212  
  2213  // simdZDupGp emits an SVE broadcast of a general register to every lane with
  2214  // the given element arrangement, e.g. ZDUPW R0, Z0.B.
  2215  func simdZDupGp(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2216  	p := s.Prog(v.Op.Asm())
  2217  	p.From.Type = obj.TYPE_REG
  2218  	p.From.Reg = v.Args[0].Reg()
  2219  	p.To.Type = obj.TYPE_REG
  2220  	p.To.Reg = zregArng(v.Reg(), arng)
  2221  	return p
  2222  }
  2223  
  2224  // simdZDupIndexed emits an SVE broadcast of element auxint of a vector
  2225  // register to every lane, e.g. ZDUP Z0.S[0], Z1.S.
  2226  func simdZDupIndexed(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2227  	p := s.Prog(v.Op.Asm())
  2228  	p.From.Type = obj.TYPE_REG
  2229  	p.From.Reg = zregArngElem(v.Args[0].Reg(), arng)
  2230  	p.From.Index = int16(v.AuxInt)
  2231  	p.To.Type = obj.TYPE_REG
  2232  	p.To.Reg = zregArng(v.Reg(), arng)
  2233  	return p
  2234  }
  2235  
  2236  // simdZCpyConst emits a zeroing-predicated SVE broadcast of a signed 8-bit
  2237  // immediate, e.g. ZCPY $5, P0.Z, Z0.H.
  2238  func simdZCpyConst(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2239  	p := s.Prog(v.Op.Asm())
  2240  	p.From.Type = obj.TYPE_CONST
  2241  	p.From.Offset = v.AuxInt
  2242  	p.AddRestSourceReg(pregMask(v.Args[0].Reg(), arm64.PRED_Z))
  2243  	p.To.Type = obj.TYPE_REG
  2244  	p.To.Reg = zregArng(v.Reg(), arng)
  2245  	return p
  2246  }
  2247  
  2248  // simdZCpyMergingConst emits a merging-predicated SVE broadcast of a signed
  2249  // 8-bit immediate, e.g. ZCPY $5, P0.M, Z0.H. SSA provides arg0=the value the
  2250  // inactive lanes keep (pinned to the destination by resultInArg0) and
  2251  // arg1=the governing predicate.
  2252  func simdZCpyMergingConst(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2253  	p := s.Prog(v.Op.Asm())
  2254  	p.From.Type = obj.TYPE_CONST
  2255  	p.From.Offset = v.AuxInt
  2256  	p.AddRestSourceReg(pregMask(v.Args[1].Reg(), arm64.PRED_M))
  2257  	p.To.Type = obj.TYPE_REG
  2258  	p.To.Reg = zregArng(v.Reg(), arng)
  2259  	return p
  2260  }
  2261  
  2262  // simdZCpyMergingGp emits a merging-predicated SVE broadcast of a general
  2263  // register, e.g. ZCPYW R0, P0.M, Z0.B. SSA provides arg0=the value the
  2264  // inactive lanes keep (pinned to the destination by resultInArg0),
  2265  // arg1=scalar, and arg2=the governing predicate.
  2266  func simdZCpyMergingGp(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2267  	p := s.Prog(v.Op.Asm())
  2268  	p.From.Type = obj.TYPE_REG
  2269  	p.From.Reg = v.Args[1].Reg()
  2270  	p.AddRestSourceReg(pregMask(v.Args[2].Reg(), arm64.PRED_M))
  2271  	p.To.Type = obj.TYPE_REG
  2272  	p.To.Reg = zregArng(v.Reg(), arng)
  2273  	return p
  2274  }
  2275  
  2276  // simdZCpyMergingVec emits a merging-predicated SVE broadcast of element 0 of
  2277  // a SIMD/FP register, e.g. ZCPYS V1, P0.M, Z0.S. SSA provides arg0=the value
  2278  // the inactive lanes keep (pinned to the destination by resultInArg0),
  2279  // arg1=scalar, and arg2=the governing predicate.
  2280  func simdZCpyMergingVec(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2281  	p := s.Prog(v.Op.Asm())
  2282  	p.From.Type = obj.TYPE_REG
  2283  	p.From.Reg = v.Args[1].Reg() - arm64.REG_F0 + arm64.REG_V0
  2284  	p.AddRestSourceReg(pregMask(v.Args[2].Reg(), arm64.PRED_M))
  2285  	p.To.Type = obj.TYPE_REG
  2286  	p.To.Reg = zregArng(v.Reg(), arng)
  2287  	return p
  2288  }
  2289  
  2290  // simdZ2kv emits an SVE instruction that takes a predicate as a plain data
  2291  // operand rather than a governing predicate, e.g. ZSEL Z1.B, Z0.B, P0, Z2.B.
  2292  // Unlike a predicated instruction it is constructive: the destination is
  2293  // independent of the sources, so it needs no MOVPRFX. SSA provides arg0=x (kept
  2294  // where the predicate is true), arg1=y (kept where false) and arg2=the predicate.
  2295  func simdZ2kv(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2296  	p := s.Prog(v.Op.Asm())
  2297  	p.From.Type = obj.TYPE_REG
  2298  	p.From.Reg = zregArng(v.Args[1].Reg(), arng)        // Zm
  2299  	p.AddRestSourceReg(zregArng(v.Args[0].Reg(), arng)) // Zn
  2300  	p.AddRestSourceReg(v.Args[2].Reg())                 // Pv, unqualified
  2301  	p.To.Type = obj.TYPE_REG
  2302  	p.To.Reg = zregArng(v.Reg(), arng) // Zd
  2303  	return p
  2304  }
  2305  
  2306  // simdZ2kvPred emits an SVE predicated binary operation, e.g. ZADD Z1.B, Z0.B, P0.M,
  2307  // Z0.B. These instructions are destructive: the destination is also the first
  2308  // source. How the destination is put in place depends on what the register
  2309  // allocator chose:
  2310  //
  2311  //   - dst == arg0: already destructive-ready, emit the instruction alone.
  2312  //   - dst == arg1: only reachable for a commutative operation (a
  2313  //     non-commutative one is marked resultInArg0, which pins dst to arg0), so
  2314  //     swap the sources and it becomes the case above.
  2315  //   - anything else: prefix MOVPRFX, which hints the hardware to fuse the pair
  2316  //     into one constructive operation and leaves both sources intact.
  2317  func simdZ2kvPred(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2318  	x, y := v.Args[0].Reg(), v.Args[1].Reg()
  2319  	d := v.Reg()
  2320  	switch d {
  2321  	case x:
  2322  	case y:
  2323  		// Only reachable for a commutative operation: a non-commutative one is
  2324  		// marked resultInArg0, which pins the destination to arg0. Swapping the
  2325  		// sources of one would compute y-x where x-y was asked for.
  2326  		if v.Op.ResultInArg0() {
  2327  			v.Fatalf("destination is arg1 of resultInArg0 op %v", v.Op)
  2328  		}
  2329  		x, y = y, x
  2330  	default:
  2331  		mp := s.Prog(arm64.AZMOVPRFX)
  2332  		mp.From.Type = obj.TYPE_REG
  2333  		mp.From.Reg = pzreg(x)
  2334  		mp.To.Type = obj.TYPE_REG
  2335  		mp.To.Reg = pzreg(d)
  2336  	}
  2337  	p := s.Prog(v.Op.Asm())
  2338  	p.From.Type = obj.TYPE_REG
  2339  	p.From.Reg = zregArng(y, arng)                              // Zm
  2340  	p.AddRestSourceReg(zregArng(d, arng))                       // Zdn
  2341  	p.AddRestSourceReg(pregMask(v.Args[2].Reg(), arm64.PRED_M)) // Pg/M
  2342  	p.To.Type = obj.TYPE_REG
  2343  	p.To.Reg = zregArng(d, arng) // Zdn
  2344  	return p
  2345  }
  2346  
  2347  // simdZkv emits a predicated-only SVE unary operation standing in for an
  2348  // unpredicated one, e.g. ZABS Z0.B, P0.M, Z1.B. SSA provides arg0=x and
  2349  // arg1=the governing predicate, which the lowering rule synthesized as
  2350  // all-true. With every lane active, merging predication leaves nothing of the
  2351  // destination behind, so the operation is unpredicated in effect.
  2352  func simdZkv(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2353  	return sveUnaryPred(s, v, arng, v.Args[0].Reg(), v.Args[1].Reg(), arm64.PRED_M)
  2354  }
  2355  
  2356  // simdZkfp emits an SVE reduction across the lanes of arg0 that arg1 governs
  2357  // into a floating-point scalar register, e.g. ZFADDVS Z0.S, P0, V1. The
  2358  // scalar is the low lane of the vector register with the same number, which
  2359  // is where a float value lives.
  2360  func simdZkfp(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2361  	p := s.Prog(v.Op.Asm())
  2362  	p.From.Type = obj.TYPE_REG
  2363  	p.From.Reg = zregArng(v.Args[0].Reg(), arng) // Zn
  2364  	p.AddRestSourceReg(v.Args[1].Reg())          // Pg, unqualified
  2365  	p.To.Type = obj.TYPE_REG
  2366  	p.To.Reg = v.Reg() - arm64.REG_F0 + arm64.REG_V0 // Vd
  2367  	return p
  2368  }
  2369  
  2370  // simdZ11Imm emits an unpredicated SVE operation on one vector and the
  2371  // immediate in the aux, e.g. ZLSL $3, Z0.S, Z1.S.
  2372  func simdZ11Imm(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2373  	p := s.Prog(v.Op.Asm())
  2374  	p.From.Type = obj.TYPE_CONST
  2375  	p.From.Offset = int64(v.AuxUInt8())
  2376  	p.AddRestSourceReg(zregArng(v.Args[0].Reg(), arng)) // Zn
  2377  	p.To.Type = obj.TYPE_REG
  2378  	p.To.Reg = zregArng(v.Reg(), arng) // Zd
  2379  	return p
  2380  }
  2381  
  2382  // simdZkvPredImm emits the predicated form of an SVE operation on one vector
  2383  // and the immediate in the aux, e.g. ZLSL $3, Z0.S, P0.M, Z0.S. It is
  2384  // destructive, so resultInArg0 has already put the destination in arg0.
  2385  func simdZkvPredImm(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2386  	d := v.Reg()
  2387  	if d != v.Args[0].Reg() {
  2388  		v.Fatalf("destination is not arg0 of destructive op %v", v.Op)
  2389  	}
  2390  	p := s.Prog(v.Op.Asm())
  2391  	p.From.Type = obj.TYPE_CONST
  2392  	p.From.Offset = int64(v.AuxUInt8())
  2393  	p.AddRestSourceReg(zregArng(d, arng))                       // Zdn
  2394  	p.AddRestSourceReg(pregMask(v.Args[1].Reg(), arm64.PRED_M)) // Pg/M
  2395  	p.To.Type = obj.TYPE_REG
  2396  	p.To.Reg = zregArng(d, arng) // Zdn
  2397  	return p
  2398  }
  2399  
  2400  // simdZ21Wide emits an unpredicated SVE "wide elements" operation, whose
  2401  // second source has 64-bit elements, e.g. ZLSL Z2.D, Z0.S, Z1.S: each element
  2402  // of arg0 is combined with the 64-bit element of arg1 that overlaps it.
  2403  func simdZ21Wide(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2404  	p := s.Prog(v.Op.Asm())
  2405  	p.From.Type = obj.TYPE_REG
  2406  	p.From.Reg = zregArng(v.Args[1].Reg(), arm64.ARNG_D) // Zm
  2407  	p.AddRestSourceReg(zregArng(v.Args[0].Reg(), arng))  // Zn
  2408  	p.To.Type = obj.TYPE_REG
  2409  	p.To.Reg = zregArng(v.Reg(), arng) // Zd
  2410  	return p
  2411  }
  2412  
  2413  // simdZ2kvPredWide emits the predicated form of an SVE "wide elements"
  2414  // operation, e.g. ZLSL Z1.D, Z0.S, P0.M, Z0.S. It is destructive and not
  2415  // commutative, so resultInArg0 has already put the destination in arg0.
  2416  func simdZ2kvPredWide(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2417  	d := v.Reg()
  2418  	if d != v.Args[0].Reg() {
  2419  		v.Fatalf("destination is not arg0 of destructive op %v", v.Op)
  2420  	}
  2421  	p := s.Prog(v.Op.Asm())
  2422  	p.From.Type = obj.TYPE_REG
  2423  	p.From.Reg = zregArng(v.Args[1].Reg(), arm64.ARNG_D)        // Zm
  2424  	p.AddRestSourceReg(zregArng(d, arng))                       // Zdn
  2425  	p.AddRestSourceReg(pregMask(v.Args[2].Reg(), arm64.PRED_M)) // Pg/M
  2426  	p.To.Type = obj.TYPE_REG
  2427  	p.To.Reg = zregArng(d, arng) // Zdn
  2428  	return p
  2429  }
  2430  
  2431  // simdZ2kvPredResultInArg0 emits the merging form of a predicated SVE unary
  2432  // operation, e.g. ZABS Z1.B, P0.M, Z0.B. SSA provides arg0=the value the
  2433  // inactive lanes keep, arg1=x, arg2=mask, and resultInArg0 puts that value in
  2434  // the destination. The instruction is constructive -- it names its destination
  2435  // apart from its source -- so merging into that destination needs no MOVPRFX.
  2436  func simdZ2kvPredResultInArg0(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2437  	return sveUnaryPred(s, v, arng, v.Args[1].Reg(), v.Args[2].Reg(), arm64.PRED_M)
  2438  }
  2439  
  2440  // sveUnaryPred emits a predicated SVE unary operation: OP Zn.T, Pg/<qual>, Zd.T.
  2441  func sveUnaryPred(s *ssagen.State, v *ssa.Value, arng int16, zn, pg int16, qual int16) *obj.Prog {
  2442  	p := s.Prog(v.Op.Asm())
  2443  	p.From.Type = obj.TYPE_REG
  2444  	p.From.Reg = zregArng(zn, arng)        // Zn
  2445  	p.AddRestSourceReg(pregMask(pg, qual)) // Pg/M or Pg/Z
  2446  	p.To.Type = obj.TYPE_REG
  2447  	p.To.Reg = zregArng(v.Reg(), arng) // Zd
  2448  	return p
  2449  }
  2450  
  2451  // simdZ3kvPredAcc emits a predicated SVE accumulating operation, e.g.
  2452  // ZFMLA Zm.S, Zn.S, P0.M, Zda.S. SSA provides arg0=the accumulator, arg1=x,
  2453  // arg2=y, arg3=the governing predicate; the accumulator is the register the
  2454  // instruction merges into, so resultInArg0 pins the destination to it.
  2455  //
  2456  // This helper is only for <Zda>-style operations, whose destination is not
  2457  // repeated in the assembly: it emits one more distinct register than the
  2458  // <Zdn>-destructive helpers (simdZ2kvPred, simdZ3kvPredResultInArg0) do.
  2459  func simdZ3kvPredAcc(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2460  	p := s.Prog(v.Op.Asm())
  2461  	p.From.Type = obj.TYPE_REG
  2462  	p.From.Reg = zregArng(v.Args[2].Reg(), arng)                // Zm
  2463  	p.AddRestSourceReg(zregArng(v.Args[1].Reg(), arng))         // Zn
  2464  	p.AddRestSourceReg(pregMask(v.Args[3].Reg(), arm64.PRED_M)) // Pg/M
  2465  	p.To.Type = obj.TYPE_REG
  2466  	p.To.Reg = zregArng(v.Reg(), arng) // Zda
  2467  	return p
  2468  }
  2469  
  2470  // simdZ3kvPredResultInArg0 emits an SVE merging-predicated binary operation
  2471  // whose inactive lanes come from a value that is neither of its sources, e.g.
  2472  // x.Add(y).IfElse(mask, z). SSA provides arg0=z, arg1=x, arg2=y, arg3=mask, and
  2473  // resultInArg0 puts z in the destination register.
  2474  //
  2475  //	ZMOVPRFX Zx, Pg/M, Zd    // Zd = x on the active lanes, z on the rest
  2476  //	ZADD     Zy, Zd, Pg/M, Zd // Zd = x+y on the active lanes, z on the rest
  2477  //
  2478  // The prefix must be the predicated MOVPRFX, not the unpredicated one: the
  2479  // whole-register form would leave x rather than z in the inactive lanes.
  2480  //
  2481  // A prefixed instruction may not name its destination in any operand position
  2482  // other than the destructive one, so Zd must differ from the operation's Zm.
  2483  // Only a commutative operation gets this lowering (see sveMergingPrefixedOp in
  2484  // simdgen), so when the register allocator puts the destination on one source —
  2485  // which it can only do when z is that source — the other one becomes Zm and the
  2486  // prefix is unnecessary.
  2487  func simdZ3kvPredResultInArg0(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2488  	d := v.Reg()
  2489  	x, y := v.Args[1].Reg(), v.Args[2].Reg()
  2490  	pg := v.Args[3].Reg()
  2491  	zn, zm := x, y
  2492  	if d == zm {
  2493  		zn, zm = zm, zn
  2494  	}
  2495  	if d != zn {
  2496  		mp := s.Prog(arm64.AZMOVPRFX)
  2497  		mp.From.Type = obj.TYPE_REG
  2498  		mp.From.Reg = zregArng(zn, arng)
  2499  		mp.AddRestSourceReg(pregMask(pg, arm64.PRED_M))
  2500  		mp.To.Type = obj.TYPE_REG
  2501  		mp.To.Reg = zregArng(d, arng)
  2502  	}
  2503  	p := s.Prog(v.Op.Asm())
  2504  	p.From.Type = obj.TYPE_REG
  2505  	p.From.Reg = zregArng(zm, arng)                // Zm
  2506  	p.AddRestSourceReg(zregArng(d, arng))          // Zdn
  2507  	p.AddRestSourceReg(pregMask(pg, arm64.PRED_M)) // Pg/M
  2508  	p.To.Type = obj.TYPE_REG
  2509  	p.To.Reg = zregArng(d, arng) // Zdn
  2510  	return p
  2511  }
  2512  
  2513  // simdPNext emits a PNEXT with the given element arrangement, e.g.
  2514  // PPNEXT P0.H, P1, P0.H. SSA provides arg0=the predicate the destination
  2515  // overwrites (resultInArg0) and arg1=the candidate lanes.
  2516  func simdPNext(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2517  	p := s.Prog(v.Op.Asm())
  2518  	p.From.Type = obj.TYPE_REG
  2519  	p.From.Reg = pregArng(v.Args[0].Reg(), arng)
  2520  	p.AddRestSourceReg(v.Args[1].Reg())
  2521  	p.To.Type = obj.TYPE_REG
  2522  	p.To.Reg = pregArng(v.Reg(), arng)
  2523  	return p
  2524  }
  2525  
  2526  // simdPWHILELT emits a PWHILELT that fills a predicate with lanes [lo,hi) set for
  2527  // the given element arrangement, e.g. PWHILELT R0, R1, P0.B. SSA provides
  2528  // arg0=lo, arg1=hi.
  2529  func simdPWHILELT(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2530  	p := s.Prog(v.Op.Asm())
  2531  	p.From.Type = obj.TYPE_REG
  2532  	p.From.Reg = v.Args[1].Reg()
  2533  	p.AddRestSourceReg(v.Args[0].Reg())
  2534  	p.To.Type = obj.TYPE_REG
  2535  	p.To.Reg = pregArng(v.Reg0(), arng)
  2536  	return p
  2537  }
  2538  
  2539  // simdZ2kk emits a predicated SVE integer compare that produces a predicate
  2540  // mask, e.g. ZCMPGT Z1.B, Z0.B, P0.Z, P1.B. SSA provides arg0=x (Zn), arg1=y
  2541  // (Zm) and arg2=governing predicate (Pg); the result is the predicate mask Pd.
  2542  func simdZ2kk(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
  2543  	p := s.Prog(v.Op.Asm())
  2544  	p.From.Type = obj.TYPE_REG
  2545  	p.From.Reg = zregArng(v.Args[1].Reg(), arng)                // Zm
  2546  	p.AddRestSourceReg(zregArng(v.Args[0].Reg(), arng))         // Zn
  2547  	p.AddRestSourceReg(pregMask(v.Args[2].Reg(), arm64.PRED_Z)) // Pg/Z
  2548  	p.To.Type = obj.TYPE_REG
  2549  	p.To.Reg = pregArng(v.Reg(), arng) // Pd
  2550  	return p
  2551  }
  2552  
  2553  var condBits = map[ssaop.Op]arm64.SpecialOperand{
  2554  	ssaop.OpARM64Equal:         arm64.SPOP_EQ,
  2555  	ssaop.OpARM64NotEqual:      arm64.SPOP_NE,
  2556  	ssaop.OpARM64LessThan:      arm64.SPOP_LT,
  2557  	ssaop.OpARM64LessThanU:     arm64.SPOP_LO,
  2558  	ssaop.OpARM64LessEqual:     arm64.SPOP_LE,
  2559  	ssaop.OpARM64LessEqualU:    arm64.SPOP_LS,
  2560  	ssaop.OpARM64GreaterThan:   arm64.SPOP_GT,
  2561  	ssaop.OpARM64GreaterThanU:  arm64.SPOP_HI,
  2562  	ssaop.OpARM64GreaterEqual:  arm64.SPOP_GE,
  2563  	ssaop.OpARM64GreaterEqualU: arm64.SPOP_HS,
  2564  	ssaop.OpARM64LessThanF:     arm64.SPOP_MI, // Less than
  2565  	ssaop.OpARM64LessEqualF:    arm64.SPOP_LS, // Less than or equal to
  2566  	ssaop.OpARM64GreaterThanF:  arm64.SPOP_GT, // Greater than
  2567  	ssaop.OpARM64GreaterEqualF: arm64.SPOP_GE, // Greater than or equal to
  2568  
  2569  	// The following condition codes have unordered to handle comparisons related to NaN.
  2570  	ssaop.OpARM64NotLessThanF:     arm64.SPOP_PL, // Greater than, equal to, or unordered
  2571  	ssaop.OpARM64NotLessEqualF:    arm64.SPOP_HI, // Greater than or unordered
  2572  	ssaop.OpARM64NotGreaterThanF:  arm64.SPOP_LE, // Less than, equal to or unordered
  2573  	ssaop.OpARM64NotGreaterEqualF: arm64.SPOP_LT, // Less than or unordered
  2574  
  2575  	ssaop.OpARM64LessThanNoov:     arm64.SPOP_MI, // Less than but without honoring overflow
  2576  	ssaop.OpARM64GreaterEqualNoov: arm64.SPOP_PL, // Greater than or equal to but without honoring overflow
  2577  }
  2578  
  2579  var blockJump = map[block.BlockKind]struct {
  2580  	asm, invasm obj.As
  2581  }{
  2582  	block.BlockARM64EQ:     {arm64.ABEQ, arm64.ABNE},
  2583  	block.BlockARM64NE:     {arm64.ABNE, arm64.ABEQ},
  2584  	block.BlockARM64LT:     {arm64.ABLT, arm64.ABGE},
  2585  	block.BlockARM64GE:     {arm64.ABGE, arm64.ABLT},
  2586  	block.BlockARM64LE:     {arm64.ABLE, arm64.ABGT},
  2587  	block.BlockARM64GT:     {arm64.ABGT, arm64.ABLE},
  2588  	block.BlockARM64ULT:    {arm64.ABLO, arm64.ABHS},
  2589  	block.BlockARM64UGE:    {arm64.ABHS, arm64.ABLO},
  2590  	block.BlockARM64UGT:    {arm64.ABHI, arm64.ABLS},
  2591  	block.BlockARM64ULE:    {arm64.ABLS, arm64.ABHI},
  2592  	block.BlockARM64Z:      {arm64.ACBZ, arm64.ACBNZ},
  2593  	block.BlockARM64NZ:     {arm64.ACBNZ, arm64.ACBZ},
  2594  	block.BlockARM64ZW:     {arm64.ACBZW, arm64.ACBNZW},
  2595  	block.BlockARM64NZW:    {arm64.ACBNZW, arm64.ACBZW},
  2596  	block.BlockARM64TBZ:    {arm64.ATBZ, arm64.ATBNZ},
  2597  	block.BlockARM64TBNZ:   {arm64.ATBNZ, arm64.ATBZ},
  2598  	block.BlockARM64FLT:    {arm64.ABMI, arm64.ABPL},
  2599  	block.BlockARM64FGE:    {arm64.ABGE, arm64.ABLT},
  2600  	block.BlockARM64FLE:    {arm64.ABLS, arm64.ABHI},
  2601  	block.BlockARM64FGT:    {arm64.ABGT, arm64.ABLE},
  2602  	block.BlockARM64LTnoov: {arm64.ABMI, arm64.ABPL},
  2603  	block.BlockARM64GEnoov: {arm64.ABPL, arm64.ABMI},
  2604  }
  2605  
  2606  // To model a 'LEnoov' ('<=' without overflow checking) branching.
  2607  var leJumps = [2][2]ssagen.IndexJump{
  2608  	{{Jump: arm64.ABEQ, Index: 0}, {Jump: arm64.ABPL, Index: 1}}, // next == b.Succs[0]
  2609  	{{Jump: arm64.ABMI, Index: 0}, {Jump: arm64.ABEQ, Index: 0}}, // next == b.Succs[1]
  2610  }
  2611  
  2612  // To model a 'GTnoov' ('>' without overflow checking) branching.
  2613  var gtJumps = [2][2]ssagen.IndexJump{
  2614  	{{Jump: arm64.ABMI, Index: 1}, {Jump: arm64.ABEQ, Index: 1}}, // next == b.Succs[0]
  2615  	{{Jump: arm64.ABEQ, Index: 1}, {Jump: arm64.ABPL, Index: 0}}, // next == b.Succs[1]
  2616  }
  2617  
  2618  func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
  2619  	switch b.Kind {
  2620  	case block.BlockPlain, block.BlockDefer:
  2621  		if b.Succs[0].Block() != next {
  2622  			p := s.Prog(obj.AJMP)
  2623  			p.To.Type = obj.TYPE_BRANCH
  2624  			s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
  2625  		}
  2626  
  2627  	case block.BlockExit, block.BlockRetJmp:
  2628  
  2629  	case block.BlockRet:
  2630  		s.Prog(obj.ARET)
  2631  
  2632  	case block.BlockARM64EQ, block.BlockARM64NE,
  2633  		block.BlockARM64LT, block.BlockARM64GE,
  2634  		block.BlockARM64LE, block.BlockARM64GT,
  2635  		block.BlockARM64ULT, block.BlockARM64UGT,
  2636  		block.BlockARM64ULE, block.BlockARM64UGE,
  2637  		block.BlockARM64Z, block.BlockARM64NZ,
  2638  		block.BlockARM64ZW, block.BlockARM64NZW,
  2639  		block.BlockARM64FLT, block.BlockARM64FGE,
  2640  		block.BlockARM64FLE, block.BlockARM64FGT,
  2641  		block.BlockARM64LTnoov, block.BlockARM64GEnoov:
  2642  		jmp := blockJump[b.Kind]
  2643  		var p *obj.Prog
  2644  		switch next {
  2645  		case b.Succs[0].Block():
  2646  			p = s.Br(jmp.invasm, b.Succs[1].Block())
  2647  		case b.Succs[1].Block():
  2648  			p = s.Br(jmp.asm, b.Succs[0].Block())
  2649  		default:
  2650  			if b.Likely != ssa.BranchUnlikely {
  2651  				p = s.Br(jmp.asm, b.Succs[0].Block())
  2652  				s.Br(obj.AJMP, b.Succs[1].Block())
  2653  			} else {
  2654  				p = s.Br(jmp.invasm, b.Succs[1].Block())
  2655  				s.Br(obj.AJMP, b.Succs[0].Block())
  2656  			}
  2657  		}
  2658  		if !b.Controls[0].Type.IsFlags() {
  2659  			p.From.Type = obj.TYPE_REG
  2660  			p.From.Reg = b.Controls[0].Reg()
  2661  		}
  2662  	case block.BlockARM64TBZ, block.BlockARM64TBNZ:
  2663  		jmp := blockJump[b.Kind]
  2664  		var p *obj.Prog
  2665  		switch next {
  2666  		case b.Succs[0].Block():
  2667  			p = s.Br(jmp.invasm, b.Succs[1].Block())
  2668  		case b.Succs[1].Block():
  2669  			p = s.Br(jmp.asm, b.Succs[0].Block())
  2670  		default:
  2671  			if b.Likely != ssa.BranchUnlikely {
  2672  				p = s.Br(jmp.asm, b.Succs[0].Block())
  2673  				s.Br(obj.AJMP, b.Succs[1].Block())
  2674  			} else {
  2675  				p = s.Br(jmp.invasm, b.Succs[1].Block())
  2676  				s.Br(obj.AJMP, b.Succs[0].Block())
  2677  			}
  2678  		}
  2679  		p.From.Offset = b.AuxInt
  2680  		p.From.Type = obj.TYPE_CONST
  2681  		p.Reg = b.Controls[0].Reg()
  2682  
  2683  	case block.BlockARM64LEnoov:
  2684  		s.CombJump(b, next, &leJumps)
  2685  	case block.BlockARM64GTnoov:
  2686  		s.CombJump(b, next, &gtJumps)
  2687  
  2688  	case block.BlockARM64JUMPTABLE:
  2689  		// MOVD	(TABLE)(IDX<<3), Rtmp
  2690  		// JMP	(Rtmp)
  2691  		p := s.Prog(arm64.AMOVD)
  2692  		p.From = genIndexedOperand(ssaop.OpARM64MOVDloadidx8, b.Controls[1].Reg(), b.Controls[0].Reg())
  2693  		p.To.Type = obj.TYPE_REG
  2694  		p.To.Reg = arm64.REGTMP
  2695  		p = s.Prog(obj.AJMP)
  2696  		p.To.Type = obj.TYPE_MEM
  2697  		p.To.Reg = arm64.REGTMP
  2698  		// Save jump tables for later resolution of the target blocks.
  2699  		s.JumpTables = append(s.JumpTables, b)
  2700  
  2701  	default:
  2702  		b.Fatalf("branch not implemented: %s", b.LongString())
  2703  	}
  2704  }
  2705  
  2706  func loadRegResult(s *ssagen.State, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  2707  	p := s.Prog(loadByType(t))
  2708  	p.From.Type = obj.TYPE_MEM
  2709  	p.From.Name = obj.NAME_AUTO
  2710  	p.From.Sym = n.Linksym()
  2711  	p.From.Offset = n.FrameOffset() + off
  2712  	p.To.Type = obj.TYPE_REG
  2713  	p.To.Reg = reg
  2714  	return p
  2715  }
  2716  
  2717  func spillArgReg(pp *objw.Progs, p *obj.Prog, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  2718  	p = pp.Append(p, storeByType(t), obj.TYPE_REG, reg, 0, obj.TYPE_MEM, 0, n.FrameOffset()+off)
  2719  	p.To.Name = obj.NAME_PARAM
  2720  	p.To.Sym = n.Linksym()
  2721  	p.Pos = p.Pos.WithNotStmt()
  2722  	return p
  2723  }
  2724  
  2725  // zero16 zeroes 16 bytes at reg+off.
  2726  // If postInc is true, increment reg by 16.
  2727  func zero16(s *ssagen.State, reg int16, off int64, postInc bool) {
  2728  	//   STP     (ZR, ZR), off(reg)
  2729  	p := s.Prog(arm64.ASTP)
  2730  	p.From.Type = obj.TYPE_REGREG
  2731  	p.From.Reg = arm64.REGZERO
  2732  	p.From.Offset = int64(arm64.REGZERO)
  2733  	p.To.Type = obj.TYPE_MEM
  2734  	p.To.Reg = reg
  2735  	p.To.Offset = off
  2736  	if postInc {
  2737  		if off != 0 {
  2738  			panic("can't postinc with non-zero offset")
  2739  		}
  2740  		//   STP.P  (ZR, ZR), 16(reg)
  2741  		p.Scond = arm64.C_XPOST
  2742  		p.To.Offset = 16
  2743  	}
  2744  }
  2745  
  2746  // zero8 zeroes 8 bytes at reg+off.
  2747  func zero8(s *ssagen.State, reg int16, off int64) {
  2748  	//   MOVD     ZR, off(reg)
  2749  	p := s.Prog(arm64.AMOVD)
  2750  	p.From.Type = obj.TYPE_REG
  2751  	p.From.Reg = arm64.REGZERO
  2752  	p.To.Type = obj.TYPE_MEM
  2753  	p.To.Reg = reg
  2754  	p.To.Offset = off
  2755  }
  2756  
  2757  // move32 copies 32 bytes at src+off to dst+off.
  2758  // Uses registers tmp1 and tmp2.
  2759  // If postInc is true, increment src and dst by 32.
  2760  func move32(s *ssagen.State, src, dst, tmp1, tmp2 int16, off int64, postInc bool) {
  2761  	// FLDPQ   off(src), (tmp1, tmp2)
  2762  	ld := s.Prog(arm64.AFLDPQ)
  2763  	ld.From.Type = obj.TYPE_MEM
  2764  	ld.From.Reg = src
  2765  	ld.From.Offset = off
  2766  	ld.To.Type = obj.TYPE_REGREG
  2767  	ld.To.Reg = tmp1
  2768  	ld.To.Offset = int64(tmp2)
  2769  	// FSTPQ   (tmp1, tmp2), off(dst)
  2770  	st := s.Prog(arm64.AFSTPQ)
  2771  	st.From.Type = obj.TYPE_REGREG
  2772  	st.From.Reg = tmp1
  2773  	st.From.Offset = int64(tmp2)
  2774  	st.To.Type = obj.TYPE_MEM
  2775  	st.To.Reg = dst
  2776  	st.To.Offset = off
  2777  	if postInc {
  2778  		if off != 0 {
  2779  			panic("can't postinc with non-zero offset")
  2780  		}
  2781  		ld.Scond = arm64.C_XPOST
  2782  		st.Scond = arm64.C_XPOST
  2783  		ld.From.Offset = 32
  2784  		st.To.Offset = 32
  2785  	}
  2786  }
  2787  
  2788  // move16 copies 16 bytes at src+off to dst+off.
  2789  // Uses register tmp1
  2790  // If postInc is true, increment src and dst by 16.
  2791  func move16(s *ssagen.State, src, dst, tmp1 int16, off int64, postInc bool) {
  2792  	// FMOVQ     off(src), tmp1
  2793  	ld := s.Prog(arm64.AFMOVQ)
  2794  	ld.From.Type = obj.TYPE_MEM
  2795  	ld.From.Reg = src
  2796  	ld.From.Offset = off
  2797  	ld.To.Type = obj.TYPE_REG
  2798  	ld.To.Reg = tmp1
  2799  	// FMOVQ     tmp1, off(dst)
  2800  	st := s.Prog(arm64.AFMOVQ)
  2801  	st.From.Type = obj.TYPE_REG
  2802  	st.From.Reg = tmp1
  2803  	st.To.Type = obj.TYPE_MEM
  2804  	st.To.Reg = dst
  2805  	st.To.Offset = off
  2806  	if postInc {
  2807  		if off != 0 {
  2808  			panic("can't postinc with non-zero offset")
  2809  		}
  2810  		ld.Scond = arm64.C_XPOST
  2811  		st.Scond = arm64.C_XPOST
  2812  		ld.From.Offset = 16
  2813  		st.To.Offset = 16
  2814  	}
  2815  }
  2816  
  2817  // move8 copies 8 bytes at src+off to dst+off.
  2818  // Uses register tmp.
  2819  func move8(s *ssagen.State, src, dst, tmp int16, off int64) {
  2820  	// MOVD    off(src), tmp
  2821  	ld := s.Prog(arm64.AMOVD)
  2822  	ld.From.Type = obj.TYPE_MEM
  2823  	ld.From.Reg = src
  2824  	ld.From.Offset = off
  2825  	ld.To.Type = obj.TYPE_REG
  2826  	ld.To.Reg = tmp
  2827  	// MOVD    tmp, off(dst)
  2828  	st := s.Prog(arm64.AMOVD)
  2829  	st.From.Type = obj.TYPE_REG
  2830  	st.From.Reg = tmp
  2831  	st.To.Type = obj.TYPE_MEM
  2832  	st.To.Reg = dst
  2833  	st.To.Offset = off
  2834  }
  2835  
  2836  func zregArng(r int16, arng int16) int16 {
  2837  	if r >= arm64.REG_F0 && r <= arm64.REG_F31 &&
  2838  		arng >= arm64.ARNG_B && arng <= arm64.ARNG_Q {
  2839  		return arm64.REG_ZARNG + (r - arm64.REG_F0) | (arng << 5)
  2840  	}
  2841  	panic("Bad Z reg with arrangement")
  2842  }
  2843  
  2844  // zregArngElem is the element-indexed counterpart of zregArng: it encodes
  2845  // Zn.<T> ready for an index (Zn.<T>[i], with the index carried separately in
  2846  // the operand's Index field).
  2847  func zregArngElem(r int16, arng int16) int16 {
  2848  	if r >= arm64.REG_F0 && r <= arm64.REG_F31 &&
  2849  		arng >= arm64.ARNG_B && arng <= arm64.ARNG_Q {
  2850  		return arm64.REG_ZARNGELEM + (r - arm64.REG_F0) | (arng << 5)
  2851  	}
  2852  	panic("Bad Z reg with arrangement")
  2853  }
  2854  
  2855  func pregArng(r int16, mode int16) int16 {
  2856  	if r >= arm64.REG_P0 && r <= arm64.REG_P15 &&
  2857  		mode >= arm64.ARNG_B && mode <= arm64.ARNG_Q {
  2858  		return arm64.REG_PARNGZM + (r - arm64.REG_P0) | (mode << 5)
  2859  	}
  2860  	panic("Bad P reg with arrangement")
  2861  }
  2862  
  2863  func pregMask(r int16, mode int16) int16 {
  2864  	if r >= arm64.REG_P0 && r <= arm64.REG_P15 &&
  2865  		mode >= arm64.PRED_M && mode <= arm64.PRED_Z {
  2866  		return arm64.REG_PARNGZM + (r - arm64.REG_P0) | (mode << 5)
  2867  	}
  2868  	panic("Bad P reg with arrangement")
  2869  }
  2870  
  2871  func pzreg(r int16) int16 {
  2872  	if r >= arm64.REG_F0 && r <= arm64.REG_F31 {
  2873  		return r - arm64.REG_F0 + arm64.REG_Z0
  2874  	} else if r >= arm64.REG_P0 && r <= arm64.REG_P15 {
  2875  		return r
  2876  	}
  2877  	panic("Bad P or Z reg")
  2878  }
  2879  
  2880  // regListArr constructs an vector register arrangement in a register list.
  2881  func regListArr(name, arng string) int64 {
  2882  	var curQ, curSize, prefix uint16
  2883  	if name[0] != 'V' && name[0] != 'Z' && name[0] != 'P' {
  2884  		panic("expect V0-V31, Z0-Z31, or P0-P15; found: " + name)
  2885  	}
  2886  	switch name[0] {
  2887  	case 'V':
  2888  		prefix = 0
  2889  	case 'Z':
  2890  		prefix = 1
  2891  	case 'P':
  2892  		prefix = 2
  2893  	}
  2894  	switch arng {
  2895  	case "B8":
  2896  		curSize = 0
  2897  		curQ = 0
  2898  	case "B16":
  2899  		curSize = 0
  2900  		curQ = 1
  2901  	case "H4":
  2902  		curSize = 1
  2903  		curQ = 0
  2904  	case "H8":
  2905  		curSize = 1
  2906  		curQ = 1
  2907  	case "S2":
  2908  		curSize = 2
  2909  		curQ = 0
  2910  	case "S4":
  2911  		curSize = 2
  2912  		curQ = 1
  2913  	case "D1":
  2914  		curSize = 3
  2915  		curQ = 0
  2916  	case "D2":
  2917  		curSize = 3
  2918  		curQ = 1
  2919  	case "B":
  2920  		curSize = 1
  2921  		curQ = 2
  2922  	case "H":
  2923  		curSize = 2
  2924  		curQ = 2
  2925  	case "S":
  2926  		curSize = 3
  2927  		curQ = 2
  2928  	case "D":
  2929  		curSize = 1
  2930  		curQ = 3
  2931  	case "Q":
  2932  		curSize = 2
  2933  		curQ = 3
  2934  	default:
  2935  		panic("invalid arrangement in ARM64 register list")
  2936  	}
  2937  	return (int64(prefix) << 32) | (int64(curQ) & 3 << 30) | (int64(curSize&3) << 10)
  2938  }
  2939  

View as plain text