Source file src/cmd/compile/internal/ppc64/ssa.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ppc64
     6  
     7  import (
     8  	"cmd/compile/internal/base"
     9  	"cmd/compile/internal/ir"
    10  	"cmd/compile/internal/logopt"
    11  	"cmd/compile/internal/objw"
    12  	"cmd/compile/internal/ssa"
    13  	"cmd/compile/internal/ssa/block"
    14  	"cmd/compile/internal/ssa/ssaop"
    15  	"cmd/compile/internal/ssagen"
    16  	"cmd/compile/internal/types"
    17  	"cmd/internal/obj"
    18  	"cmd/internal/obj/ppc64"
    19  	"internal/abi"
    20  	"internal/buildcfg"
    21  	"math"
    22  	"strings"
    23  )
    24  
    25  // ssaMarkMoves marks any MOVXconst ops that need to avoid clobbering flags.
    26  func ssaMarkMoves(s *ssagen.State, b *ssa.Block) {
    27  	//	flive := b.FlagsLiveAtEnd
    28  	//	if b.Control != nil && b.Control.Type.IsFlags() {
    29  	//		flive = true
    30  	//	}
    31  	//	for i := len(b.Values) - 1; i >= 0; i-- {
    32  	//		v := b.Values[i]
    33  	//		if flive && (v.Op == v.Op == ssa.OpPPC64MOVDconst) {
    34  	//			// The "mark" is any non-nil Aux value.
    35  	//			v.Aux = v
    36  	//		}
    37  	//		if v.Type.IsFlags() {
    38  	//			flive = false
    39  	//		}
    40  	//		for _, a := range v.Args {
    41  	//			if a.Type.IsFlags() {
    42  	//				flive = true
    43  	//			}
    44  	//		}
    45  	//	}
    46  }
    47  
    48  // simdReg returns the VSX register operand to use when emitting a VSX
    49  // instruction (LXV/STXV/XXLOR/...) for a vector value allocated to register r.
    50  // With simdregmask == vsxReg, a Vec128 value may be allocated to the F0-F31
    51  // bank (= VSR0-VSR31); VSX instructions require a VSX operand there, and the
    52  // assembler does not accept an F-name where a VSX register is expected. V0-V31
    53  // (= VSR32-VSR63) are already accepted via the C_VREG->C_VSREG compatibility,
    54  // so they are returned unchanged.
    55  func simdReg(r int16) int16 {
    56  	if r >= ppc64.REG_F0 && r <= ppc64.REG_F31 {
    57  		return ppc64.REG_VS0 + (r - ppc64.REG_F0)
    58  	}
    59  	return r
    60  }
    61  
    62  // loadByType returns the load instruction of the given type.
    63  func loadByType(t *types.Type) obj.As {
    64  	if t.IsSIMD() {
    65  		if buildcfg.GOPPC64 < 9 {
    66  			base.Fatalf("loadByType: SIMD type %v requires POWER9 or later", t)
    67  		}
    68  		return ppc64.ALXV
    69  	}
    70  	if t.Size() > 8 {
    71  		base.Fatalf("loadByType: unexpected >8-byte non-SIMD type %v", t)
    72  	}
    73  	if t.IsFloat() {
    74  		switch t.Size() {
    75  		case 4:
    76  			return ppc64.AFMOVS
    77  		case 8:
    78  			return ppc64.AFMOVD
    79  		}
    80  	} else {
    81  		switch t.Size() {
    82  		case 1:
    83  			if t.IsSigned() {
    84  				return ppc64.AMOVB
    85  			} else {
    86  				return ppc64.AMOVBZ
    87  			}
    88  		case 2:
    89  			if t.IsSigned() {
    90  				return ppc64.AMOVH
    91  			} else {
    92  				return ppc64.AMOVHZ
    93  			}
    94  		case 4:
    95  			if t.IsSigned() {
    96  				return ppc64.AMOVW
    97  			} else {
    98  				return ppc64.AMOVWZ
    99  			}
   100  		case 8:
   101  			return ppc64.AMOVD
   102  		}
   103  	}
   104  	panic("bad load type")
   105  }
   106  
   107  // storeByType returns the store instruction of the given type.
   108  func storeByType(t *types.Type) obj.As {
   109  	if t.IsSIMD() {
   110  		if buildcfg.GOPPC64 < 9 {
   111  			base.Fatalf("storeByType: SIMD type %v requires POWER9 or later", t)
   112  		}
   113  		return ppc64.ASTXV
   114  	}
   115  	if t.Size() > 8 {
   116  		base.Fatalf("storeByType: unexpected >8-byte non-SIMD type %v", t)
   117  	}
   118  	if t.IsFloat() {
   119  		switch t.Size() {
   120  		case 4:
   121  			return ppc64.AFMOVS
   122  		case 8:
   123  			return ppc64.AFMOVD
   124  		}
   125  	} else {
   126  		switch t.Size() {
   127  		case 1:
   128  			return ppc64.AMOVB
   129  		case 2:
   130  			return ppc64.AMOVH
   131  		case 4:
   132  			return ppc64.AMOVW
   133  		case 8:
   134  			return ppc64.AMOVD
   135  		}
   136  	}
   137  	panic("bad store type")
   138  }
   139  
   140  func ssaGenValue(s *ssagen.State, v *ssa.Value) {
   141  	switch v.Op {
   142  	case ssaop.OpCopy:
   143  		t := v.Type
   144  		if t.IsMemory() {
   145  			return
   146  		}
   147  		x := v.Args[0].Reg()
   148  		y := v.Reg()
   149  		if x != y {
   150  			rt := obj.TYPE_REG
   151  			op := ppc64.AMOVD
   152  
   153  			if t.IsSIMD() {
   154  				op = ppc64.AXXLOR
   155  				// A Vec128 may live in the F0-F31 bank; emit it as a
   156  				// VSX operand (see simdReg).
   157  				x = simdReg(x)
   158  				y = simdReg(y)
   159  			} else if t.IsFloat() {
   160  				op = ppc64.AFMOVD
   161  			}
   162  			p := s.Prog(op)
   163  			p.From.Type = rt
   164  			p.From.Reg = x
   165  			p.To.Type = rt
   166  			p.To.Reg = y
   167  			if t.IsSIMD() {
   168  				p.Reg = x
   169  			}
   170  		}
   171  
   172  	case ssaop.OpPPC64LoweredAtomicAnd8,
   173  		ssaop.OpPPC64LoweredAtomicAnd32,
   174  		ssaop.OpPPC64LoweredAtomicOr8,
   175  		ssaop.OpPPC64LoweredAtomicOr32:
   176  		// LWSYNC
   177  		// LBAR/LWAR	(Rarg0), Rtmp
   178  		// AND/OR	Rarg1, Rtmp
   179  		// STBCCC/STWCCC Rtmp, (Rarg0)
   180  		// BNE		-3(PC)
   181  		// LWSYNC
   182  		ld := ppc64.ALBAR
   183  		st := ppc64.ASTBCCC
   184  		if v.Op == ssaop.OpPPC64LoweredAtomicAnd32 || v.Op == ssaop.OpPPC64LoweredAtomicOr32 {
   185  			ld = ppc64.ALWAR
   186  			st = ppc64.ASTWCCC
   187  		}
   188  		r0 := v.Args[0].Reg()
   189  		r1 := v.Args[1].Reg()
   190  		// LWSYNC - Assuming shared data not write-through-required nor
   191  		// caching-inhibited. See Appendix B.2.2.2 in the ISA 2.07b.
   192  		s.Prog(ppc64.ALWSYNC)
   193  		// LBAR or LWAR
   194  		p := s.Prog(ld)
   195  		p.From.Type = obj.TYPE_MEM
   196  		p.From.Reg = r0
   197  		p.To.Type = obj.TYPE_REG
   198  		p.To.Reg = ppc64.REGTMP
   199  		// AND/OR reg1,out
   200  		p1 := s.Prog(v.Op.Asm())
   201  		p1.From.Type = obj.TYPE_REG
   202  		p1.From.Reg = r1
   203  		p1.To.Type = obj.TYPE_REG
   204  		p1.To.Reg = ppc64.REGTMP
   205  		// STBCCC or STWCCC
   206  		p2 := s.Prog(st)
   207  		p2.From.Type = obj.TYPE_REG
   208  		p2.From.Reg = ppc64.REGTMP
   209  		p2.To.Type = obj.TYPE_MEM
   210  		p2.To.Reg = r0
   211  		p2.RegTo2 = ppc64.REGTMP
   212  		// BNE retry
   213  		p3 := s.Prog(ppc64.ABNE)
   214  		p3.To.Type = obj.TYPE_BRANCH
   215  		p3.To.SetTarget(p)
   216  		// LWSYNC - Provide acquire ordering to pair with the
   217  		// release (pre-LWSYNC) above, making the operation
   218  		// sequentially consistent.
   219  		s.Prog(ppc64.ALWSYNC)
   220  
   221  	case ssaop.OpPPC64LoweredAtomicAdd32,
   222  		ssaop.OpPPC64LoweredAtomicAdd64:
   223  		// LWSYNC
   224  		// LDAR/LWAR    (Rarg0), Rout
   225  		// ADD		Rarg1, Rout
   226  		// STDCCC/STWCCC Rout, (Rarg0)
   227  		// BNE         -3(PC)
   228  		// MOVW		Rout,Rout (if Add32)
   229  		// LWSYNC
   230  		ld := ppc64.ALDAR
   231  		st := ppc64.ASTDCCC
   232  		if v.Op == ssaop.OpPPC64LoweredAtomicAdd32 {
   233  			ld = ppc64.ALWAR
   234  			st = ppc64.ASTWCCC
   235  		}
   236  		r0 := v.Args[0].Reg()
   237  		r1 := v.Args[1].Reg()
   238  		out := v.Reg0()
   239  		// LWSYNC - Assuming shared data not write-through-required nor
   240  		// caching-inhibited. See Appendix B.2.2.2 in the ISA 2.07b.
   241  		s.Prog(ppc64.ALWSYNC)
   242  		// LDAR or LWAR
   243  		p := s.Prog(ld)
   244  		p.From.Type = obj.TYPE_MEM
   245  		p.From.Reg = r0
   246  		p.To.Type = obj.TYPE_REG
   247  		p.To.Reg = out
   248  		// ADD reg1,out
   249  		p1 := s.Prog(ppc64.AADD)
   250  		p1.From.Type = obj.TYPE_REG
   251  		p1.From.Reg = r1
   252  		p1.To.Reg = out
   253  		p1.To.Type = obj.TYPE_REG
   254  		// STDCCC or STWCCC
   255  		p3 := s.Prog(st)
   256  		p3.From.Type = obj.TYPE_REG
   257  		p3.From.Reg = out
   258  		p3.To.Type = obj.TYPE_MEM
   259  		p3.To.Reg = r0
   260  		// BNE retry
   261  		p4 := s.Prog(ppc64.ABNE)
   262  		p4.To.Type = obj.TYPE_BRANCH
   263  		p4.To.SetTarget(p)
   264  
   265  		// Ensure a 32 bit result
   266  		if v.Op == ssaop.OpPPC64LoweredAtomicAdd32 {
   267  			p5 := s.Prog(ppc64.AMOVWZ)
   268  			p5.To.Type = obj.TYPE_REG
   269  			p5.To.Reg = out
   270  			p5.From.Type = obj.TYPE_REG
   271  			p5.From.Reg = out
   272  		}
   273  		// LWSYNC - Provide acquire ordering to pair with the
   274  		// release (pre-LWSYNC) above, making the operation
   275  		// sequentially consistent.
   276  		s.Prog(ppc64.ALWSYNC)
   277  
   278  	case ssaop.OpPPC64LoweredAtomicExchange8,
   279  		ssaop.OpPPC64LoweredAtomicExchange32,
   280  		ssaop.OpPPC64LoweredAtomicExchange64:
   281  		// LWSYNC
   282  		// LDAR/LWAR/LBAR        (Rarg0), Rout
   283  		// STDCCC/STWCCC/STBWCCC Rout, (Rarg0)
   284  		// BNE         -2(PC)
   285  		// ISYNC
   286  		ld := ppc64.ALDAR
   287  		st := ppc64.ASTDCCC
   288  		switch v.Op {
   289  		case ssaop.OpPPC64LoweredAtomicExchange8:
   290  			ld = ppc64.ALBAR
   291  			st = ppc64.ASTBCCC
   292  		case ssaop.OpPPC64LoweredAtomicExchange32:
   293  			ld = ppc64.ALWAR
   294  			st = ppc64.ASTWCCC
   295  		}
   296  		r0 := v.Args[0].Reg()
   297  		r1 := v.Args[1].Reg()
   298  		out := v.Reg0()
   299  		// LWSYNC - Assuming shared data not write-through-required nor
   300  		// caching-inhibited. See Appendix B.2.2.2 in the ISA 2.07b.
   301  		s.Prog(ppc64.ALWSYNC)
   302  		// L[B|W|D]AR
   303  		p := s.Prog(ld)
   304  		p.From.Type = obj.TYPE_MEM
   305  		p.From.Reg = r0
   306  		p.To.Type = obj.TYPE_REG
   307  		p.To.Reg = out
   308  		// ST[B|W|D]CCC
   309  		p1 := s.Prog(st)
   310  		p1.From.Type = obj.TYPE_REG
   311  		p1.From.Reg = r1
   312  		p1.To.Type = obj.TYPE_MEM
   313  		p1.To.Reg = r0
   314  		// BNE retry
   315  		p2 := s.Prog(ppc64.ABNE)
   316  		p2.To.Type = obj.TYPE_BRANCH
   317  		p2.To.SetTarget(p)
   318  		// ISYNC
   319  		s.Prog(ppc64.AISYNC)
   320  
   321  	case ssaop.OpPPC64LoweredAtomicLoad8,
   322  		ssaop.OpPPC64LoweredAtomicLoad32,
   323  		ssaop.OpPPC64LoweredAtomicLoad64,
   324  		ssaop.OpPPC64LoweredAtomicLoadPtr:
   325  		// SYNC
   326  		// MOVB/MOVD/MOVW (Rarg0), Rout
   327  		// CMP Rout,Rout
   328  		// BNE 1(PC)
   329  		// ISYNC
   330  		ld := ppc64.AMOVD
   331  		cmp := ppc64.ACMP
   332  		switch v.Op {
   333  		case ssaop.OpPPC64LoweredAtomicLoad8:
   334  			ld = ppc64.AMOVBZ
   335  		case ssaop.OpPPC64LoweredAtomicLoad32:
   336  			ld = ppc64.AMOVWZ
   337  			cmp = ppc64.ACMPW
   338  		}
   339  		arg0 := v.Args[0].Reg()
   340  		out := v.Reg0()
   341  		// SYNC when AuxInt == 1; otherwise, load-acquire
   342  		if v.AuxInt == 1 {
   343  			s.Prog(ppc64.ASYNC)
   344  		}
   345  		// Load
   346  		p := s.Prog(ld)
   347  		p.From.Type = obj.TYPE_MEM
   348  		p.From.Reg = arg0
   349  		p.To.Type = obj.TYPE_REG
   350  		p.To.Reg = out
   351  		// CMP
   352  		p1 := s.Prog(cmp)
   353  		p1.From.Type = obj.TYPE_REG
   354  		p1.From.Reg = out
   355  		p1.To.Type = obj.TYPE_REG
   356  		p1.To.Reg = out
   357  		// BNE
   358  		p2 := s.Prog(ppc64.ABNE)
   359  		p2.To.Type = obj.TYPE_BRANCH
   360  		// ISYNC
   361  		pisync := s.Prog(ppc64.AISYNC)
   362  		p2.To.SetTarget(pisync)
   363  
   364  	case ssaop.OpPPC64LoweredAtomicStore8,
   365  		ssaop.OpPPC64LoweredAtomicStore32,
   366  		ssaop.OpPPC64LoweredAtomicStore64:
   367  		// SYNC or LWSYNC
   368  		// MOVB/MOVW/MOVD arg1,(arg0)
   369  		st := ppc64.AMOVD
   370  		switch v.Op {
   371  		case ssaop.OpPPC64LoweredAtomicStore8:
   372  			st = ppc64.AMOVB
   373  		case ssaop.OpPPC64LoweredAtomicStore32:
   374  			st = ppc64.AMOVW
   375  		}
   376  		arg0 := v.Args[0].Reg()
   377  		arg1 := v.Args[1].Reg()
   378  		// If AuxInt == 0, LWSYNC (Store-Release), else SYNC
   379  		// SYNC
   380  		syncOp := ppc64.ASYNC
   381  		if v.AuxInt == 0 {
   382  			syncOp = ppc64.ALWSYNC
   383  		}
   384  		s.Prog(syncOp)
   385  		// Store
   386  		p := s.Prog(st)
   387  		p.To.Type = obj.TYPE_MEM
   388  		p.To.Reg = arg0
   389  		p.From.Type = obj.TYPE_REG
   390  		p.From.Reg = arg1
   391  
   392  	case ssaop.OpPPC64LoweredAtomicCas64,
   393  		ssaop.OpPPC64LoweredAtomicCas32:
   394  		// MOVD        $0, Rout
   395  		// LWSYNC
   396  		// loop:
   397  		// LDAR        (Rarg0), MutexHint, Rtmp
   398  		// CMP         Rarg1, Rtmp
   399  		// BNE         end
   400  		// STDCCC      Rarg2, (Rarg0)
   401  		// BNE         loop
   402  		// MOVD        $1, Rout
   403  		// end:
   404  		// LWSYNC      // Only for sequential consistency; not required in CasRel.
   405  		ld := ppc64.ALDAR
   406  		st := ppc64.ASTDCCC
   407  		cmp := ppc64.ACMP
   408  		if v.Op == ssaop.OpPPC64LoweredAtomicCas32 {
   409  			ld = ppc64.ALWAR
   410  			st = ppc64.ASTWCCC
   411  			cmp = ppc64.ACMPW
   412  		}
   413  		r0 := v.Args[0].Reg()
   414  		r1 := v.Args[1].Reg()
   415  		r2 := v.Args[2].Reg()
   416  		out := v.Reg0()
   417  		// Initialize return value to false
   418  		p := s.Prog(ppc64.AMOVD)
   419  		p.From.Type = obj.TYPE_CONST
   420  		p.From.Offset = 0
   421  		p.To.Type = obj.TYPE_REG
   422  		p.To.Reg = out
   423  		// LWSYNC - Assuming shared data not write-through-required nor
   424  		// caching-inhibited. See Appendix B.2.2.2 in the ISA 2.07b.
   425  		s.Prog(ppc64.ALWSYNC)
   426  		// LDAR or LWAR
   427  		p0 := s.Prog(ld)
   428  		p0.From.Type = obj.TYPE_MEM
   429  		p0.From.Reg = r0
   430  		p0.To.Type = obj.TYPE_REG
   431  		p0.To.Reg = ppc64.REGTMP
   432  		// If it is a Compare-and-Swap-Release operation, set the EH field with
   433  		// the release hint.
   434  		if v.AuxInt == 0 {
   435  			p0.AddRestSourceConst(0)
   436  		}
   437  		// CMP reg1,reg2
   438  		p1 := s.Prog(cmp)
   439  		p1.From.Type = obj.TYPE_REG
   440  		p1.From.Reg = r1
   441  		p1.To.Reg = ppc64.REGTMP
   442  		p1.To.Type = obj.TYPE_REG
   443  		// BNE done with return value = false
   444  		p2 := s.Prog(ppc64.ABNE)
   445  		p2.To.Type = obj.TYPE_BRANCH
   446  		// STDCCC or STWCCC
   447  		p3 := s.Prog(st)
   448  		p3.From.Type = obj.TYPE_REG
   449  		p3.From.Reg = r2
   450  		p3.To.Type = obj.TYPE_MEM
   451  		p3.To.Reg = r0
   452  		// BNE retry
   453  		p4 := s.Prog(ppc64.ABNE)
   454  		p4.To.Type = obj.TYPE_BRANCH
   455  		p4.To.SetTarget(p0)
   456  		// return value true
   457  		p5 := s.Prog(ppc64.AMOVD)
   458  		p5.From.Type = obj.TYPE_CONST
   459  		p5.From.Offset = 1
   460  		p5.To.Type = obj.TYPE_REG
   461  		p5.To.Reg = out
   462  		// LWSYNC - Assuming shared data not write-through-required nor
   463  		// caching-inhibited. See Appendix B.2.1.1 in the ISA 2.07b.
   464  		// If the operation is a CAS-Release, then synchronization is not necessary.
   465  		if v.AuxInt != 0 {
   466  			plwsync2 := s.Prog(ppc64.ALWSYNC)
   467  			p2.To.SetTarget(plwsync2)
   468  		} else {
   469  			// done (label)
   470  			p6 := s.Prog(obj.ANOP)
   471  			p2.To.SetTarget(p6)
   472  		}
   473  
   474  	case ssaop.OpPPC64LoweredPubBarrier:
   475  		// LWSYNC
   476  		s.Prog(v.Op.Asm())
   477  
   478  	case ssaop.OpPPC64LoweredGetClosurePtr:
   479  		// Closure pointer is R11 (already)
   480  		ssagen.CheckLoweredGetClosurePtr(v)
   481  
   482  	case ssaop.OpPPC64LoweredGetCallerSP:
   483  		// caller's SP is FixedFrameSize below the address of the first arg
   484  		p := s.Prog(ppc64.AMOVD)
   485  		p.From.Type = obj.TYPE_ADDR
   486  		p.From.Offset = -base.Ctxt.Arch.FixedFrameSize
   487  		p.From.Name = obj.NAME_PARAM
   488  		p.To.Type = obj.TYPE_REG
   489  		p.To.Reg = v.Reg()
   490  
   491  	case ssaop.OpPPC64LoweredGetCallerPC:
   492  		p := s.Prog(obj.AGETCALLERPC)
   493  		p.To.Type = obj.TYPE_REG
   494  		p.To.Reg = v.Reg()
   495  
   496  	case ssaop.OpPPC64LoweredRound32F, ssaop.OpPPC64LoweredRound64F:
   497  		// input is already rounded
   498  
   499  	case ssaop.OpLoadReg:
   500  		loadOp := loadByType(v.Type)
   501  		p := s.Prog(loadOp)
   502  		ssagen.AddrAuto(&p.From, v.Args[0])
   503  		p.To.Type = obj.TYPE_REG
   504  		p.To.Reg = v.Reg()
   505  		if v.Type.IsSIMD() {
   506  			p.To.Reg = simdReg(v.Reg())
   507  		}
   508  
   509  	case ssaop.OpStoreReg:
   510  		storeOp := storeByType(v.Type)
   511  		p := s.Prog(storeOp)
   512  		p.From.Type = obj.TYPE_REG
   513  		p.From.Reg = v.Args[0].Reg()
   514  		if v.Type.IsSIMD() {
   515  			p.From.Reg = simdReg(v.Args[0].Reg())
   516  		}
   517  		ssagen.AddrAuto(&p.To, v)
   518  
   519  	case ssaop.OpArgIntReg, ssaop.OpArgFloatReg:
   520  		// The assembler needs to wrap the entry safepoint/stack growth code with spill/unspill
   521  		// The loop only runs once.
   522  		for _, a := range v.Block.Func.RegArgs {
   523  			// Pass the spill/unspill information along to the assembler, offset by size of
   524  			// the saved LR slot.
   525  			addr := ssagen.SpillSlotAddr(a, ppc64.REGSP, base.Ctxt.Arch.FixedFrameSize)
   526  			s.FuncInfo().AddSpill(
   527  				obj.RegSpill{Reg: a.Reg, Addr: addr, Unspill: loadByType(a.Type), Spill: storeByType(a.Type)})
   528  		}
   529  		v.Block.Func.RegArgs = nil
   530  
   531  		ssagen.CheckArgReg(v)
   532  
   533  	case ssaop.OpPPC64DIVD:
   534  		// For now,
   535  		//
   536  		// cmp arg1, -1
   537  		// be  ahead
   538  		// v = arg0 / arg1
   539  		// b over
   540  		// ahead: v = - arg0
   541  		// over: nop
   542  		//
   543  		// The compare writes CR0, so DIVD and DIVW must stay marked
   544  		// clobberFlags in PPC64Ops.go.
   545  		r := v.Reg()
   546  		r0 := v.Args[0].Reg()
   547  		r1 := v.Args[1].Reg()
   548  
   549  		p := s.Prog(ppc64.ACMP)
   550  		p.From.Type = obj.TYPE_REG
   551  		p.From.Reg = r1
   552  		p.To.Type = obj.TYPE_CONST
   553  		p.To.Offset = -1
   554  
   555  		pbahead := s.Prog(ppc64.ABEQ)
   556  		pbahead.To.Type = obj.TYPE_BRANCH
   557  
   558  		p = s.Prog(v.Op.Asm())
   559  		p.From.Type = obj.TYPE_REG
   560  		p.From.Reg = r1
   561  		p.Reg = r0
   562  		p.To.Type = obj.TYPE_REG
   563  		p.To.Reg = r
   564  
   565  		pbover := s.Prog(obj.AJMP)
   566  		pbover.To.Type = obj.TYPE_BRANCH
   567  
   568  		p = s.Prog(ppc64.ANEG)
   569  		p.To.Type = obj.TYPE_REG
   570  		p.To.Reg = r
   571  		p.From.Type = obj.TYPE_REG
   572  		p.From.Reg = r0
   573  		pbahead.To.SetTarget(p)
   574  
   575  		p = s.Prog(obj.ANOP)
   576  		pbover.To.SetTarget(p)
   577  
   578  	case ssaop.OpPPC64DIVW:
   579  		// word-width version of above
   580  		r := v.Reg()
   581  		r0 := v.Args[0].Reg()
   582  		r1 := v.Args[1].Reg()
   583  
   584  		p := s.Prog(ppc64.ACMPW)
   585  		p.From.Type = obj.TYPE_REG
   586  		p.From.Reg = r1
   587  		p.To.Type = obj.TYPE_CONST
   588  		p.To.Offset = -1
   589  
   590  		pbahead := s.Prog(ppc64.ABEQ)
   591  		pbahead.To.Type = obj.TYPE_BRANCH
   592  
   593  		p = s.Prog(v.Op.Asm())
   594  		p.From.Type = obj.TYPE_REG
   595  		p.From.Reg = r1
   596  		p.Reg = r0
   597  		p.To.Type = obj.TYPE_REG
   598  		p.To.Reg = r
   599  
   600  		pbover := s.Prog(obj.AJMP)
   601  		pbover.To.Type = obj.TYPE_BRANCH
   602  
   603  		p = s.Prog(ppc64.ANEG)
   604  		p.To.Type = obj.TYPE_REG
   605  		p.To.Reg = r
   606  		p.From.Type = obj.TYPE_REG
   607  		p.From.Reg = r0
   608  		pbahead.To.SetTarget(p)
   609  
   610  		p = s.Prog(obj.ANOP)
   611  		pbover.To.SetTarget(p)
   612  
   613  	case ssaop.OpPPC64CLRLSLWI:
   614  		r := v.Reg()
   615  		r1 := v.Args[0].Reg()
   616  		shifts := v.AuxInt
   617  		p := s.Prog(v.Op.Asm())
   618  		// clrlslwi ra,rs,mb,sh will become rlwinm ra,rs,sh,mb-sh,31-sh as described in ISA
   619  		p.From = obj.Addr{Type: obj.TYPE_CONST, Offset: ssa.GetPPC64Shiftmb(shifts)}
   620  		p.AddRestSourceConst(ssa.GetPPC64Shiftsh(shifts))
   621  		p.Reg = r1
   622  		p.To.Type = obj.TYPE_REG
   623  		p.To.Reg = r
   624  
   625  	case ssaop.OpPPC64CLRLSLDI:
   626  		r := v.Reg()
   627  		r1 := v.Args[0].Reg()
   628  		shifts := v.AuxInt
   629  		p := s.Prog(v.Op.Asm())
   630  		// clrlsldi ra,rs,mb,sh will become rldic ra,rs,sh,mb-sh
   631  		p.From = obj.Addr{Type: obj.TYPE_CONST, Offset: ssa.GetPPC64Shiftmb(shifts)}
   632  		p.AddRestSourceConst(ssa.GetPPC64Shiftsh(shifts))
   633  		p.Reg = r1
   634  		p.To.Type = obj.TYPE_REG
   635  		p.To.Reg = r
   636  
   637  	case ssaop.OpPPC64ADD, ssaop.OpPPC64FADD, ssaop.OpPPC64FADDS, ssaop.OpPPC64SUB, ssaop.OpPPC64FSUB, ssaop.OpPPC64FSUBS,
   638  		ssaop.OpPPC64MULLD, ssaop.OpPPC64MULLW, ssaop.OpPPC64DIVDU, ssaop.OpPPC64DIVWU,
   639  		ssaop.OpPPC64SRAD, ssaop.OpPPC64SRAW, ssaop.OpPPC64SRD, ssaop.OpPPC64SRW, ssaop.OpPPC64SLD, ssaop.OpPPC64SLW,
   640  		ssaop.OpPPC64ROTL, ssaop.OpPPC64ROTLW,
   641  		ssaop.OpPPC64MULHD, ssaop.OpPPC64MULHW, ssaop.OpPPC64MULHDU, ssaop.OpPPC64MULHWU,
   642  		ssaop.OpPPC64FMUL, ssaop.OpPPC64FMULS, ssaop.OpPPC64FDIV, ssaop.OpPPC64FDIVS, ssaop.OpPPC64FCPSGN,
   643  		ssaop.OpPPC64AND, ssaop.OpPPC64OR, ssaop.OpPPC64ANDN, ssaop.OpPPC64ORN, ssaop.OpPPC64NOR, ssaop.OpPPC64XOR, ssaop.OpPPC64EQV,
   644  		ssaop.OpPPC64MODUD, ssaop.OpPPC64MODSD, ssaop.OpPPC64MODUW, ssaop.OpPPC64MODSW, ssaop.OpPPC64XSMINJDP, ssaop.OpPPC64XSMAXJDP:
   645  		r := v.Reg()
   646  		r1 := v.Args[0].Reg()
   647  		r2 := v.Args[1].Reg()
   648  		p := s.Prog(v.Op.Asm())
   649  		p.From.Type = obj.TYPE_REG
   650  		p.From.Reg = r2
   651  		p.Reg = r1
   652  		p.To.Type = obj.TYPE_REG
   653  		p.To.Reg = r
   654  
   655  	case ssaop.OpPPC64ADDCC, ssaop.OpPPC64ANDCC, ssaop.OpPPC64SUBCC, ssaop.OpPPC64ORCC, ssaop.OpPPC64XORCC, ssaop.OpPPC64NORCC,
   656  		ssaop.OpPPC64ANDNCC, ssaop.OpPPC64MULHDUCC:
   657  		r1 := v.Args[0].Reg()
   658  		r2 := v.Args[1].Reg()
   659  		p := s.Prog(v.Op.Asm())
   660  		p.From.Type = obj.TYPE_REG
   661  		p.From.Reg = r2
   662  		p.Reg = r1
   663  		p.To.Type = obj.TYPE_REG
   664  		p.To.Reg = v.Reg0()
   665  
   666  	case ssaop.OpPPC64NEGCC, ssaop.OpPPC64CNTLZDCC:
   667  		p := s.Prog(v.Op.Asm())
   668  		p.To.Type = obj.TYPE_REG
   669  		p.To.Reg = v.Reg0()
   670  		p.From.Type = obj.TYPE_REG
   671  		p.From.Reg = v.Args[0].Reg()
   672  
   673  	case ssaop.OpPPC64ROTLconst, ssaop.OpPPC64ROTLWconst:
   674  		p := s.Prog(v.Op.Asm())
   675  		p.From.Type = obj.TYPE_CONST
   676  		p.From.Offset = v.AuxInt
   677  		p.Reg = v.Args[0].Reg()
   678  		p.To.Type = obj.TYPE_REG
   679  		p.To.Reg = v.Reg()
   680  
   681  		// Auxint holds encoded rotate + mask
   682  	case ssaop.OpPPC64RLWINM, ssaop.OpPPC64RLWMI:
   683  		sh, mb, me, _ := ssa.DecodePPC64RotateMask(v.AuxInt)
   684  		p := s.Prog(v.Op.Asm())
   685  		p.To = obj.Addr{Type: obj.TYPE_REG, Reg: v.Reg()}
   686  		p.Reg = v.Args[0].Reg()
   687  		p.From = obj.Addr{Type: obj.TYPE_CONST, Offset: sh}
   688  		p.AddRestSourceArgs([]obj.Addr{{Type: obj.TYPE_CONST, Offset: mb}, {Type: obj.TYPE_CONST, Offset: me}})
   689  		// Auxint holds mask
   690  
   691  	case ssaop.OpPPC64RLDICL, ssaop.OpPPC64RLDICLCC, ssaop.OpPPC64RLDICR:
   692  		sh, mb, me, _ := ssa.DecodePPC64RotateMask(v.AuxInt)
   693  		p := s.Prog(v.Op.Asm())
   694  		p.From = obj.Addr{Type: obj.TYPE_CONST, Offset: sh}
   695  		switch v.Op {
   696  		case ssaop.OpPPC64RLDICL, ssaop.OpPPC64RLDICLCC:
   697  			p.AddRestSourceConst(mb)
   698  		case ssaop.OpPPC64RLDICR:
   699  			p.AddRestSourceConst(me)
   700  		}
   701  		p.Reg = v.Args[0].Reg()
   702  		p.To = obj.Addr{Type: obj.TYPE_REG, Reg: v.ResultReg()}
   703  
   704  	case ssaop.OpPPC64RLWNM:
   705  		_, mb, me, _ := ssa.DecodePPC64RotateMask(v.AuxInt)
   706  		p := s.Prog(v.Op.Asm())
   707  		p.To = obj.Addr{Type: obj.TYPE_REG, Reg: v.Reg()}
   708  		p.Reg = v.Args[0].Reg()
   709  		p.From = obj.Addr{Type: obj.TYPE_REG, Reg: v.Args[1].Reg()}
   710  		p.AddRestSourceArgs([]obj.Addr{{Type: obj.TYPE_CONST, Offset: mb}, {Type: obj.TYPE_CONST, Offset: me}})
   711  
   712  	case ssaop.OpPPC64MADDLD:
   713  		r := v.Reg()
   714  		r1 := v.Args[0].Reg()
   715  		r2 := v.Args[1].Reg()
   716  		r3 := v.Args[2].Reg()
   717  		// r = r1*r2 ± r3
   718  		p := s.Prog(v.Op.Asm())
   719  		p.From.Type = obj.TYPE_REG
   720  		p.From.Reg = r1
   721  		p.Reg = r2
   722  		p.AddRestSourceReg(r3)
   723  		p.To.Type = obj.TYPE_REG
   724  		p.To.Reg = r
   725  
   726  	case ssaop.OpPPC64FMADD, ssaop.OpPPC64FMADDS, ssaop.OpPPC64FMSUB, ssaop.OpPPC64FMSUBS:
   727  		r := v.Reg()
   728  		r1 := v.Args[0].Reg()
   729  		r2 := v.Args[1].Reg()
   730  		r3 := v.Args[2].Reg()
   731  		// r = r1*r2 ± r3
   732  		p := s.Prog(v.Op.Asm())
   733  		p.From.Type = obj.TYPE_REG
   734  		p.From.Reg = r1
   735  		p.Reg = r3
   736  		p.AddRestSourceReg(r2)
   737  		p.To.Type = obj.TYPE_REG
   738  		p.To.Reg = r
   739  
   740  	case ssaop.OpPPC64NEG, ssaop.OpPPC64FNEG, ssaop.OpPPC64FSQRT, ssaop.OpPPC64FSQRTS, ssaop.OpPPC64FFLOOR, ssaop.OpPPC64FTRUNC, ssaop.OpPPC64FCEIL,
   741  		ssaop.OpPPC64FCTIDZ, ssaop.OpPPC64FCTIWZ, ssaop.OpPPC64FCFID, ssaop.OpPPC64FCFIDS, ssaop.OpPPC64FRSP, ssaop.OpPPC64CNTLZD, ssaop.OpPPC64CNTLZW,
   742  		ssaop.OpPPC64POPCNTD, ssaop.OpPPC64POPCNTW, ssaop.OpPPC64POPCNTB, ssaop.OpPPC64MFVSRD, ssaop.OpPPC64MTVSRD, ssaop.OpPPC64FABS, ssaop.OpPPC64FNABS,
   743  		ssaop.OpPPC64FROUND, ssaop.OpPPC64CNTTZW, ssaop.OpPPC64CNTTZD, ssaop.OpPPC64BRH, ssaop.OpPPC64BRW, ssaop.OpPPC64BRD:
   744  		r := v.Reg()
   745  		p := s.Prog(v.Op.Asm())
   746  		p.To.Type = obj.TYPE_REG
   747  		p.To.Reg = r
   748  		p.From.Type = obj.TYPE_REG
   749  		p.From.Reg = v.Args[0].Reg()
   750  
   751  	case ssaop.OpPPC64ADDconst, ssaop.OpPPC64ORconst, ssaop.OpPPC64XORconst,
   752  		ssaop.OpPPC64SRADconst, ssaop.OpPPC64SRAWconst, ssaop.OpPPC64SRDconst, ssaop.OpPPC64SRWconst,
   753  		ssaop.OpPPC64SLDconst, ssaop.OpPPC64SLWconst, ssaop.OpPPC64EXTSWSLconst, ssaop.OpPPC64MULLWconst, ssaop.OpPPC64MULLDconst,
   754  		ssaop.OpPPC64ANDconst:
   755  		p := s.Prog(v.Op.Asm())
   756  		p.Reg = v.Args[0].Reg()
   757  		p.From.Type = obj.TYPE_CONST
   758  		p.From.Offset = v.AuxInt
   759  		p.To.Type = obj.TYPE_REG
   760  		p.To.Reg = v.Reg()
   761  
   762  	case ssaop.OpPPC64ADDC, ssaop.OpPPC64ADDE, ssaop.OpPPC64SUBC, ssaop.OpPPC64SUBE:
   763  		r := v.Reg0() // CA is the first, implied argument.
   764  		r1 := v.Args[0].Reg()
   765  		r2 := v.Args[1].Reg()
   766  		p := s.Prog(v.Op.Asm())
   767  		p.From.Type = obj.TYPE_REG
   768  		p.From.Reg = r2
   769  		p.Reg = r1
   770  		p.To.Type = obj.TYPE_REG
   771  		p.To.Reg = r
   772  
   773  	case ssaop.OpPPC64ADDZE:
   774  		p := s.Prog(v.Op.Asm())
   775  		p.From.Type = obj.TYPE_REG
   776  		p.From.Reg = v.Args[0].Reg()
   777  		p.To.Type = obj.TYPE_REG
   778  		p.To.Reg = v.Reg0()
   779  
   780  	case ssaop.OpPPC64ADDZEzero, ssaop.OpPPC64SUBZEzero:
   781  		p := s.Prog(v.Op.Asm())
   782  		p.From.Type = obj.TYPE_REG
   783  		p.From.Reg = ppc64.REG_R0
   784  		p.To.Type = obj.TYPE_REG
   785  		p.To.Reg = v.Reg()
   786  
   787  	case ssaop.OpPPC64ADDCconst:
   788  		p := s.Prog(v.Op.Asm())
   789  		p.Reg = v.Args[0].Reg()
   790  		p.From.Type = obj.TYPE_CONST
   791  		p.From.Offset = v.AuxInt
   792  		p.To.Type = obj.TYPE_REG
   793  		// Output is a pair, the second is the CA, which is implied.
   794  		p.To.Reg = v.Reg0()
   795  
   796  	case ssaop.OpPPC64SUBCconst:
   797  		p := s.Prog(v.Op.Asm())
   798  		p.AddRestSourceConst(v.AuxInt)
   799  		p.From.Type = obj.TYPE_REG
   800  		p.From.Reg = v.Args[0].Reg()
   801  		p.To.Type = obj.TYPE_REG
   802  		p.To.Reg = v.Reg0()
   803  
   804  	case ssaop.OpPPC64SUBFCconst:
   805  		p := s.Prog(v.Op.Asm())
   806  		p.AddRestSourceConst(v.AuxInt)
   807  		p.From.Type = obj.TYPE_REG
   808  		p.From.Reg = v.Args[0].Reg()
   809  		p.To.Type = obj.TYPE_REG
   810  		p.To.Reg = v.Reg()
   811  
   812  	case ssaop.OpPPC64ADDCCconst, ssaop.OpPPC64ANDCCconst:
   813  		p := s.Prog(v.Op.Asm())
   814  		p.Reg = v.Args[0].Reg()
   815  		p.From.Type = obj.TYPE_CONST
   816  		p.From.Offset = v.AuxInt
   817  		p.To.Type = obj.TYPE_REG
   818  		p.To.Reg = v.Reg0()
   819  
   820  	case ssaop.OpPPC64MOVDaddr:
   821  		switch v.Aux.(type) {
   822  		default:
   823  			v.Fatalf("aux in MOVDaddr is of unknown type %T", v.Aux)
   824  		case nil:
   825  			// If aux offset and aux int are both 0, and the same
   826  			// input and output regs are used, no instruction
   827  			// needs to be generated, since it would just be
   828  			// addi rx, rx, 0.
   829  			if v.AuxInt != 0 || v.Args[0].Reg() != v.Reg() {
   830  				p := s.Prog(ppc64.AMOVD)
   831  				p.From.Type = obj.TYPE_ADDR
   832  				p.From.Reg = v.Args[0].Reg()
   833  				p.From.Offset = v.AuxInt
   834  				p.To.Type = obj.TYPE_REG
   835  				p.To.Reg = v.Reg()
   836  			}
   837  
   838  		case *obj.LSym, ir.Node:
   839  			p := s.Prog(ppc64.AMOVD)
   840  			p.From.Type = obj.TYPE_ADDR
   841  			p.From.Reg = v.Args[0].Reg()
   842  			p.To.Type = obj.TYPE_REG
   843  			p.To.Reg = v.Reg()
   844  			ssagen.AddAux(&p.From, v)
   845  
   846  		}
   847  
   848  	case ssaop.OpPPC64MOVDconst:
   849  		p := s.Prog(v.Op.Asm())
   850  		p.From.Type = obj.TYPE_CONST
   851  		p.From.Offset = v.AuxInt
   852  		p.To.Type = obj.TYPE_REG
   853  		p.To.Reg = v.Reg()
   854  
   855  	case ssaop.OpPPC64FMOVDconst, ssaop.OpPPC64FMOVSconst:
   856  		p := s.Prog(v.Op.Asm())
   857  		p.From.Type = obj.TYPE_FCONST
   858  		p.From.Val = math.Float64frombits(uint64(v.AuxInt))
   859  		p.To.Type = obj.TYPE_REG
   860  		p.To.Reg = v.Reg()
   861  
   862  	case ssaop.OpPPC64FCMPU, ssaop.OpPPC64CMP, ssaop.OpPPC64CMPW, ssaop.OpPPC64CMPU, ssaop.OpPPC64CMPWU:
   863  		p := s.Prog(v.Op.Asm())
   864  		p.From.Type = obj.TYPE_REG
   865  		p.From.Reg = v.Args[0].Reg()
   866  		p.To.Type = obj.TYPE_REG
   867  		p.To.Reg = v.Args[1].Reg()
   868  
   869  	case ssaop.OpPPC64CMPconst, ssaop.OpPPC64CMPUconst, ssaop.OpPPC64CMPWconst, ssaop.OpPPC64CMPWUconst:
   870  		p := s.Prog(v.Op.Asm())
   871  		p.From.Type = obj.TYPE_REG
   872  		p.From.Reg = v.Args[0].Reg()
   873  		p.To.Type = obj.TYPE_CONST
   874  		p.To.Offset = v.AuxInt
   875  
   876  	case ssaop.OpPPC64MOVBreg, ssaop.OpPPC64MOVBZreg, ssaop.OpPPC64MOVHreg, ssaop.OpPPC64MOVHZreg, ssaop.OpPPC64MOVWreg, ssaop.OpPPC64MOVWZreg:
   877  		// Shift in register to required size
   878  		p := s.Prog(v.Op.Asm())
   879  		p.From.Type = obj.TYPE_REG
   880  		p.From.Reg = v.Args[0].Reg()
   881  		p.To.Reg = v.Reg()
   882  		p.To.Type = obj.TYPE_REG
   883  
   884  	case ssaop.OpPPC64MOVDload, ssaop.OpPPC64MOVWload:
   885  
   886  		// MOVDload and MOVWload are DS form instructions that are restricted to
   887  		// offsets that are a multiple of 4. If the offset is not a multiple of 4,
   888  		// then the address of the symbol to be loaded is computed (base + offset)
   889  		// and used as the new base register and the offset field in the instruction
   890  		// can be set to zero.
   891  
   892  		// This same problem can happen with gostrings since the final offset is not
   893  		// known yet, but could be unaligned after the relocation is resolved.
   894  		// So gostrings are handled the same way.
   895  
   896  		// This allows the MOVDload and MOVWload to be generated in more cases and
   897  		// eliminates some offset and alignment checking in the rules file.
   898  
   899  		fromAddr := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[0].Reg()}
   900  		ssagen.AddAux(&fromAddr, v)
   901  
   902  		genAddr := false
   903  
   904  		switch fromAddr.Name {
   905  		case obj.NAME_EXTERN, obj.NAME_STATIC:
   906  			// Special case for a rule combines the bytes of gostring.
   907  			// The v alignment might seem OK, but we don't want to load it
   908  			// using an offset because relocation comes later.
   909  			genAddr = strings.HasPrefix(fromAddr.Sym.Name, "go:string") || v.Type.Alignment()%4 != 0 || fromAddr.Offset%4 != 0
   910  		default:
   911  			genAddr = fromAddr.Offset%4 != 0
   912  		}
   913  		if genAddr {
   914  			// Load full address into the temp register.
   915  			p := s.Prog(ppc64.AMOVD)
   916  			p.From.Type = obj.TYPE_ADDR
   917  			p.From.Reg = v.Args[0].Reg()
   918  			ssagen.AddAux(&p.From, v)
   919  			// Load target using temp as base register
   920  			// and offset zero. Setting NAME_NONE
   921  			// prevents any extra offsets from being
   922  			// added.
   923  			p.To.Type = obj.TYPE_REG
   924  			p.To.Reg = ppc64.REGTMP
   925  			fromAddr.Reg = ppc64.REGTMP
   926  			// Clear the offset field and other
   927  			// information that might be used
   928  			// by the assembler to add to the
   929  			// final offset value.
   930  			fromAddr.Offset = 0
   931  			fromAddr.Name = obj.NAME_NONE
   932  			fromAddr.Sym = nil
   933  		}
   934  		p := s.Prog(v.Op.Asm())
   935  		p.From = fromAddr
   936  		p.To.Type = obj.TYPE_REG
   937  		p.To.Reg = v.Reg()
   938  
   939  	case ssaop.OpPPC64MOVHload, ssaop.OpPPC64MOVWZload, ssaop.OpPPC64MOVBZload, ssaop.OpPPC64MOVHZload, ssaop.OpPPC64FMOVDload, ssaop.OpPPC64FMOVSload:
   940  		p := s.Prog(v.Op.Asm())
   941  		p.From.Type = obj.TYPE_MEM
   942  		p.From.Reg = v.Args[0].Reg()
   943  		ssagen.AddAux(&p.From, v)
   944  		p.To.Type = obj.TYPE_REG
   945  		p.To.Reg = v.Reg()
   946  
   947  	case ssaop.OpPPC64MOVDBRload, ssaop.OpPPC64MOVWBRload, ssaop.OpPPC64MOVHBRload:
   948  		p := s.Prog(v.Op.Asm())
   949  		p.From.Type = obj.TYPE_MEM
   950  		p.From.Reg = v.Args[0].Reg()
   951  		p.To.Type = obj.TYPE_REG
   952  		p.To.Reg = v.Reg()
   953  
   954  	case ssaop.OpPPC64MOVDBRstore, ssaop.OpPPC64MOVWBRstore, ssaop.OpPPC64MOVHBRstore:
   955  		p := s.Prog(v.Op.Asm())
   956  		p.To.Type = obj.TYPE_MEM
   957  		p.To.Reg = v.Args[0].Reg()
   958  		p.From.Type = obj.TYPE_REG
   959  		p.From.Reg = v.Args[1].Reg()
   960  
   961  	case ssaop.OpPPC64MOVDloadidx, ssaop.OpPPC64MOVWloadidx, ssaop.OpPPC64MOVHloadidx, ssaop.OpPPC64MOVWZloadidx,
   962  		ssaop.OpPPC64MOVBZloadidx, ssaop.OpPPC64MOVHZloadidx, ssaop.OpPPC64FMOVDloadidx, ssaop.OpPPC64FMOVSloadidx,
   963  		ssaop.OpPPC64MOVDBRloadidx, ssaop.OpPPC64MOVWBRloadidx, ssaop.OpPPC64MOVHBRloadidx:
   964  		p := s.Prog(v.Op.Asm())
   965  		p.From.Type = obj.TYPE_MEM
   966  		p.From.Reg = v.Args[0].Reg()
   967  		p.From.Index = v.Args[1].Reg()
   968  		p.To.Type = obj.TYPE_REG
   969  		p.To.Reg = v.Reg()
   970  
   971  	case ssaop.OpPPC64DCBT:
   972  		p := s.Prog(v.Op.Asm())
   973  		p.From.Type = obj.TYPE_MEM
   974  		p.From.Reg = v.Args[0].Reg()
   975  		p.To.Type = obj.TYPE_CONST
   976  		p.To.Offset = v.AuxInt
   977  
   978  	case ssaop.OpPPC64MOVWstorezero, ssaop.OpPPC64MOVHstorezero, ssaop.OpPPC64MOVBstorezero:
   979  		p := s.Prog(v.Op.Asm())
   980  		p.From.Type = obj.TYPE_REG
   981  		p.From.Reg = ppc64.REGZERO
   982  		p.To.Type = obj.TYPE_MEM
   983  		p.To.Reg = v.Args[0].Reg()
   984  		ssagen.AddAux(&p.To, v)
   985  
   986  	case ssaop.OpPPC64MOVDstore, ssaop.OpPPC64MOVDstorezero:
   987  
   988  		// MOVDstore and MOVDstorezero become DS form instructions that are restricted
   989  		// to offset values that are a multiple of 4. If the offset field is not a
   990  		// multiple of 4, then the full address of the store target is computed (base +
   991  		// offset) and used as the new base register and the offset in the instruction
   992  		// is set to 0.
   993  
   994  		// This allows the MOVDstore and MOVDstorezero to be generated in more cases,
   995  		// and prevents checking of the offset value and alignment in the rules.
   996  
   997  		toAddr := obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[0].Reg()}
   998  		ssagen.AddAux(&toAddr, v)
   999  
  1000  		if toAddr.Offset%4 != 0 {
  1001  			p := s.Prog(ppc64.AMOVD)
  1002  			p.From.Type = obj.TYPE_ADDR
  1003  			p.From.Reg = v.Args[0].Reg()
  1004  			ssagen.AddAux(&p.From, v)
  1005  			p.To.Type = obj.TYPE_REG
  1006  			p.To.Reg = ppc64.REGTMP
  1007  			toAddr.Reg = ppc64.REGTMP
  1008  			// Clear the offset field and other
  1009  			// information that might be used
  1010  			// by the assembler to add to the
  1011  			// final offset value.
  1012  			toAddr.Offset = 0
  1013  			toAddr.Name = obj.NAME_NONE
  1014  			toAddr.Sym = nil
  1015  		}
  1016  		p := s.Prog(v.Op.Asm())
  1017  		p.To = toAddr
  1018  		p.From.Type = obj.TYPE_REG
  1019  		if v.Op == ssaop.OpPPC64MOVDstorezero {
  1020  			p.From.Reg = ppc64.REGZERO
  1021  		} else {
  1022  			p.From.Reg = v.Args[1].Reg()
  1023  		}
  1024  
  1025  	case ssaop.OpPPC64MOVWstore, ssaop.OpPPC64MOVHstore, ssaop.OpPPC64MOVBstore, ssaop.OpPPC64FMOVDstore, ssaop.OpPPC64FMOVSstore:
  1026  		p := s.Prog(v.Op.Asm())
  1027  		p.From.Type = obj.TYPE_REG
  1028  		p.From.Reg = v.Args[1].Reg()
  1029  		p.To.Type = obj.TYPE_MEM
  1030  		p.To.Reg = v.Args[0].Reg()
  1031  		ssagen.AddAux(&p.To, v)
  1032  
  1033  	case ssaop.OpPPC64MOVDstoreidx, ssaop.OpPPC64MOVWstoreidx, ssaop.OpPPC64MOVHstoreidx, ssaop.OpPPC64MOVBstoreidx,
  1034  		ssaop.OpPPC64FMOVDstoreidx, ssaop.OpPPC64FMOVSstoreidx, ssaop.OpPPC64MOVDBRstoreidx, ssaop.OpPPC64MOVWBRstoreidx,
  1035  		ssaop.OpPPC64MOVHBRstoreidx:
  1036  		p := s.Prog(v.Op.Asm())
  1037  		p.From.Type = obj.TYPE_REG
  1038  		p.From.Reg = v.Args[2].Reg()
  1039  		p.To.Index = v.Args[1].Reg()
  1040  		p.To.Type = obj.TYPE_MEM
  1041  		p.To.Reg = v.Args[0].Reg()
  1042  
  1043  	case ssaop.OpPPC64ISEL, ssaop.OpPPC64ISELZ:
  1044  		// ISEL  AuxInt ? arg0 : arg1
  1045  		// ISELZ is a special case of ISEL where arg1 is implicitly $0.
  1046  		//
  1047  		// AuxInt value indicates conditions 0=LT 1=GT 2=EQ 3=SO 4=GE 5=LE 6=NE 7=NSO.
  1048  		// ISEL accepts a CR bit argument, not a condition as expressed by AuxInt.
  1049  		// Convert the condition to a CR bit argument by the following conversion:
  1050  		//
  1051  		// AuxInt&3 ? arg0 : arg1 for conditions LT, GT, EQ, SO
  1052  		// AuxInt&3 ? arg1 : arg0 for conditions GE, LE, NE, NSO
  1053  		p := s.Prog(v.Op.Asm())
  1054  		p.To = obj.Addr{Type: obj.TYPE_REG, Reg: v.Reg()}
  1055  		p.Reg = v.Args[0].Reg()
  1056  		if v.Op == ssaop.OpPPC64ISEL {
  1057  			p.AddRestSourceReg(v.Args[1].Reg())
  1058  		} else {
  1059  			p.AddRestSourceReg(ppc64.REG_R0)
  1060  		}
  1061  		// AuxInt values 4,5,6 implemented with reverse operand order from 0,1,2
  1062  		if v.AuxInt > 3 {
  1063  			p.Reg, p.GetFrom3().Reg = p.GetFrom3().Reg, p.Reg
  1064  		}
  1065  		p.From.SetConst(v.AuxInt & 3)
  1066  
  1067  	case ssaop.OpPPC64SETBC, ssaop.OpPPC64SETBCR:
  1068  		p := s.Prog(v.Op.Asm())
  1069  		p.To.Type = obj.TYPE_REG
  1070  		p.To.Reg = v.Reg()
  1071  		p.From.Type = obj.TYPE_REG
  1072  		p.From.Reg = int16(ppc64.REG_CR0LT + v.AuxInt)
  1073  
  1074  	case ssaop.OpPPC64Zero128:
  1075  		// The output may be allocated to the F0-F31 bank; XXLXOR is a VSX
  1076  		// instruction and needs a VSX operand there (see simdReg).
  1077  		r := simdReg(v.Reg())
  1078  		p := s.Prog(ppc64.AXXLXOR)
  1079  		p.From.Type = obj.TYPE_REG
  1080  		p.From.Reg = r
  1081  		p.To.Type = obj.TYPE_REG
  1082  		p.To.Reg = r
  1083  		p.Reg = r
  1084  
  1085  	case ssaop.OpPPC64LoweredQuadZero, ssaop.OpPPC64LoweredQuadZeroShort:
  1086  		// The LoweredQuad code generation
  1087  		// generates STXV instructions on
  1088  		// power9. The Short variation is used
  1089  		// if no loop is generated.
  1090  		//
  1091  		// arg0 = destination address
  1092  		// arg1 = scratch zero vector (produced by Zero128)
  1093  		// arg2 = mem
  1094  
  1095  		// The scratch vector register is allocated by regalloc
  1096  		// and zeroed by the Zero128 op that feeds arg1.
  1097  		vreg := v.Args[1].Reg()
  1098  
  1099  		// Bytes per iteration
  1100  		ctr := v.AuxInt / 64
  1101  
  1102  		// Remainder bytes
  1103  		rem := v.AuxInt % 64
  1104  
  1105  		// Only generate a loop if there is more
  1106  		// than 1 iteration.
  1107  		if ctr > 1 {
  1108  			// Set up CTR loop counter
  1109  			p := s.Prog(ppc64.AMOVD)
  1110  			p.From.Type = obj.TYPE_CONST
  1111  			p.From.Offset = ctr
  1112  			p.To.Type = obj.TYPE_REG
  1113  			p.To.Reg = ppc64.REGTMP
  1114  
  1115  			p = s.Prog(ppc64.AMOVD)
  1116  			p.From.Type = obj.TYPE_REG
  1117  			p.From.Reg = ppc64.REGTMP
  1118  			p.To.Type = obj.TYPE_REG
  1119  			p.To.Reg = ppc64.REG_CTR
  1120  
  1121  			// Don't generate padding for
  1122  			// loops with few iterations.
  1123  			if ctr > 3 {
  1124  				p = s.Prog(obj.APCALIGN)
  1125  				p.From.Type = obj.TYPE_CONST
  1126  				p.From.Offset = 16
  1127  			}
  1128  
  1129  			// generate 4 STXVs to zero 64 bytes
  1130  			var top *obj.Prog
  1131  
  1132  			p = s.Prog(ppc64.ASTXV)
  1133  			p.From.Type = obj.TYPE_REG
  1134  			p.From.Reg = vreg
  1135  			p.To.Type = obj.TYPE_MEM
  1136  			p.To.Reg = v.Args[0].Reg()
  1137  
  1138  			//  Save the top of loop
  1139  			if top == nil {
  1140  				top = p
  1141  			}
  1142  			p = s.Prog(ppc64.ASTXV)
  1143  			p.From.Type = obj.TYPE_REG
  1144  			p.From.Reg = vreg
  1145  			p.To.Type = obj.TYPE_MEM
  1146  			p.To.Reg = v.Args[0].Reg()
  1147  			p.To.Offset = 16
  1148  
  1149  			p = s.Prog(ppc64.ASTXV)
  1150  			p.From.Type = obj.TYPE_REG
  1151  			p.From.Reg = vreg
  1152  			p.To.Type = obj.TYPE_MEM
  1153  			p.To.Reg = v.Args[0].Reg()
  1154  			p.To.Offset = 32
  1155  
  1156  			p = s.Prog(ppc64.ASTXV)
  1157  			p.From.Type = obj.TYPE_REG
  1158  			p.From.Reg = vreg
  1159  			p.To.Type = obj.TYPE_MEM
  1160  			p.To.Reg = v.Args[0].Reg()
  1161  			p.To.Offset = 48
  1162  
  1163  			// Increment address for the
  1164  			// 64 bytes just zeroed.
  1165  			p = s.Prog(ppc64.AADD)
  1166  			p.Reg = v.Args[0].Reg()
  1167  			p.From.Type = obj.TYPE_CONST
  1168  			p.From.Offset = 64
  1169  			p.To.Type = obj.TYPE_REG
  1170  			p.To.Reg = v.Args[0].Reg()
  1171  
  1172  			// Branch back to top of loop
  1173  			// based on CTR
  1174  			// BC with BO_BCTR generates bdnz
  1175  			p = s.Prog(ppc64.ABC)
  1176  			p.From.Type = obj.TYPE_CONST
  1177  			p.From.Offset = ppc64.BO_BCTR
  1178  			p.Reg = ppc64.REG_CR0LT
  1179  			p.To.Type = obj.TYPE_BRANCH
  1180  			p.To.SetTarget(top)
  1181  		}
  1182  		// When ctr == 1 the loop was not generated but
  1183  		// there are at least 64 bytes to clear, so add
  1184  		// that to the remainder to generate the code
  1185  		// to clear those doublewords
  1186  		if ctr == 1 {
  1187  			rem += 64
  1188  		}
  1189  
  1190  		// Clear the remainder starting at offset zero
  1191  		offset := int64(0)
  1192  
  1193  		// Generate STXV for 32 or 64
  1194  		// bytes.
  1195  		for rem >= 32 {
  1196  			p := s.Prog(ppc64.ASTXV)
  1197  			p.From.Type = obj.TYPE_REG
  1198  			p.From.Reg = vreg
  1199  			p.To.Type = obj.TYPE_MEM
  1200  			p.To.Reg = v.Args[0].Reg()
  1201  			p.To.Offset = offset
  1202  
  1203  			p = s.Prog(ppc64.ASTXV)
  1204  			p.From.Type = obj.TYPE_REG
  1205  			p.From.Reg = vreg
  1206  			p.To.Type = obj.TYPE_MEM
  1207  			p.To.Reg = v.Args[0].Reg()
  1208  			p.To.Offset = offset + 16
  1209  			offset += 32
  1210  			rem -= 32
  1211  		}
  1212  		// Generate 16 bytes
  1213  		if rem >= 16 {
  1214  			p := s.Prog(ppc64.ASTXV)
  1215  			p.From.Type = obj.TYPE_REG
  1216  			p.From.Reg = vreg
  1217  			p.To.Type = obj.TYPE_MEM
  1218  			p.To.Reg = v.Args[0].Reg()
  1219  			p.To.Offset = offset
  1220  			offset += 16
  1221  			rem -= 16
  1222  		}
  1223  
  1224  		// first clear as many doublewords as possible
  1225  		// then clear remaining sizes as available
  1226  		for rem > 0 {
  1227  			op, size := ppc64.AMOVB, int64(1)
  1228  			switch {
  1229  			case rem >= 8:
  1230  				op, size = ppc64.AMOVD, 8
  1231  			case rem >= 4:
  1232  				op, size = ppc64.AMOVW, 4
  1233  			case rem >= 2:
  1234  				op, size = ppc64.AMOVH, 2
  1235  			}
  1236  			p := s.Prog(op)
  1237  			p.From.Type = obj.TYPE_REG
  1238  			p.From.Reg = ppc64.REG_R0
  1239  			p.To.Type = obj.TYPE_MEM
  1240  			p.To.Reg = v.Args[0].Reg()
  1241  			p.To.Offset = offset
  1242  			rem -= size
  1243  			offset += size
  1244  		}
  1245  
  1246  	case ssaop.OpPPC64LoweredZero, ssaop.OpPPC64LoweredZeroShort:
  1247  
  1248  		// Unaligned data doesn't hurt performance
  1249  		// for these instructions on power8.
  1250  
  1251  		// For sizes >= 64 generate a loop as follows:
  1252  
  1253  		// Set up loop counter in CTR, used by BC
  1254  		//       XXLXOR VS32,VS32,VS32
  1255  		//	 MOVD len/32,REG_TMP
  1256  		//	 MOVD REG_TMP,CTR
  1257  		//       MOVD $16,REG_TMP
  1258  		//	 loop:
  1259  		//	 STXVD2X VS32,(R0)(R20)
  1260  		//	 STXVD2X VS32,(R31)(R20)
  1261  		//	 ADD  $32,R20
  1262  		//	 BC   16, 0, loop
  1263  		//
  1264  		// any remainder is done as described below
  1265  
  1266  		// for sizes < 64 bytes, first clear as many doublewords as possible,
  1267  		// then handle the remainder
  1268  		//	MOVD R0,(R20)
  1269  		//	MOVD R0,8(R20)
  1270  		// .... etc.
  1271  		//
  1272  		// the remainder bytes are cleared using one or more
  1273  		// of the following instructions with the appropriate
  1274  		// offsets depending which instructions are needed
  1275  		//
  1276  		//	MOVW R0,n1(R20)	4 bytes
  1277  		//	MOVH R0,n2(R20)	2 bytes
  1278  		//	MOVB R0,n3(R20)	1 byte
  1279  		//
  1280  		// 7 bytes: MOVW, MOVH, MOVB
  1281  		// 6 bytes: MOVW, MOVH
  1282  		// 5 bytes: MOVW, MOVB
  1283  		// 3 bytes: MOVH, MOVB
  1284  
  1285  		// each loop iteration does 32 bytes
  1286  		ctr := v.AuxInt / 32
  1287  
  1288  		// remainder bytes
  1289  		rem := v.AuxInt % 32
  1290  
  1291  		// only generate a loop if there is more
  1292  		// than 1 iteration.
  1293  		if ctr > 1 {
  1294  			// Set up VS32 (V0) to hold 0s
  1295  			p := s.Prog(ppc64.AXXLXOR)
  1296  			p.From.Type = obj.TYPE_REG
  1297  			p.From.Reg = ppc64.REG_VS32
  1298  			p.To.Type = obj.TYPE_REG
  1299  			p.To.Reg = ppc64.REG_VS32
  1300  			p.Reg = ppc64.REG_VS32
  1301  
  1302  			// Set up CTR loop counter
  1303  			p = s.Prog(ppc64.AMOVD)
  1304  			p.From.Type = obj.TYPE_CONST
  1305  			p.From.Offset = ctr
  1306  			p.To.Type = obj.TYPE_REG
  1307  			p.To.Reg = ppc64.REGTMP
  1308  
  1309  			p = s.Prog(ppc64.AMOVD)
  1310  			p.From.Type = obj.TYPE_REG
  1311  			p.From.Reg = ppc64.REGTMP
  1312  			p.To.Type = obj.TYPE_REG
  1313  			p.To.Reg = ppc64.REG_CTR
  1314  
  1315  			// Set up R31 to hold index value 16
  1316  			p = s.Prog(ppc64.AMOVD)
  1317  			p.From.Type = obj.TYPE_CONST
  1318  			p.From.Offset = 16
  1319  			p.To.Type = obj.TYPE_REG
  1320  			p.To.Reg = ppc64.REGTMP
  1321  
  1322  			// Don't add padding for alignment
  1323  			// with few loop iterations.
  1324  			if ctr > 3 {
  1325  				p = s.Prog(obj.APCALIGN)
  1326  				p.From.Type = obj.TYPE_CONST
  1327  				p.From.Offset = 16
  1328  			}
  1329  
  1330  			// generate 2 STXVD2Xs to store 16 bytes
  1331  			// when this is a loop then the top must be saved
  1332  			var top *obj.Prog
  1333  			// This is the top of loop
  1334  
  1335  			p = s.Prog(ppc64.ASTXVD2X)
  1336  			p.From.Type = obj.TYPE_REG
  1337  			p.From.Reg = ppc64.REG_VS32
  1338  			p.To.Type = obj.TYPE_MEM
  1339  			p.To.Reg = v.Args[0].Reg()
  1340  			p.To.Index = ppc64.REGZERO
  1341  			// Save the top of loop
  1342  			if top == nil {
  1343  				top = p
  1344  			}
  1345  			p = s.Prog(ppc64.ASTXVD2X)
  1346  			p.From.Type = obj.TYPE_REG
  1347  			p.From.Reg = ppc64.REG_VS32
  1348  			p.To.Type = obj.TYPE_MEM
  1349  			p.To.Reg = v.Args[0].Reg()
  1350  			p.To.Index = ppc64.REGTMP
  1351  
  1352  			// Increment address for the
  1353  			// 4 doublewords just zeroed.
  1354  			p = s.Prog(ppc64.AADD)
  1355  			p.Reg = v.Args[0].Reg()
  1356  			p.From.Type = obj.TYPE_CONST
  1357  			p.From.Offset = 32
  1358  			p.To.Type = obj.TYPE_REG
  1359  			p.To.Reg = v.Args[0].Reg()
  1360  
  1361  			// Branch back to top of loop
  1362  			// based on CTR
  1363  			// BC with BO_BCTR generates bdnz
  1364  			p = s.Prog(ppc64.ABC)
  1365  			p.From.Type = obj.TYPE_CONST
  1366  			p.From.Offset = ppc64.BO_BCTR
  1367  			p.Reg = ppc64.REG_CR0LT
  1368  			p.To.Type = obj.TYPE_BRANCH
  1369  			p.To.SetTarget(top)
  1370  		}
  1371  
  1372  		// when ctr == 1 the loop was not generated but
  1373  		// there are at least 32 bytes to clear, so add
  1374  		// that to the remainder to generate the code
  1375  		// to clear those doublewords
  1376  		if ctr == 1 {
  1377  			rem += 32
  1378  		}
  1379  
  1380  		// clear the remainder starting at offset zero
  1381  		offset := int64(0)
  1382  
  1383  		// first clear as many doublewords as possible
  1384  		// then clear remaining sizes as available
  1385  		for rem > 0 {
  1386  			op, size := ppc64.AMOVB, int64(1)
  1387  			switch {
  1388  			case rem >= 8:
  1389  				op, size = ppc64.AMOVD, 8
  1390  			case rem >= 4:
  1391  				op, size = ppc64.AMOVW, 4
  1392  			case rem >= 2:
  1393  				op, size = ppc64.AMOVH, 2
  1394  			}
  1395  			p := s.Prog(op)
  1396  			p.From.Type = obj.TYPE_REG
  1397  			p.From.Reg = ppc64.REG_R0
  1398  			p.To.Type = obj.TYPE_MEM
  1399  			p.To.Reg = v.Args[0].Reg()
  1400  			p.To.Offset = offset
  1401  			rem -= size
  1402  			offset += size
  1403  		}
  1404  
  1405  	case ssaop.OpPPC64LoweredMove, ssaop.OpPPC64LoweredMoveShort:
  1406  
  1407  		bytesPerLoop := int64(32)
  1408  		// This will be used when moving more
  1409  		// than 8 bytes.  Moves start with
  1410  		// as many 8 byte moves as possible, then
  1411  		// 4, 2, or 1 byte(s) as remaining.  This will
  1412  		// work and be efficient for power8 or later.
  1413  		// If there are 64 or more bytes, then a
  1414  		// loop is generated to move 32 bytes and
  1415  		// update the src and dst addresses on each
  1416  		// iteration. When < 64 bytes, the appropriate
  1417  		// number of moves are generated based on the
  1418  		// size.
  1419  		// When moving >= 64 bytes a loop is used
  1420  		//	MOVD len/32,REG_TMP
  1421  		//	MOVD REG_TMP,CTR
  1422  		//	MOVD $16,REG_TMP
  1423  		// top:
  1424  		//	LXVD2X (R0)(R21),V0
  1425  		//	LXVD2X (R31)(R21),V1
  1426  		//	ADD $32,R21
  1427  		//	STXVD2X V0,(R0)(R20)
  1428  		//	STXVD2X V1,(R31)(R20)
  1429  		//	ADD $32,R20
  1430  		//	BC 16,0,top
  1431  		// Bytes not moved by this loop are moved
  1432  		// with a combination of the following instructions,
  1433  		// starting with the largest sizes and generating as
  1434  		// many as needed, using the appropriate offset value.
  1435  		//	MOVD  n(R21),R31
  1436  		//	MOVD  R31,n(R20)
  1437  		//	MOVW  n1(R21),R31
  1438  		//	MOVW  R31,n1(R20)
  1439  		//	MOVH  n2(R21),R31
  1440  		//	MOVH  R31,n2(R20)
  1441  		//	MOVB  n3(R21),R31
  1442  		//	MOVB  R31,n3(R20)
  1443  
  1444  		// Each loop iteration moves 32 bytes
  1445  		ctr := v.AuxInt / bytesPerLoop
  1446  
  1447  		// Remainder after the loop
  1448  		rem := v.AuxInt % bytesPerLoop
  1449  
  1450  		dstReg := v.Args[0].Reg()
  1451  		srcReg := v.Args[1].Reg()
  1452  
  1453  		// The set of registers used here, must match the clobbered reg list
  1454  		// in PPC64Ops.go.
  1455  		offset := int64(0)
  1456  
  1457  		// top of the loop
  1458  		var top *obj.Prog
  1459  		// Only generate looping code when loop counter is > 1 for >= 64 bytes
  1460  		if ctr > 1 {
  1461  			// Set up the CTR
  1462  			p := s.Prog(ppc64.AMOVD)
  1463  			p.From.Type = obj.TYPE_CONST
  1464  			p.From.Offset = ctr
  1465  			p.To.Type = obj.TYPE_REG
  1466  			p.To.Reg = ppc64.REGTMP
  1467  
  1468  			p = s.Prog(ppc64.AMOVD)
  1469  			p.From.Type = obj.TYPE_REG
  1470  			p.From.Reg = ppc64.REGTMP
  1471  			p.To.Type = obj.TYPE_REG
  1472  			p.To.Reg = ppc64.REG_CTR
  1473  
  1474  			// Use REGTMP as index reg
  1475  			p = s.Prog(ppc64.AMOVD)
  1476  			p.From.Type = obj.TYPE_CONST
  1477  			p.From.Offset = 16
  1478  			p.To.Type = obj.TYPE_REG
  1479  			p.To.Reg = ppc64.REGTMP
  1480  
  1481  			// Don't adding padding for
  1482  			// alignment with small iteration
  1483  			// counts.
  1484  			if ctr > 3 {
  1485  				p = s.Prog(obj.APCALIGN)
  1486  				p.From.Type = obj.TYPE_CONST
  1487  				p.From.Offset = 16
  1488  			}
  1489  
  1490  			// Generate 16 byte loads and stores.
  1491  			// Use temp register for index (16)
  1492  			// on the second one.
  1493  
  1494  			p = s.Prog(ppc64.ALXVD2X)
  1495  			p.From.Type = obj.TYPE_MEM
  1496  			p.From.Reg = srcReg
  1497  			p.From.Index = ppc64.REGZERO
  1498  			p.To.Type = obj.TYPE_REG
  1499  			p.To.Reg = ppc64.REG_V0
  1500  			if top == nil {
  1501  				top = p
  1502  			}
  1503  			p = s.Prog(ppc64.ALXVD2X)
  1504  			p.From.Type = obj.TYPE_MEM
  1505  			p.From.Reg = srcReg
  1506  			p.From.Index = ppc64.REGTMP
  1507  			p.To.Type = obj.TYPE_REG
  1508  			p.To.Reg = ppc64.REG_V1
  1509  
  1510  			// increment the src reg for next iteration
  1511  			p = s.Prog(ppc64.AADD)
  1512  			p.Reg = srcReg
  1513  			p.From.Type = obj.TYPE_CONST
  1514  			p.From.Offset = bytesPerLoop
  1515  			p.To.Type = obj.TYPE_REG
  1516  			p.To.Reg = srcReg
  1517  
  1518  			// generate 16 byte stores
  1519  			p = s.Prog(ppc64.ASTXVD2X)
  1520  			p.From.Type = obj.TYPE_REG
  1521  			p.From.Reg = ppc64.REG_V0
  1522  			p.To.Type = obj.TYPE_MEM
  1523  			p.To.Reg = dstReg
  1524  			p.To.Index = ppc64.REGZERO
  1525  
  1526  			p = s.Prog(ppc64.ASTXVD2X)
  1527  			p.From.Type = obj.TYPE_REG
  1528  			p.From.Reg = ppc64.REG_V1
  1529  			p.To.Type = obj.TYPE_MEM
  1530  			p.To.Reg = dstReg
  1531  			p.To.Index = ppc64.REGTMP
  1532  
  1533  			// increment the dst reg for next iteration
  1534  			p = s.Prog(ppc64.AADD)
  1535  			p.Reg = dstReg
  1536  			p.From.Type = obj.TYPE_CONST
  1537  			p.From.Offset = bytesPerLoop
  1538  			p.To.Type = obj.TYPE_REG
  1539  			p.To.Reg = dstReg
  1540  
  1541  			// BC with BO_BCTR generates bdnz to branch on nonzero CTR
  1542  			// to loop top.
  1543  			p = s.Prog(ppc64.ABC)
  1544  			p.From.Type = obj.TYPE_CONST
  1545  			p.From.Offset = ppc64.BO_BCTR
  1546  			p.Reg = ppc64.REG_CR0LT
  1547  			p.To.Type = obj.TYPE_BRANCH
  1548  			p.To.SetTarget(top)
  1549  
  1550  			// srcReg and dstReg were incremented in the loop, so
  1551  			// later instructions start with offset 0.
  1552  			offset = int64(0)
  1553  		}
  1554  
  1555  		// No loop was generated for one iteration, so
  1556  		// add 32 bytes to the remainder to move those bytes.
  1557  		if ctr == 1 {
  1558  			rem += bytesPerLoop
  1559  		}
  1560  
  1561  		if rem >= 16 {
  1562  			// Generate 16 byte loads and stores.
  1563  			// Use temp register for index (value 16)
  1564  			// on the second one.
  1565  			p := s.Prog(ppc64.ALXVD2X)
  1566  			p.From.Type = obj.TYPE_MEM
  1567  			p.From.Reg = srcReg
  1568  			p.From.Index = ppc64.REGZERO
  1569  			p.To.Type = obj.TYPE_REG
  1570  			p.To.Reg = ppc64.REG_V0
  1571  
  1572  			p = s.Prog(ppc64.ASTXVD2X)
  1573  			p.From.Type = obj.TYPE_REG
  1574  			p.From.Reg = ppc64.REG_V0
  1575  			p.To.Type = obj.TYPE_MEM
  1576  			p.To.Reg = dstReg
  1577  			p.To.Index = ppc64.REGZERO
  1578  
  1579  			offset = 16
  1580  			rem -= 16
  1581  
  1582  			if rem >= 16 {
  1583  				// Use REGTMP as index reg
  1584  				p := s.Prog(ppc64.AMOVD)
  1585  				p.From.Type = obj.TYPE_CONST
  1586  				p.From.Offset = 16
  1587  				p.To.Type = obj.TYPE_REG
  1588  				p.To.Reg = ppc64.REGTMP
  1589  
  1590  				p = s.Prog(ppc64.ALXVD2X)
  1591  				p.From.Type = obj.TYPE_MEM
  1592  				p.From.Reg = srcReg
  1593  				p.From.Index = ppc64.REGTMP
  1594  				p.To.Type = obj.TYPE_REG
  1595  				p.To.Reg = ppc64.REG_V0
  1596  
  1597  				p = s.Prog(ppc64.ASTXVD2X)
  1598  				p.From.Type = obj.TYPE_REG
  1599  				p.From.Reg = ppc64.REG_V0
  1600  				p.To.Type = obj.TYPE_MEM
  1601  				p.To.Reg = dstReg
  1602  				p.To.Index = ppc64.REGTMP
  1603  
  1604  				offset = 32
  1605  				rem -= 16
  1606  			}
  1607  		}
  1608  
  1609  		// Generate all the remaining load and store pairs, starting with
  1610  		// as many 8 byte moves as possible, then 4, 2, 1.
  1611  		for rem > 0 {
  1612  			op, size := ppc64.AMOVB, int64(1)
  1613  			switch {
  1614  			case rem >= 8:
  1615  				op, size = ppc64.AMOVD, 8
  1616  			case rem >= 4:
  1617  				op, size = ppc64.AMOVWZ, 4
  1618  			case rem >= 2:
  1619  				op, size = ppc64.AMOVH, 2
  1620  			}
  1621  			// Load
  1622  			p := s.Prog(op)
  1623  			p.To.Type = obj.TYPE_REG
  1624  			p.To.Reg = ppc64.REGTMP
  1625  			p.From.Type = obj.TYPE_MEM
  1626  			p.From.Reg = srcReg
  1627  			p.From.Offset = offset
  1628  
  1629  			// Store
  1630  			p = s.Prog(op)
  1631  			p.From.Type = obj.TYPE_REG
  1632  			p.From.Reg = ppc64.REGTMP
  1633  			p.To.Type = obj.TYPE_MEM
  1634  			p.To.Reg = dstReg
  1635  			p.To.Offset = offset
  1636  			rem -= size
  1637  			offset += size
  1638  		}
  1639  
  1640  	case ssaop.OpPPC64LoweredQuadMove, ssaop.OpPPC64LoweredQuadMoveShort:
  1641  		bytesPerLoop := int64(64)
  1642  		// This is used when moving more
  1643  		// than 8 bytes on power9.  Moves start with
  1644  		// as many 8 byte moves as possible, then
  1645  		// 4, 2, or 1 byte(s) as remaining.  This will
  1646  		// work and be efficient for power8 or later.
  1647  		// If there are 64 or more bytes, then a
  1648  		// loop is generated to move 32 bytes and
  1649  		// update the src and dst addresses on each
  1650  		// iteration. When < 64 bytes, the appropriate
  1651  		// number of moves are generated based on the
  1652  		// size.
  1653  		// When moving >= 64 bytes a loop is used
  1654  		//      MOVD len/32,REG_TMP
  1655  		//      MOVD REG_TMP,CTR
  1656  		// top:
  1657  		//      LXV 0(R21),V0
  1658  		//      LXV 16(R21),V1
  1659  		//      ADD $32,R21
  1660  		//      STXV V0,0(R20)
  1661  		//      STXV V1,16(R20)
  1662  		//      ADD $32,R20
  1663  		//      BC 16,0,top
  1664  		// Bytes not moved by this loop are moved
  1665  		// with a combination of the following instructions,
  1666  		// starting with the largest sizes and generating as
  1667  		// many as needed, using the appropriate offset value.
  1668  		//      MOVD  n(R21),R31
  1669  		//      MOVD  R31,n(R20)
  1670  		//      MOVW  n1(R21),R31
  1671  		//      MOVW  R31,n1(R20)
  1672  		//      MOVH  n2(R21),R31
  1673  		//      MOVH  R31,n2(R20)
  1674  		//      MOVB  n3(R21),R31
  1675  		//      MOVB  R31,n3(R20)
  1676  
  1677  		// Each loop iteration moves 32 bytes
  1678  		ctr := v.AuxInt / bytesPerLoop
  1679  
  1680  		// Remainder after the loop
  1681  		rem := v.AuxInt % bytesPerLoop
  1682  
  1683  		dstReg := v.Args[0].Reg()
  1684  		srcReg := v.Args[1].Reg()
  1685  
  1686  		offset := int64(0)
  1687  
  1688  		// top of the loop
  1689  		var top *obj.Prog
  1690  
  1691  		// Only generate looping code when loop counter is > 1 for >= 64 bytes
  1692  		if ctr > 1 {
  1693  			// Set up the CTR
  1694  			p := s.Prog(ppc64.AMOVD)
  1695  			p.From.Type = obj.TYPE_CONST
  1696  			p.From.Offset = ctr
  1697  			p.To.Type = obj.TYPE_REG
  1698  			p.To.Reg = ppc64.REGTMP
  1699  
  1700  			p = s.Prog(ppc64.AMOVD)
  1701  			p.From.Type = obj.TYPE_REG
  1702  			p.From.Reg = ppc64.REGTMP
  1703  			p.To.Type = obj.TYPE_REG
  1704  			p.To.Reg = ppc64.REG_CTR
  1705  
  1706  			p = s.Prog(obj.APCALIGN)
  1707  			p.From.Type = obj.TYPE_CONST
  1708  			p.From.Offset = 16
  1709  
  1710  			// Generate 16 byte loads and stores.
  1711  			p = s.Prog(ppc64.ALXV)
  1712  			p.From.Type = obj.TYPE_MEM
  1713  			p.From.Reg = srcReg
  1714  			p.From.Offset = offset
  1715  			p.To.Type = obj.TYPE_REG
  1716  			p.To.Reg = ppc64.REG_V0
  1717  			if top == nil {
  1718  				top = p
  1719  			}
  1720  			p = s.Prog(ppc64.ALXV)
  1721  			p.From.Type = obj.TYPE_MEM
  1722  			p.From.Reg = srcReg
  1723  			p.From.Offset = offset + 16
  1724  			p.To.Type = obj.TYPE_REG
  1725  			p.To.Reg = ppc64.REG_V1
  1726  
  1727  			// generate 16 byte stores
  1728  			p = s.Prog(ppc64.ASTXV)
  1729  			p.From.Type = obj.TYPE_REG
  1730  			p.From.Reg = ppc64.REG_V0
  1731  			p.To.Type = obj.TYPE_MEM
  1732  			p.To.Reg = dstReg
  1733  			p.To.Offset = offset
  1734  
  1735  			p = s.Prog(ppc64.ASTXV)
  1736  			p.From.Type = obj.TYPE_REG
  1737  			p.From.Reg = ppc64.REG_V1
  1738  			p.To.Type = obj.TYPE_MEM
  1739  			p.To.Reg = dstReg
  1740  			p.To.Offset = offset + 16
  1741  
  1742  			// Generate 16 byte loads and stores.
  1743  			p = s.Prog(ppc64.ALXV)
  1744  			p.From.Type = obj.TYPE_MEM
  1745  			p.From.Reg = srcReg
  1746  			p.From.Offset = offset + 32
  1747  			p.To.Type = obj.TYPE_REG
  1748  			p.To.Reg = ppc64.REG_V0
  1749  
  1750  			p = s.Prog(ppc64.ALXV)
  1751  			p.From.Type = obj.TYPE_MEM
  1752  			p.From.Reg = srcReg
  1753  			p.From.Offset = offset + 48
  1754  			p.To.Type = obj.TYPE_REG
  1755  			p.To.Reg = ppc64.REG_V1
  1756  
  1757  			// generate 16 byte stores
  1758  			p = s.Prog(ppc64.ASTXV)
  1759  			p.From.Type = obj.TYPE_REG
  1760  			p.From.Reg = ppc64.REG_V0
  1761  			p.To.Type = obj.TYPE_MEM
  1762  			p.To.Reg = dstReg
  1763  			p.To.Offset = offset + 32
  1764  
  1765  			p = s.Prog(ppc64.ASTXV)
  1766  			p.From.Type = obj.TYPE_REG
  1767  			p.From.Reg = ppc64.REG_V1
  1768  			p.To.Type = obj.TYPE_MEM
  1769  			p.To.Reg = dstReg
  1770  			p.To.Offset = offset + 48
  1771  
  1772  			// increment the src reg for next iteration
  1773  			p = s.Prog(ppc64.AADD)
  1774  			p.Reg = srcReg
  1775  			p.From.Type = obj.TYPE_CONST
  1776  			p.From.Offset = bytesPerLoop
  1777  			p.To.Type = obj.TYPE_REG
  1778  			p.To.Reg = srcReg
  1779  
  1780  			// increment the dst reg for next iteration
  1781  			p = s.Prog(ppc64.AADD)
  1782  			p.Reg = dstReg
  1783  			p.From.Type = obj.TYPE_CONST
  1784  			p.From.Offset = bytesPerLoop
  1785  			p.To.Type = obj.TYPE_REG
  1786  			p.To.Reg = dstReg
  1787  
  1788  			// BC with BO_BCTR generates bdnz to branch on nonzero CTR
  1789  			// to loop top.
  1790  			p = s.Prog(ppc64.ABC)
  1791  			p.From.Type = obj.TYPE_CONST
  1792  			p.From.Offset = ppc64.BO_BCTR
  1793  			p.Reg = ppc64.REG_CR0LT
  1794  			p.To.Type = obj.TYPE_BRANCH
  1795  			p.To.SetTarget(top)
  1796  
  1797  			// srcReg and dstReg were incremented in the loop, so
  1798  			// later instructions start with offset 0.
  1799  			offset = int64(0)
  1800  		}
  1801  
  1802  		// No loop was generated for one iteration, so
  1803  		// add 32 bytes to the remainder to move those bytes.
  1804  		if ctr == 1 {
  1805  			rem += bytesPerLoop
  1806  		}
  1807  		if rem >= 32 {
  1808  			p := s.Prog(ppc64.ALXV)
  1809  			p.From.Type = obj.TYPE_MEM
  1810  			p.From.Reg = srcReg
  1811  			p.To.Type = obj.TYPE_REG
  1812  			p.To.Reg = ppc64.REG_V0
  1813  
  1814  			p = s.Prog(ppc64.ALXV)
  1815  			p.From.Type = obj.TYPE_MEM
  1816  			p.From.Reg = srcReg
  1817  			p.From.Offset = 16
  1818  			p.To.Type = obj.TYPE_REG
  1819  			p.To.Reg = ppc64.REG_V1
  1820  
  1821  			p = s.Prog(ppc64.ASTXV)
  1822  			p.From.Type = obj.TYPE_REG
  1823  			p.From.Reg = ppc64.REG_V0
  1824  			p.To.Type = obj.TYPE_MEM
  1825  			p.To.Reg = dstReg
  1826  
  1827  			p = s.Prog(ppc64.ASTXV)
  1828  			p.From.Type = obj.TYPE_REG
  1829  			p.From.Reg = ppc64.REG_V1
  1830  			p.To.Type = obj.TYPE_MEM
  1831  			p.To.Reg = dstReg
  1832  			p.To.Offset = 16
  1833  
  1834  			offset = 32
  1835  			rem -= 32
  1836  		}
  1837  
  1838  		if rem >= 16 {
  1839  			// Generate 16 byte loads and stores.
  1840  			p := s.Prog(ppc64.ALXV)
  1841  			p.From.Type = obj.TYPE_MEM
  1842  			p.From.Reg = srcReg
  1843  			p.From.Offset = offset
  1844  			p.To.Type = obj.TYPE_REG
  1845  			p.To.Reg = ppc64.REG_V0
  1846  
  1847  			p = s.Prog(ppc64.ASTXV)
  1848  			p.From.Type = obj.TYPE_REG
  1849  			p.From.Reg = ppc64.REG_V0
  1850  			p.To.Type = obj.TYPE_MEM
  1851  			p.To.Reg = dstReg
  1852  			p.To.Offset = offset
  1853  
  1854  			offset += 16
  1855  			rem -= 16
  1856  
  1857  			if rem >= 16 {
  1858  				p := s.Prog(ppc64.ALXV)
  1859  				p.From.Type = obj.TYPE_MEM
  1860  				p.From.Reg = srcReg
  1861  				p.From.Offset = offset
  1862  				p.To.Type = obj.TYPE_REG
  1863  				p.To.Reg = ppc64.REG_V0
  1864  
  1865  				p = s.Prog(ppc64.ASTXV)
  1866  				p.From.Type = obj.TYPE_REG
  1867  				p.From.Reg = ppc64.REG_V0
  1868  				p.To.Type = obj.TYPE_MEM
  1869  				p.To.Reg = dstReg
  1870  				p.To.Offset = offset
  1871  
  1872  				offset += 16
  1873  				rem -= 16
  1874  			}
  1875  		}
  1876  		// Generate all the remaining load and store pairs, starting with
  1877  		// as many 8 byte moves as possible, then 4, 2, 1.
  1878  		for rem > 0 {
  1879  			op, size := ppc64.AMOVB, int64(1)
  1880  			switch {
  1881  			case rem >= 8:
  1882  				op, size = ppc64.AMOVD, 8
  1883  			case rem >= 4:
  1884  				op, size = ppc64.AMOVWZ, 4
  1885  			case rem >= 2:
  1886  				op, size = ppc64.AMOVH, 2
  1887  			}
  1888  			// Load
  1889  			p := s.Prog(op)
  1890  			p.To.Type = obj.TYPE_REG
  1891  			p.To.Reg = ppc64.REGTMP
  1892  			p.From.Type = obj.TYPE_MEM
  1893  			p.From.Reg = srcReg
  1894  			p.From.Offset = offset
  1895  
  1896  			// Store
  1897  			p = s.Prog(op)
  1898  			p.From.Type = obj.TYPE_REG
  1899  			p.From.Reg = ppc64.REGTMP
  1900  			p.To.Type = obj.TYPE_MEM
  1901  			p.To.Reg = dstReg
  1902  			p.To.Offset = offset
  1903  			rem -= size
  1904  			offset += size
  1905  		}
  1906  
  1907  	case ssaop.OpPPC64CALLstatic:
  1908  		s.Call(v)
  1909  
  1910  	case ssaop.OpPPC64CALLtail, ssaop.OpPPC64CALLtailinter:
  1911  		s.TailCall(v)
  1912  
  1913  	case ssaop.OpPPC64CALLclosure, ssaop.OpPPC64CALLinter:
  1914  		p := s.Prog(ppc64.AMOVD)
  1915  		p.From.Type = obj.TYPE_REG
  1916  		p.From.Reg = v.Args[0].Reg()
  1917  		p.To.Type = obj.TYPE_REG
  1918  		p.To.Reg = ppc64.REG_LR
  1919  
  1920  		if v.Args[0].Reg() != ppc64.REG_R12 {
  1921  			v.Fatalf("Function address for %v should be in R12 %d but is in %d", v.LongString(), ppc64.REG_R12, p.From.Reg)
  1922  		}
  1923  
  1924  		pp := s.Call(v)
  1925  
  1926  		// Convert the call into a blrl with hint this is not a subroutine return.
  1927  		// The full bclrl opcode must be specified when passing a hint.
  1928  		pp.As = ppc64.ABCL
  1929  		pp.From.Type = obj.TYPE_CONST
  1930  		pp.From.Offset = ppc64.BO_ALWAYS
  1931  		pp.Reg = ppc64.REG_CR0LT // The preferred value if BI is ignored.
  1932  		pp.To.Reg = ppc64.REG_LR
  1933  		pp.AddRestSourceConst(1)
  1934  
  1935  		if ppc64.NeedTOCpointer(base.Ctxt) {
  1936  			// When compiling Go into PIC, the function we just
  1937  			// called via pointer might have been implemented in
  1938  			// a separate module and so overwritten the TOC
  1939  			// pointer in R2; reload it.
  1940  			q := s.Prog(ppc64.AMOVD)
  1941  			q.From.Type = obj.TYPE_MEM
  1942  			q.From.Offset = 24
  1943  			q.From.Reg = ppc64.REGSP
  1944  			q.To.Type = obj.TYPE_REG
  1945  			q.To.Reg = ppc64.REG_R2
  1946  		}
  1947  
  1948  	case ssaop.OpPPC64LoweredWB:
  1949  		p := s.Prog(obj.ACALL)
  1950  		p.To.Type = obj.TYPE_MEM
  1951  		p.To.Name = obj.NAME_EXTERN
  1952  		// AuxInt encodes how many buffer entries we need.
  1953  		p.To.Sym = ir.Syms.GCWriteBarrier[v.AuxInt-1]
  1954  
  1955  	case ssaop.OpPPC64LoweredPanicBoundsRR, ssaop.OpPPC64LoweredPanicBoundsRC, ssaop.OpPPC64LoweredPanicBoundsCR, ssaop.OpPPC64LoweredPanicBoundsCC:
  1956  		// Compute the constant we put in the PCData entry for this call.
  1957  		code, signed := ssa.BoundsKind(v.AuxInt).Code()
  1958  		xIsReg := false
  1959  		yIsReg := false
  1960  		xVal := 0
  1961  		yVal := 0
  1962  		switch v.Op {
  1963  		case ssaop.OpPPC64LoweredPanicBoundsRR:
  1964  			xIsReg = true
  1965  			xVal = int(v.Args[0].Reg() - ppc64.REG_R3)
  1966  			yIsReg = true
  1967  			yVal = int(v.Args[1].Reg() - ppc64.REG_R3)
  1968  		case ssaop.OpPPC64LoweredPanicBoundsRC:
  1969  			xIsReg = true
  1970  			xVal = int(v.Args[0].Reg() - ppc64.REG_R3)
  1971  			c := v.Aux.(ssa.PanicBoundsC).C
  1972  			if c >= 0 && c <= abi.BoundsMaxConst {
  1973  				yVal = int(c)
  1974  			} else {
  1975  				// Move constant to a register
  1976  				yIsReg = true
  1977  				if yVal == xVal {
  1978  					yVal = 1
  1979  				}
  1980  				p := s.Prog(ppc64.AMOVD)
  1981  				p.From.Type = obj.TYPE_CONST
  1982  				p.From.Offset = c
  1983  				p.To.Type = obj.TYPE_REG
  1984  				p.To.Reg = ppc64.REG_R3 + int16(yVal)
  1985  			}
  1986  		case ssaop.OpPPC64LoweredPanicBoundsCR:
  1987  			yIsReg = true
  1988  			yVal = int(v.Args[0].Reg() - ppc64.REG_R3)
  1989  			c := v.Aux.(ssa.PanicBoundsC).C
  1990  			if c >= 0 && c <= abi.BoundsMaxConst {
  1991  				xVal = int(c)
  1992  			} else {
  1993  				// Move constant to a register
  1994  				if xVal == yVal {
  1995  					xVal = 1
  1996  				}
  1997  				p := s.Prog(ppc64.AMOVD)
  1998  				p.From.Type = obj.TYPE_CONST
  1999  				p.From.Offset = c
  2000  				p.To.Type = obj.TYPE_REG
  2001  				p.To.Reg = ppc64.REG_R3 + int16(xVal)
  2002  			}
  2003  		case ssaop.OpPPC64LoweredPanicBoundsCC:
  2004  			c := v.Aux.(ssa.PanicBoundsCC).Cx
  2005  			if c >= 0 && c <= abi.BoundsMaxConst {
  2006  				xVal = int(c)
  2007  			} else {
  2008  				// Move constant to a register
  2009  				xIsReg = true
  2010  				p := s.Prog(ppc64.AMOVD)
  2011  				p.From.Type = obj.TYPE_CONST
  2012  				p.From.Offset = c
  2013  				p.To.Type = obj.TYPE_REG
  2014  				p.To.Reg = ppc64.REG_R3 + int16(xVal)
  2015  			}
  2016  			c = v.Aux.(ssa.PanicBoundsCC).Cy
  2017  			if c >= 0 && c <= abi.BoundsMaxConst {
  2018  				yVal = int(c)
  2019  			} else {
  2020  				// Move constant to a register
  2021  				yIsReg = true
  2022  				yVal = 1
  2023  				p := s.Prog(ppc64.AMOVD)
  2024  				p.From.Type = obj.TYPE_CONST
  2025  				p.From.Offset = c
  2026  				p.To.Type = obj.TYPE_REG
  2027  				p.To.Reg = ppc64.REG_R3 + int16(yVal)
  2028  			}
  2029  		}
  2030  		c := abi.BoundsEncode(code, signed, xIsReg, yIsReg, xVal, yVal)
  2031  
  2032  		p := s.Prog(obj.APCDATA)
  2033  		p.From.SetConst(abi.PCDATA_PanicBounds)
  2034  		p.To.SetConst(int64(c))
  2035  		p = s.Prog(obj.ACALL)
  2036  		p.To.Type = obj.TYPE_MEM
  2037  		p.To.Name = obj.NAME_EXTERN
  2038  		p.To.Sym = ir.Syms.PanicBounds
  2039  
  2040  	case ssaop.OpPPC64LoweredNilCheck:
  2041  		if buildcfg.GOOS == "aix" {
  2042  			// CMP Rarg0, $0
  2043  			// BNE 2(PC)
  2044  			// STW R0, 0(R0)
  2045  			// NOP (so the BNE has somewhere to land)
  2046  
  2047  			// CMP Rarg0, $0
  2048  			p := s.Prog(ppc64.ACMP)
  2049  			p.From.Type = obj.TYPE_REG
  2050  			p.From.Reg = v.Args[0].Reg()
  2051  			p.To.Type = obj.TYPE_CONST
  2052  			p.To.Offset = 0
  2053  
  2054  			// BNE 2(PC)
  2055  			p2 := s.Prog(ppc64.ABNE)
  2056  			p2.To.Type = obj.TYPE_BRANCH
  2057  
  2058  			// STW R0, 0(R0)
  2059  			// Write at 0 is forbidden and will trigger a SIGSEGV
  2060  			p = s.Prog(ppc64.AMOVW)
  2061  			p.From.Type = obj.TYPE_REG
  2062  			p.From.Reg = ppc64.REG_R0
  2063  			p.To.Type = obj.TYPE_MEM
  2064  			p.To.Reg = ppc64.REG_R0
  2065  
  2066  			// NOP (so the BNE has somewhere to land)
  2067  			nop := s.Prog(obj.ANOP)
  2068  			p2.To.SetTarget(nop)
  2069  
  2070  		} else {
  2071  			// Issue a load which will fault if arg is nil.
  2072  			p := s.Prog(ppc64.AMOVBZ)
  2073  			p.From.Type = obj.TYPE_MEM
  2074  			p.From.Reg = v.Args[0].Reg()
  2075  			ssagen.AddAux(&p.From, v)
  2076  			p.To.Type = obj.TYPE_REG
  2077  			p.To.Reg = ppc64.REGTMP
  2078  		}
  2079  		if logopt.Enabled() {
  2080  			logopt.LogOpt(v.Pos, "nilcheck", "genssa", v.Block.Func.Name)
  2081  		}
  2082  		if base.Debug.Nil != 0 && v.Pos.Line() > 1 { // v.Pos.Line()==1 in generated wrappers
  2083  			base.WarnfAt(v.Pos, "generated nil check")
  2084  		}
  2085  
  2086  	// These should be resolved by rules and not make it here.
  2087  	case ssaop.OpPPC64Equal, ssaop.OpPPC64NotEqual, ssaop.OpPPC64LessThan, ssaop.OpPPC64FLessThan,
  2088  		ssaop.OpPPC64LessEqual, ssaop.OpPPC64GreaterThan, ssaop.OpPPC64FGreaterThan, ssaop.OpPPC64GreaterEqual,
  2089  		ssaop.OpPPC64FLessEqual, ssaop.OpPPC64FGreaterEqual:
  2090  		v.Fatalf("Pseudo-op should not make it to codegen: %s ###\n", v.LongString())
  2091  	case ssaop.OpPPC64InvertFlags:
  2092  		v.Fatalf("InvertFlags should never make it to codegen %v", v.LongString())
  2093  	case ssaop.OpPPC64FlagEQ, ssaop.OpPPC64FlagLT, ssaop.OpPPC64FlagGT:
  2094  		v.Fatalf("Flag* ops should never make it to codegen %v", v.LongString())
  2095  	case ssaop.OpClobber, ssaop.OpClobberReg:
  2096  		// TODO: implement for clobberdead experiment. Nop is ok for now.
  2097  	default:
  2098  		v.Fatalf("genValue not implemented: %s", v.LongString())
  2099  	}
  2100  }
  2101  
  2102  var blockJump = [...]struct {
  2103  	asm, invasm     obj.As
  2104  	asmeq, invasmun bool
  2105  }{
  2106  	block.BlockPPC64EQ: {ppc64.ABEQ, ppc64.ABNE, false, false},
  2107  	block.BlockPPC64NE: {ppc64.ABNE, ppc64.ABEQ, false, false},
  2108  
  2109  	block.BlockPPC64LT: {ppc64.ABLT, ppc64.ABGE, false, false},
  2110  	block.BlockPPC64GE: {ppc64.ABGE, ppc64.ABLT, false, false},
  2111  	block.BlockPPC64LE: {ppc64.ABLE, ppc64.ABGT, false, false},
  2112  	block.BlockPPC64GT: {ppc64.ABGT, ppc64.ABLE, false, false},
  2113  
  2114  	// TODO: need to work FP comparisons into block jumps
  2115  	block.BlockPPC64FLT: {ppc64.ABLT, ppc64.ABGE, false, false},
  2116  	block.BlockPPC64FGE: {ppc64.ABGT, ppc64.ABLT, true, true}, // GE = GT or EQ; !GE = LT or UN
  2117  	block.BlockPPC64FLE: {ppc64.ABLT, ppc64.ABGT, true, true}, // LE = LT or EQ; !LE = GT or UN
  2118  	block.BlockPPC64FGT: {ppc64.ABGT, ppc64.ABLE, false, false},
  2119  }
  2120  
  2121  func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
  2122  	switch b.Kind {
  2123  	case block.BlockPlain, block.BlockDefer:
  2124  		if b.Succs[0].Block() != next {
  2125  			p := s.Prog(obj.AJMP)
  2126  			p.To.Type = obj.TYPE_BRANCH
  2127  			s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
  2128  		}
  2129  	case block.BlockExit, block.BlockRetJmp:
  2130  	case block.BlockRet:
  2131  		s.Prog(obj.ARET)
  2132  
  2133  	case block.BlockPPC64EQ, block.BlockPPC64NE,
  2134  		block.BlockPPC64LT, block.BlockPPC64GE,
  2135  		block.BlockPPC64LE, block.BlockPPC64GT,
  2136  		block.BlockPPC64FLT, block.BlockPPC64FGE,
  2137  		block.BlockPPC64FLE, block.BlockPPC64FGT:
  2138  		jmp := blockJump[b.Kind]
  2139  		switch next {
  2140  		case b.Succs[0].Block():
  2141  			s.Br(jmp.invasm, b.Succs[1].Block())
  2142  			if jmp.invasmun {
  2143  				// TODO: The second branch is probably predict-not-taken since it is for FP unordered
  2144  				s.Br(ppc64.ABVS, b.Succs[1].Block())
  2145  			}
  2146  		case b.Succs[1].Block():
  2147  			s.Br(jmp.asm, b.Succs[0].Block())
  2148  			if jmp.asmeq {
  2149  				s.Br(ppc64.ABEQ, b.Succs[0].Block())
  2150  			}
  2151  		default:
  2152  			if b.Likely != ssa.BranchUnlikely {
  2153  				s.Br(jmp.asm, b.Succs[0].Block())
  2154  				if jmp.asmeq {
  2155  					s.Br(ppc64.ABEQ, b.Succs[0].Block())
  2156  				}
  2157  				s.Br(obj.AJMP, b.Succs[1].Block())
  2158  			} else {
  2159  				s.Br(jmp.invasm, b.Succs[1].Block())
  2160  				if jmp.invasmun {
  2161  					// TODO: The second branch is probably predict-not-taken since it is for FP unordered
  2162  					s.Br(ppc64.ABVS, b.Succs[1].Block())
  2163  				}
  2164  				s.Br(obj.AJMP, b.Succs[0].Block())
  2165  			}
  2166  		}
  2167  	default:
  2168  		b.Fatalf("branch not implemented: %s", b.LongString())
  2169  	}
  2170  }
  2171  
  2172  func loadRegResult(s *ssagen.State, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  2173  	p := s.Prog(loadByType(t))
  2174  	p.From.Type = obj.TYPE_MEM
  2175  	p.From.Name = obj.NAME_AUTO
  2176  	p.From.Sym = n.Linksym()
  2177  	p.From.Offset = n.FrameOffset() + off
  2178  	p.To.Type = obj.TYPE_REG
  2179  	p.To.Reg = reg
  2180  	return p
  2181  }
  2182  
  2183  func spillArgReg(pp *objw.Progs, p *obj.Prog, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
  2184  	p = pp.Append(p, storeByType(t), obj.TYPE_REG, reg, 0, obj.TYPE_MEM, 0, n.FrameOffset()+off)
  2185  	p.To.Name = obj.NAME_PARAM
  2186  	p.To.Sym = n.Linksym()
  2187  	p.Pos = p.Pos.WithNotStmt()
  2188  	return p
  2189  }
  2190  

View as plain text