Source file src/cmd/compile/internal/ssa/expand_calls.go

     1  // Copyright 2020 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ssa
     6  
     7  import (
     8  	"cmd/compile/internal/abi"
     9  	"cmd/compile/internal/base"
    10  	"cmd/compile/internal/ir"
    11  	"cmd/compile/internal/ssa/block"
    12  	"cmd/compile/internal/types"
    13  	"cmd/internal/src"
    14  	"fmt"
    15  )
    16  
    17  func postExpandCallsDecompose(f *Func) {
    18  	decomposeUser(f)    // redo user decompose to cleanup after expand calls
    19  	decomposeBuiltin(f) // handles both regular decomposition and cleanup.
    20  }
    21  
    22  func expandCalls(f *Func) {
    23  	// Convert each aggregate arg to a call into "dismantle aggregate, store/pass parts"
    24  	// Convert each aggregate result from a call into "assemble aggregate from parts"
    25  	// Convert each multivalue exit into "dismantle aggregate, store/return parts"
    26  	// Convert incoming aggregate arg into assembly of parts.
    27  	// Feed modified AST to decompose.
    28  
    29  	sp, _ := f.spSb()
    30  
    31  	x := &expandState{
    32  		f:               f,
    33  		debug:           f.pass.debug,
    34  		regSize:         f.Config.RegSize,
    35  		sp:              sp,
    36  		typs:            &f.Config.Types,
    37  		wideSelects:     make(map[*Value]*Value),
    38  		commonArgs:      make(map[selKey]*Value),
    39  		commonSelectors: make(map[selKey]*Value),
    40  		memForCall:      make(map[ID]*Value),
    41  	}
    42  
    43  	// For 32-bit, need to deal with decomposition of 64-bit integers, which depends on endianness.
    44  	if f.Config.BigEndian {
    45  		x.firstOp = OpInt64Hi
    46  		x.secondOp = OpInt64Lo
    47  		x.firstType = x.typs.Int32
    48  		x.secondType = x.typs.UInt32
    49  	} else {
    50  		x.firstOp = OpInt64Lo
    51  		x.secondOp = OpInt64Hi
    52  		x.firstType = x.typs.UInt32
    53  		x.secondType = x.typs.Int32
    54  	}
    55  
    56  	// Defer select processing until after all calls and selects are seen.
    57  	var selects []*Value
    58  	var calls []*Value
    59  	var args []*Value
    60  	var exitBlocks []*Block
    61  
    62  	var m0 *Value
    63  
    64  	// Accumulate lists of calls, args, selects, and exit blocks to process,
    65  	// note "wide" selects consumed by stores,
    66  	// rewrite mem for each call,
    67  	// rewrite each OpSelectNAddr.
    68  	for _, b := range f.Blocks {
    69  		for _, v := range b.Values {
    70  			switch v.Op {
    71  			case OpInitMem:
    72  				m0 = v
    73  
    74  			case OpClosureLECall, OpInterLECall, OpStaticLECall, OpTailLECall, OpTailLECallInter:
    75  				calls = append(calls, v)
    76  
    77  			case OpArg:
    78  				args = append(args, v)
    79  
    80  			case OpStore:
    81  				if a := v.Args[1]; a.Op == OpSelectN && !CanSSA(a.Type) {
    82  					if a.Uses > 1 {
    83  						panic(fmt.Errorf("Saw double use of wide SelectN %s operand of Store %s",
    84  							a.LongString(), v.LongString()))
    85  					}
    86  					x.wideSelects[a] = v
    87  				}
    88  
    89  			case OpSelectN:
    90  				if v.Type == types.TypeMem {
    91  					// rewrite the mem selector in place
    92  					call := v.Args[0]
    93  					aux := call.Aux.(*AuxCall)
    94  					mem := x.memForCall[call.ID]
    95  					if mem == nil {
    96  						v.AuxInt = int64(aux.abiInfo.OutRegistersUsed())
    97  						x.memForCall[call.ID] = v
    98  					} else {
    99  						panic(fmt.Errorf("Saw two memories for call %v, %v and %v", call, mem, v))
   100  					}
   101  				} else {
   102  					selects = append(selects, v)
   103  				}
   104  
   105  			case OpSelectNAddr:
   106  				call := v.Args[0]
   107  				which := v.AuxInt
   108  				aux := call.Aux.(*AuxCall)
   109  				pt := v.Type
   110  				off := x.offsetFrom(x.f.Entry, x.sp, aux.OffsetOfResult(which), pt)
   111  				v.copyOf(off)
   112  			}
   113  		}
   114  
   115  		// rewrite function results from an exit block
   116  		// values returned by function need to be split out into registers.
   117  		if isBlockMultiValueExit(b) {
   118  			exitBlocks = append(exitBlocks, b)
   119  		}
   120  	}
   121  
   122  	// Convert each aggregate arg into Make of its parts (and so on, to primitive types)
   123  	for _, v := range args {
   124  		var rc registerCursor
   125  		a := x.prAssignForArg(v)
   126  		aux := x.f.OwnAux
   127  		regs := a.Registers
   128  		var offset int64
   129  		if len(regs) == 0 {
   130  			offset = a.FrameOffset(aux.abiInfo)
   131  		}
   132  		auxBase := x.offsetFrom(x.f.Entry, x.sp, offset, types.NewPtr(v.Type))
   133  		rc.init(regs, aux.abiInfo, nil, auxBase, 0)
   134  		x.rewriteSelectOrArg(f.Entry.Pos, f.Entry, v, v, m0, v.Type, rc)
   135  	}
   136  
   137  	// Rewrite selects of results (which may be aggregates) into make-aggregates of register/memory-targeted selects
   138  	for _, v := range selects {
   139  		if v.Op == OpInvalid {
   140  			continue
   141  		}
   142  
   143  		call := v.Args[0]
   144  		aux := call.Aux.(*AuxCall)
   145  		mem := x.memForCall[call.ID]
   146  		if mem == nil {
   147  			mem = call.Block.NewValue1I(call.Pos, OpSelectN, types.TypeMem, int64(aux.abiInfo.OutRegistersUsed()), call)
   148  			x.memForCall[call.ID] = mem
   149  		}
   150  
   151  		i := v.AuxInt
   152  		regs := aux.RegsOfResult(i)
   153  
   154  		// If this select cannot fit into SSA and is stored, either disaggregate to register stores, or mem-mem move.
   155  		if store := x.wideSelects[v]; store != nil {
   156  			// Use the mem that comes from the store operation.
   157  			storeAddr := store.Args[0]
   158  			mem := store.Args[2]
   159  			if len(regs) > 0 {
   160  				// Cannot do a rewrite that builds up a result from pieces; instead, copy pieces to the store operation.
   161  				var rc registerCursor
   162  				rc.init(regs, aux.abiInfo, nil, storeAddr, 0)
   163  				mem = x.rewriteWideSelectToStores(call.Pos, call.Block, v, mem, v.Type, rc)
   164  				store.copyOf(mem)
   165  			} else {
   166  				// Move directly from AuxBase to store target; rewrite the store instruction.
   167  				offset := aux.OffsetOfResult(i)
   168  				auxBase := x.offsetFrom(x.f.Entry, x.sp, offset, types.NewPtr(v.Type))
   169  				// was Store dst, v, mem
   170  				// now Move dst, auxBase, mem
   171  				move := store.Block.NewValue3A(store.Pos, OpMove, types.TypeMem, v.Type, storeAddr, auxBase, mem)
   172  				move.AuxInt = v.Type.Size()
   173  				store.copyOf(move)
   174  			}
   175  			continue
   176  		}
   177  
   178  		var auxBase *Value
   179  		if len(regs) == 0 {
   180  			offset := aux.OffsetOfResult(i)
   181  			auxBase = x.offsetFrom(x.f.Entry, x.sp, offset, types.NewPtr(v.Type))
   182  		}
   183  		var rc registerCursor
   184  		rc.init(regs, aux.abiInfo, nil, auxBase, 0)
   185  		x.rewriteSelectOrArg(call.Pos, call.Block, v, v, mem, v.Type, rc)
   186  	}
   187  
   188  	rewriteCall := func(v *Value, newOp Op, argStart int) {
   189  		// Break aggregate args passed to call into smaller pieces.
   190  		x.rewriteCallArgs(v, argStart)
   191  		v.Op = newOp
   192  		rts := abi.RegisterTypes(v.Aux.(*AuxCall).abiInfo.OutParams())
   193  		v.Type = types.NewResults(append(rts, types.TypeMem))
   194  	}
   195  
   196  	// Rewrite calls
   197  	for _, v := range calls {
   198  		switch v.Op {
   199  		case OpStaticLECall:
   200  			rewriteCall(v, OpStaticCall, 0)
   201  		case OpTailLECall:
   202  			rewriteCall(v, OpTailCall, 0)
   203  		case OpTailLECallInter:
   204  			rewriteCall(v, OpTailCallInter, 1)
   205  		case OpClosureLECall:
   206  			rewriteCall(v, OpClosureCall, 2)
   207  		case OpInterLECall:
   208  			rewriteCall(v, OpInterCall, 1)
   209  		}
   210  	}
   211  
   212  	// Rewrite results from exit blocks
   213  	for _, b := range exitBlocks {
   214  		v := b.Controls[0]
   215  		x.rewriteFuncResults(v, b, f.OwnAux)
   216  		b.SetControl(v)
   217  	}
   218  
   219  }
   220  
   221  func (x *expandState) rewriteFuncResults(v *Value, b *Block, aux *AuxCall) {
   222  	// This is very similar to rewriteCallArgs
   223  	// differences:
   224  	// firstArg + preArgs
   225  	// sp vs auxBase
   226  
   227  	m0 := v.MemoryArg()
   228  	mem := m0
   229  
   230  	allResults := []*Value{}
   231  	var oldArgs []*Value
   232  	argsWithoutMem := v.Args[:len(v.Args)-1]
   233  
   234  	for j, a := range argsWithoutMem {
   235  		oldArgs = append(oldArgs, a)
   236  		i := int64(j)
   237  		auxType := aux.TypeOfResult(i)
   238  		auxBase := b.NewValue2A(v.Pos, OpLocalAddr, types.NewPtr(auxType), aux.NameOfResult(i), x.sp, mem)
   239  		auxOffset := int64(0)
   240  		aRegs := aux.RegsOfResult(int64(j))
   241  		if a.Op == OpDereference {
   242  			a.Op = OpLoad
   243  		}
   244  		var rc registerCursor
   245  		var result *[]*Value
   246  		if len(aRegs) > 0 {
   247  			result = &allResults
   248  		} else {
   249  			if a.Op == OpLoad && a.Args[0].Op == OpLocalAddr && a.Args[0].Aux == aux.NameOfResult(i) {
   250  				continue // Self move to output parameter
   251  			}
   252  		}
   253  		rc.init(aRegs, aux.abiInfo, result, auxBase, auxOffset)
   254  		mem = x.decomposeAsNecessary(v.Pos, b, a, mem, rc)
   255  	}
   256  	v.resetArgs()
   257  	v.AddArgs(allResults...)
   258  	v.AddArg(mem)
   259  	for _, a := range oldArgs {
   260  		if a.Uses == 0 {
   261  			if x.debug > 1 {
   262  				x.Printf("...marking %v unused\n", a.LongString())
   263  			}
   264  			x.invalidateRecursively(a)
   265  		}
   266  	}
   267  	v.Type = types.NewResults(append(abi.RegisterTypes(aux.abiInfo.OutParams()), types.TypeMem))
   268  	return
   269  }
   270  
   271  func (x *expandState) rewriteCallArgs(v *Value, firstArg int) {
   272  	if x.debug > 1 {
   273  		x.indent(3)
   274  		defer x.indent(-3)
   275  		x.Printf("rewriteCallArgs(%s; %d)\n", v.LongString(), firstArg)
   276  	}
   277  	// Thread the stores on the memory arg
   278  	aux := v.Aux.(*AuxCall)
   279  	m0 := v.MemoryArg()
   280  	mem := m0
   281  	allResults := []*Value{}
   282  	oldArgs := []*Value{}
   283  	argsWithoutMem := v.Args[firstArg : len(v.Args)-1] // Also strip closure/interface Op-specific args
   284  
   285  	sp := x.sp
   286  	if v.Op == OpTailLECall || v.Op == OpTailLECallInter {
   287  		// For tail call, we unwind the frame before the call so we'll use the caller's
   288  		// SP.
   289  		sp = v.Block.NewValue1(src.NoXPos, OpGetCallerSP, x.typs.Uintptr, mem)
   290  	}
   291  
   292  	for i, a := range argsWithoutMem { // skip leading non-parameter SSA Args and trailing mem SSA Arg.
   293  		oldArgs = append(oldArgs, a)
   294  		auxI := int64(i)
   295  		aRegs := aux.RegsOfArg(auxI)
   296  		aType := aux.TypeOfArg(auxI)
   297  
   298  		if a.Op == OpDereference {
   299  			a.Op = OpLoad
   300  		}
   301  		var rc registerCursor
   302  		var result *[]*Value
   303  		var aOffset int64
   304  		if len(aRegs) > 0 {
   305  			result = &allResults
   306  		} else {
   307  			aOffset = aux.OffsetOfArg(auxI)
   308  		}
   309  		if v.Op == OpTailLECall && a.Op == OpArg && a.AuxInt == 0 {
   310  			// It's common for a tail call passing the same arguments (e.g. method wrapper),
   311  			// so this would be a self copy. Detect this and optimize it out.
   312  			n := a.Aux.(*ir.Name)
   313  			if n.Class == ir.PPARAM && n.FrameOffset()+x.f.Config.ctxt.Arch.FixedFrameSize == aOffset {
   314  				continue
   315  			}
   316  		}
   317  		if x.debug > 1 {
   318  			x.Printf("...storeArg %s, %v, %d\n", a.LongString(), aType, aOffset)
   319  		}
   320  
   321  		rc.init(aRegs, aux.abiInfo, result, sp, aOffset)
   322  		mem = x.decomposeAsNecessary(v.Pos, v.Block, a, mem, rc)
   323  	}
   324  	var preArgStore [2]*Value
   325  	preArgs := append(preArgStore[:0], v.Args[0:firstArg]...)
   326  	v.resetArgs()
   327  	v.AddArgs(preArgs...)
   328  	v.AddArgs(allResults...)
   329  	v.AddArg(mem)
   330  	for _, a := range oldArgs {
   331  		if a.Uses == 0 {
   332  			x.invalidateRecursively(a)
   333  		}
   334  	}
   335  
   336  	return
   337  }
   338  
   339  func (x *expandState) decomposePair(pos src.XPos, b *Block, a, mem *Value, t0, t1 *types.Type, o0, o1 Op, rc *registerCursor) *Value {
   340  	e := b.NewValue1(pos, o0, t0, a)
   341  	pos = pos.WithNotStmt()
   342  	mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(t0))
   343  	e = b.NewValue1(pos, o1, t1, a)
   344  	mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(t1))
   345  	return mem
   346  }
   347  
   348  func (x *expandState) decomposeOne(pos src.XPos, b *Block, a, mem *Value, t0 *types.Type, o0 Op, rc *registerCursor) *Value {
   349  	e := b.NewValue1(pos, o0, t0, a)
   350  	pos = pos.WithNotStmt()
   351  	mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(t0))
   352  	return mem
   353  }
   354  
   355  // decomposeAsNecessary converts a value (perhaps an aggregate) passed to a call or returned by a function,
   356  // into the appropriate sequence of stores and register assignments to transmit that value in a given ABI, and
   357  // returns the current memory after this convert/rewrite (it may be the input memory, perhaps stores were needed.)
   358  // 'pos' is the source position all this is tied to
   359  // 'b' is the enclosing block
   360  // 'a' is the value to decompose
   361  // 'm0' is the input memory arg used for the first store (or returned if there are no stores)
   362  // 'rc' is a registerCursor which identifies the register/memory destination for the value
   363  func (x *expandState) decomposeAsNecessary(pos src.XPos, b *Block, a, m0 *Value, rc registerCursor) *Value {
   364  	if x.debug > 1 {
   365  		x.indent(3)
   366  		defer x.indent(-3)
   367  	}
   368  	at := a.Type
   369  	if at.Size() == 0 {
   370  		return m0
   371  	}
   372  	if a.Op == OpDereference {
   373  		a.Op = OpLoad // For purposes of parameter passing expansion, a Dereference is a Load.
   374  	}
   375  
   376  	if !rc.hasRegs() && !CanSSA(at) {
   377  		dst := x.offsetFrom(b, rc.storeDest, rc.storeOffset, types.NewPtr(at))
   378  		if x.debug > 1 {
   379  			x.Printf("...recur store %s at %s\n", a.LongString(), dst.LongString())
   380  		}
   381  		if a.Op == OpLoad {
   382  			m0 = b.NewValue3A(pos, OpMove, types.TypeMem, at, dst, a.Args[0], m0)
   383  			m0.AuxInt = at.Size()
   384  			return m0
   385  		} else {
   386  			panic(fmt.Errorf("Store of not a load"))
   387  		}
   388  	}
   389  
   390  	mem := m0
   391  	switch at.Kind() {
   392  	case types.TARRAY:
   393  		et := at.Elem()
   394  		for i := int64(0); i < at.NumElem(); i++ {
   395  			e := b.NewValue1I(pos, OpArraySelect, et, i, a)
   396  			pos = pos.WithNotStmt()
   397  			mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(et))
   398  		}
   399  		return mem
   400  
   401  	case types.TSTRUCT:
   402  		if at.IsSIMD() {
   403  			break // XXX
   404  		}
   405  		for i := 0; i < at.NumFields(); i++ {
   406  			et := at.Field(i).Type // might need to read offsets from the fields
   407  			e := b.NewValue1I(pos, OpStructSelect, et, int64(i), a)
   408  			pos = pos.WithNotStmt()
   409  			if x.debug > 1 {
   410  				x.Printf("...recur decompose %s, %v\n", e.LongString(), et)
   411  			}
   412  			mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(et))
   413  		}
   414  		return mem
   415  
   416  	case types.TSLICE:
   417  		mem = x.decomposeOne(pos, b, a, mem, at.Elem().PtrTo(), OpSlicePtr, &rc)
   418  		pos = pos.WithNotStmt()
   419  		mem = x.decomposeOne(pos, b, a, mem, x.typs.Int, OpSliceLen, &rc)
   420  		return x.decomposeOne(pos, b, a, mem, x.typs.Int, OpSliceCap, &rc)
   421  
   422  	case types.TSTRING:
   423  		return x.decomposePair(pos, b, a, mem, x.typs.BytePtr, x.typs.Int, OpStringPtr, OpStringLen, &rc)
   424  
   425  	case types.TINTER:
   426  		mem = x.decomposeOne(pos, b, a, mem, x.typs.Uintptr, OpITab, &rc)
   427  		pos = pos.WithNotStmt()
   428  		// Immediate interfaces cause so many headaches.
   429  		if a.Op == OpIMake {
   430  			data := a.Args[1]
   431  			for data.Op == OpStructMake || data.Op == OpArrayMake1 {
   432  				// A struct make might have a few zero-sized fields.
   433  				// Use the pointer-y one we know is there.
   434  				for _, a := range data.Args {
   435  					if a.Type.Size() > 0 {
   436  						data = a
   437  						break
   438  					}
   439  				}
   440  			}
   441  			return x.decomposeAsNecessary(pos, b, data, mem, rc.next(data.Type))
   442  		}
   443  		return x.decomposeOne(pos, b, a, mem, x.typs.BytePtr, OpIData, &rc)
   444  
   445  	case types.TCOMPLEX64:
   446  		return x.decomposePair(pos, b, a, mem, x.typs.Float32, x.typs.Float32, OpComplexReal, OpComplexImag, &rc)
   447  
   448  	case types.TCOMPLEX128:
   449  		return x.decomposePair(pos, b, a, mem, x.typs.Float64, x.typs.Float64, OpComplexReal, OpComplexImag, &rc)
   450  
   451  	case types.TINT64:
   452  		if at.Size() > x.regSize {
   453  			return x.decomposePair(pos, b, a, mem, x.firstType, x.secondType, x.firstOp, x.secondOp, &rc)
   454  		}
   455  	case types.TUINT64:
   456  		if at.Size() > x.regSize {
   457  			return x.decomposePair(pos, b, a, mem, x.typs.UInt32, x.typs.UInt32, x.firstOp, x.secondOp, &rc)
   458  		}
   459  	}
   460  
   461  	// An atomic type, either record the register or store it and update the memory.
   462  
   463  	if rc.hasRegs() {
   464  		if x.debug > 1 {
   465  			x.Printf("...recur addArg %s\n", a.LongString())
   466  		}
   467  		rc.addArg(a)
   468  	} else {
   469  		dst := x.offsetFrom(b, rc.storeDest, rc.storeOffset, types.NewPtr(at))
   470  		if x.debug > 1 {
   471  			x.Printf("...recur store %s at %s\n", a.LongString(), dst.LongString())
   472  		}
   473  		mem = b.NewValue3A(pos, OpStore, types.TypeMem, at, dst, a, mem)
   474  	}
   475  
   476  	return mem
   477  }
   478  
   479  // Convert scalar OpArg into the proper OpWhateverArg instruction
   480  // Convert scalar OpSelectN into perhaps-differently-indexed OpSelectN
   481  // Convert aggregate OpArg into Make of its parts (which are eventually scalars)
   482  // Convert aggregate OpSelectN into Make of its parts (which are eventually scalars)
   483  // Returns the converted value.
   484  //
   485  //   - "pos" the position for any generated instructions
   486  //   - "b" the block for any generated instructions
   487  //   - "container" the outermost OpArg/OpSelectN
   488  //   - "a" the instruction to overwrite, if any (only the outermost caller)
   489  //   - "m0" the memory arg for any loads that are necessary
   490  //   - "at" the type of the Arg/part
   491  //   - "rc" the register/memory cursor locating the various parts of the Arg.
   492  func (x *expandState) rewriteSelectOrArg(pos src.XPos, b *Block, container, a, m0 *Value, at *types.Type, rc registerCursor) *Value {
   493  
   494  	if at == types.TypeMem {
   495  		a.copyOf(m0)
   496  		return a
   497  	}
   498  
   499  	makeOf := func(a *Value, op Op, args []*Value) *Value {
   500  		if a == nil {
   501  			a = b.NewValue0(pos, op, at)
   502  			a.AddArgs(args...)
   503  		} else {
   504  			a.resetArgs()
   505  			a.Aux, a.AuxInt = nil, 0
   506  			a.Pos, a.Op, a.Type = pos, op, at
   507  			a.AddArgs(args...)
   508  		}
   509  		return a
   510  	}
   511  
   512  	if at.Size() == 0 {
   513  		// For consistency, create these values even though they'll ultimately be unused
   514  		return makeOf(a, OpEmpty, nil)
   515  	}
   516  
   517  	sk := selKey{from: container, size: 0, offsetOrIndex: rc.storeOffset, typ: at}
   518  	dupe := x.commonSelectors[sk]
   519  	if dupe != nil {
   520  		if a == nil {
   521  			return dupe
   522  		}
   523  		a.copyOf(dupe)
   524  		return a
   525  	}
   526  
   527  	var argStore [10]*Value
   528  	args := argStore[:0]
   529  
   530  	addArg := func(a0 *Value) {
   531  		if a0 == nil {
   532  			as := "<nil>"
   533  			if a != nil {
   534  				as = a.LongString()
   535  			}
   536  			panic(fmt.Errorf("a0 should not be nil, a=%v, container=%v, at=%v", as, container.LongString(), at))
   537  		}
   538  		args = append(args, a0)
   539  	}
   540  
   541  	switch at.Kind() {
   542  	case types.TARRAY:
   543  		et := at.Elem()
   544  		for i := int64(0); i < at.NumElem(); i++ {
   545  			e := x.rewriteSelectOrArg(pos, b, container, nil, m0, et, rc.next(et))
   546  			addArg(e)
   547  		}
   548  		a = makeOf(a, OpArrayMake1, args)
   549  		x.commonSelectors[sk] = a
   550  		return a
   551  
   552  	case types.TSTRUCT:
   553  		// Assume ssagen/ssa.go (in buildssa) spills large aggregates so they won't appear here.
   554  		if at.IsSIMD() {
   555  			break // XXX
   556  		}
   557  		for i := 0; i < at.NumFields(); i++ {
   558  			et := at.Field(i).Type
   559  			e := x.rewriteSelectOrArg(pos, b, container, nil, m0, et, rc.next(et))
   560  			if e == nil {
   561  				panic(fmt.Errorf("nil e, et=%v, et.Size()=%d, i=%d", et, et.Size(), i))
   562  			}
   563  			addArg(e)
   564  			pos = pos.WithNotStmt()
   565  		}
   566  		if at.NumFields() > MaxStruct && !types.IsDirectIface(at) {
   567  			panic(fmt.Errorf("Too many fields (%d, %d bytes), container=%s", at.NumFields(), at.Size(), container.LongString()))
   568  		}
   569  		a = makeOf(a, OpStructMake, args)
   570  		x.commonSelectors[sk] = a
   571  		return a
   572  
   573  	case types.TSLICE:
   574  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, at.Elem().PtrTo(), rc.next(x.typs.BytePtr)))
   575  		pos = pos.WithNotStmt()
   576  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Int, rc.next(x.typs.Int)))
   577  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Int, rc.next(x.typs.Int)))
   578  		a = makeOf(a, OpSliceMake, args)
   579  		x.commonSelectors[sk] = a
   580  		return a
   581  
   582  	case types.TSTRING:
   583  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.BytePtr, rc.next(x.typs.BytePtr)))
   584  		pos = pos.WithNotStmt()
   585  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Int, rc.next(x.typs.Int)))
   586  		a = makeOf(a, OpStringMake, args)
   587  		x.commonSelectors[sk] = a
   588  		return a
   589  
   590  	case types.TINTER:
   591  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Uintptr, rc.next(x.typs.Uintptr)))
   592  		pos = pos.WithNotStmt()
   593  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.BytePtr, rc.next(x.typs.BytePtr)))
   594  		a = makeOf(a, OpIMake, args)
   595  		x.commonSelectors[sk] = a
   596  		return a
   597  
   598  	case types.TCOMPLEX64:
   599  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Float32, rc.next(x.typs.Float32)))
   600  		pos = pos.WithNotStmt()
   601  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Float32, rc.next(x.typs.Float32)))
   602  		a = makeOf(a, OpComplexMake, args)
   603  		x.commonSelectors[sk] = a
   604  		return a
   605  
   606  	case types.TCOMPLEX128:
   607  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Float64, rc.next(x.typs.Float64)))
   608  		pos = pos.WithNotStmt()
   609  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Float64, rc.next(x.typs.Float64)))
   610  		a = makeOf(a, OpComplexMake, args)
   611  		x.commonSelectors[sk] = a
   612  		return a
   613  
   614  	case types.TINT64:
   615  		if at.Size() > x.regSize {
   616  			addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.firstType, rc.next(x.firstType)))
   617  			pos = pos.WithNotStmt()
   618  			addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.secondType, rc.next(x.secondType)))
   619  			if !x.f.Config.BigEndian {
   620  				// Int64Make args are big, little
   621  				args[0], args[1] = args[1], args[0]
   622  			}
   623  			a = makeOf(a, OpInt64Make, args)
   624  			x.commonSelectors[sk] = a
   625  			return a
   626  		}
   627  	case types.TUINT64:
   628  		if at.Size() > x.regSize {
   629  			addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.UInt32, rc.next(x.typs.UInt32)))
   630  			pos = pos.WithNotStmt()
   631  			addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.UInt32, rc.next(x.typs.UInt32)))
   632  			if !x.f.Config.BigEndian {
   633  				// Int64Make args are big, little
   634  				args[0], args[1] = args[1], args[0]
   635  			}
   636  			a = makeOf(a, OpInt64Make, args)
   637  			x.commonSelectors[sk] = a
   638  			return a
   639  		}
   640  	}
   641  
   642  	// An atomic type, either record the register or store it and update the memory.
   643  
   644  	// Depending on the container Op, the leaves are either OpSelectN or OpArg{Int,Float}Reg
   645  
   646  	if container.Op == OpArg {
   647  		if rc.hasRegs() {
   648  			op, i := rc.ArgOpAndRegisterFor()
   649  			name := container.Aux.(*ir.Name)
   650  			a = makeOf(a, op, nil)
   651  			a.AuxInt = i
   652  			a.Aux = &AuxNameOffset{name, rc.storeOffset}
   653  		} else {
   654  			key := selKey{container, rc.storeOffset, at.Size(), at}
   655  			w := x.commonArgs[key]
   656  			if w != nil && w.Uses != 0 {
   657  				if a == nil {
   658  					a = w
   659  				} else {
   660  					a.copyOf(w)
   661  				}
   662  			} else {
   663  				if a == nil {
   664  					aux := container.Aux
   665  					auxInt := container.AuxInt + rc.storeOffset
   666  					a = container.Block.NewValue0IA(container.Pos, OpArg, at, auxInt, aux)
   667  				} else {
   668  					// do nothing, the original should be okay.
   669  				}
   670  				x.commonArgs[key] = a
   671  			}
   672  		}
   673  	} else if container.Op == OpSelectN {
   674  		call := container.Args[0]
   675  		aux := call.Aux.(*AuxCall)
   676  		which := container.AuxInt
   677  
   678  		if at == types.TypeMem {
   679  			if a != m0 || a != x.memForCall[call.ID] {
   680  				panic(fmt.Errorf("Memories %s, %s, and %s should all be equal after %s", a.LongString(), m0.LongString(), x.memForCall[call.ID], call.LongString()))
   681  			}
   682  		} else if rc.hasRegs() {
   683  			firstReg := uint32(0)
   684  			for i := 0; i < int(which); i++ {
   685  				firstReg += uint32(len(aux.abiInfo.OutParam(i).Registers))
   686  			}
   687  			reg := int64(rc.nextSlice + Abi1RO(firstReg))
   688  			a = makeOf(a, OpSelectN, []*Value{call})
   689  			a.AuxInt = reg
   690  		} else {
   691  			off := x.offsetFrom(x.f.Entry, x.sp, rc.storeOffset+aux.OffsetOfResult(which), types.NewPtr(at))
   692  			a = makeOf(a, OpLoad, []*Value{off, m0})
   693  		}
   694  
   695  	} else {
   696  		panic(fmt.Errorf("Expected container OpArg or OpSelectN, saw %v instead", container.LongString()))
   697  	}
   698  
   699  	x.commonSelectors[sk] = a
   700  	return a
   701  }
   702  
   703  // rewriteWideSelectToStores handles the case of a SelectN'd result from a function call that is too large for SSA,
   704  // but is transferred in registers.  In this case the register cursor tracks both operands; the register sources and
   705  // the memory destinations.
   706  // This returns the memory flowing out of the last store
   707  func (x *expandState) rewriteWideSelectToStores(pos src.XPos, b *Block, container, m0 *Value, at *types.Type, rc registerCursor) *Value {
   708  
   709  	if at.Size() == 0 {
   710  		return m0
   711  	}
   712  
   713  	switch at.Kind() {
   714  	case types.TARRAY:
   715  		et := at.Elem()
   716  		for i := int64(0); i < at.NumElem(); i++ {
   717  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, et, rc.next(et))
   718  		}
   719  		return m0
   720  
   721  	case types.TSTRUCT:
   722  		// Assume ssagen/ssa.go (in buildssa) spills large aggregates so they won't appear here.
   723  		if at.IsSIMD() {
   724  			break // XXX
   725  		}
   726  		for i := 0; i < at.NumFields(); i++ {
   727  			et := at.Field(i).Type
   728  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, et, rc.next(et))
   729  			pos = pos.WithNotStmt()
   730  		}
   731  		return m0
   732  
   733  	case types.TSLICE:
   734  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, at.Elem().PtrTo(), rc.next(x.typs.BytePtr))
   735  		pos = pos.WithNotStmt()
   736  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Int, rc.next(x.typs.Int))
   737  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Int, rc.next(x.typs.Int))
   738  		return m0
   739  
   740  	case types.TSTRING:
   741  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.BytePtr, rc.next(x.typs.BytePtr))
   742  		pos = pos.WithNotStmt()
   743  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Int, rc.next(x.typs.Int))
   744  		return m0
   745  
   746  	case types.TINTER:
   747  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Uintptr, rc.next(x.typs.Uintptr))
   748  		pos = pos.WithNotStmt()
   749  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.BytePtr, rc.next(x.typs.BytePtr))
   750  		return m0
   751  
   752  	case types.TCOMPLEX64:
   753  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Float32, rc.next(x.typs.Float32))
   754  		pos = pos.WithNotStmt()
   755  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Float32, rc.next(x.typs.Float32))
   756  		return m0
   757  
   758  	case types.TCOMPLEX128:
   759  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Float64, rc.next(x.typs.Float64))
   760  		pos = pos.WithNotStmt()
   761  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Float64, rc.next(x.typs.Float64))
   762  		return m0
   763  
   764  	case types.TINT64:
   765  		if at.Size() > x.regSize {
   766  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.firstType, rc.next(x.firstType))
   767  			pos = pos.WithNotStmt()
   768  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.secondType, rc.next(x.secondType))
   769  			return m0
   770  		}
   771  	case types.TUINT64:
   772  		if at.Size() > x.regSize {
   773  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.UInt32, rc.next(x.typs.UInt32))
   774  			pos = pos.WithNotStmt()
   775  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.UInt32, rc.next(x.typs.UInt32))
   776  			return m0
   777  		}
   778  	}
   779  
   780  	// TODO could change treatment of too-large OpArg, would deal with it here.
   781  	if container.Op == OpSelectN {
   782  		call := container.Args[0]
   783  		aux := call.Aux.(*AuxCall)
   784  		which := container.AuxInt
   785  
   786  		if rc.hasRegs() {
   787  			firstReg := uint32(0)
   788  			for i := 0; i < int(which); i++ {
   789  				firstReg += uint32(len(aux.abiInfo.OutParam(i).Registers))
   790  			}
   791  			reg := int64(rc.nextSlice + Abi1RO(firstReg))
   792  			a := b.NewValue1I(pos, OpSelectN, at, reg, call)
   793  			dst := x.offsetFrom(b, rc.storeDest, rc.storeOffset, types.NewPtr(at))
   794  			m0 = b.NewValue3A(pos, OpStore, types.TypeMem, at, dst, a, m0)
   795  		} else {
   796  			panic(fmt.Errorf("Expected rc to have registers"))
   797  		}
   798  	} else {
   799  		panic(fmt.Errorf("Expected container OpSelectN, saw %v instead", container.LongString()))
   800  	}
   801  	return m0
   802  }
   803  
   804  func isBlockMultiValueExit(b *Block) bool {
   805  	return (b.Kind == block.BlockRet || b.Kind == block.BlockRetJmp) && b.Controls[0] != nil && b.Controls[0].Op == OpMakeResult
   806  }
   807  
   808  type Abi1RO uint8 // An offset within a parameter's slice of register indices, for abi1.
   809  
   810  // A registerCursor tracks which register is used for an Arg or regValues, or a piece of such.
   811  type registerCursor struct {
   812  	storeDest   *Value // if there are no register targets, then this is the base of the store.
   813  	storeOffset int64
   814  	regs        []abi.RegIndex // the registers available for this Arg/result (which is all in registers or not at all)
   815  	nextSlice   Abi1RO         // the next register/register-slice offset
   816  	config      *abi.ABIConfig
   817  	regValues   *[]*Value // values assigned to registers accumulate here
   818  }
   819  
   820  func (c *registerCursor) String() string {
   821  	dest := "<none>"
   822  	if c.storeDest != nil {
   823  		dest = fmt.Sprintf("%s+%d", c.storeDest.String(), c.storeOffset)
   824  	}
   825  	regs := "<none>"
   826  	if c.regValues != nil {
   827  		regs = ""
   828  		for i, x := range *c.regValues {
   829  			if i > 0 {
   830  				regs = regs + "; "
   831  			}
   832  			regs = regs + x.LongString()
   833  		}
   834  	}
   835  
   836  	// not printing the config because that has not been useful
   837  	return fmt.Sprintf("RCSR{storeDest=%v, regsLen=%d, nextSlice=%d, regValues=[%s]}", dest, len(c.regs), c.nextSlice, regs)
   838  }
   839  
   840  // next effectively post-increments the register cursor; the receiver is advanced,
   841  // the (aligned) old value is returned.
   842  func (c *registerCursor) next(t *types.Type) registerCursor {
   843  	c.storeOffset = types.RoundUp(c.storeOffset, t.Alignment())
   844  	rc := *c
   845  	c.storeOffset = types.RoundUp(c.storeOffset+t.Size(), t.Alignment())
   846  	if int(c.nextSlice) < len(c.regs) {
   847  		w := c.config.NumParamRegs(t)
   848  		c.nextSlice += Abi1RO(w)
   849  	}
   850  	return rc
   851  }
   852  
   853  // plus returns a register cursor offset from the original, without modifying the original.
   854  func (c *registerCursor) plus(regWidth Abi1RO) registerCursor {
   855  	rc := *c
   856  	rc.nextSlice += regWidth
   857  	return rc
   858  }
   859  
   860  func (c *registerCursor) init(regs []abi.RegIndex, info *abi.ABIParamResultInfo, result *[]*Value, storeDest *Value, storeOffset int64) {
   861  	c.regs = regs
   862  	c.nextSlice = 0
   863  	c.storeOffset = storeOffset
   864  	c.storeDest = storeDest
   865  	c.config = info.Config()
   866  	c.regValues = result
   867  }
   868  
   869  func (c *registerCursor) addArg(v *Value) {
   870  	*c.regValues = append(*c.regValues, v)
   871  }
   872  
   873  func (c *registerCursor) hasRegs() bool {
   874  	return len(c.regs) > 0
   875  }
   876  
   877  func (c *registerCursor) ArgOpAndRegisterFor() (Op, int64) {
   878  	r := c.regs[c.nextSlice]
   879  	return ArgOpAndRegisterFor(r, c.config)
   880  }
   881  
   882  // ArgOpAndRegisterFor converts an abi register index into an ssa Op and corresponding
   883  // arg register index.
   884  func ArgOpAndRegisterFor(r abi.RegIndex, abiConfig *abi.ABIConfig) (Op, int64) {
   885  	i := abiConfig.FloatIndexFor(r)
   886  	if i >= 0 { // float PR
   887  		return OpArgFloatReg, i
   888  	}
   889  	return OpArgIntReg, int64(r)
   890  }
   891  
   892  type selKey struct {
   893  	from          *Value // what is selected from
   894  	offsetOrIndex int64  // whatever is appropriate for the selector
   895  	size          int64
   896  	typ           *types.Type
   897  }
   898  
   899  type expandState struct {
   900  	f       *Func
   901  	debug   int // odd values log lost statement markers, so likely settings are 1 (stmts), 2 (expansion), and 3 (both)
   902  	regSize int64
   903  	sp      *Value
   904  	typs    *Types
   905  
   906  	firstOp    Op          // for 64-bit integers on 32-bit machines, first word in memory
   907  	secondOp   Op          // for 64-bit integers on 32-bit machines, second word in memory
   908  	firstType  *types.Type // first half type, for Int64
   909  	secondType *types.Type // second half type, for Int64
   910  
   911  	wideSelects     map[*Value]*Value // Selects that are not SSA-able, mapped to consuming stores.
   912  	commonSelectors map[selKey]*Value // used to de-dupe selectors
   913  	commonArgs      map[selKey]*Value // used to de-dupe OpArg/OpArgIntReg/OpArgFloatReg
   914  	memForCall      map[ID]*Value     // For a call, need to know the unique selector that gets the mem.
   915  	indentLevel     int               // Indentation for debugging recursion
   916  }
   917  
   918  // offsetFrom creates an offset from a pointer, simplifying chained offsets and offsets from SP
   919  func (x *expandState) offsetFrom(b *Block, from *Value, offset int64, pt *types.Type) *Value {
   920  	ft := from.Type
   921  	if offset == 0 {
   922  		if ft == pt {
   923  			return from
   924  		}
   925  		// This captures common, (apparently) safe cases.  The unsafe cases involve ft == uintptr
   926  		if (ft.IsPtr() || ft.IsUnsafePtr()) && pt.IsPtr() {
   927  			return from
   928  		}
   929  	}
   930  	// Simplify, canonicalize
   931  	for from.Op == OpOffPtr {
   932  		offset += from.AuxInt
   933  		from = from.Args[0]
   934  	}
   935  	if from == x.sp {
   936  		return x.f.ConstOffPtrSP(pt, offset, x.sp)
   937  	}
   938  	return b.NewValue1I(from.Pos.WithNotStmt(), OpOffPtr, pt, offset, from)
   939  }
   940  
   941  // prAssignForArg returns the ABIParamAssignment for v, assumed to be an OpArg.
   942  func (x *expandState) prAssignForArg(v *Value) *abi.ABIParamAssignment {
   943  	if v.Op != OpArg {
   944  		panic(fmt.Errorf("Wanted OpArg, instead saw %s", v.LongString()))
   945  	}
   946  	return ParamAssignmentForArgName(x.f, v.Aux.(*ir.Name))
   947  }
   948  
   949  // ParamAssignmentForArgName returns the ABIParamAssignment for f's arg with matching name.
   950  func ParamAssignmentForArgName(f *Func, name *ir.Name) *abi.ABIParamAssignment {
   951  	abiInfo := f.OwnAux.abiInfo
   952  	ip := abiInfo.InParams()
   953  	for i, a := range ip {
   954  		if a.Name == name {
   955  			return &ip[i]
   956  		}
   957  	}
   958  	panic(fmt.Errorf("Did not match param %v in prInfo %+v", name, abiInfo.InParams()))
   959  }
   960  
   961  // indent increments (or decrements) the indentation.
   962  func (x *expandState) indent(n int) {
   963  	x.indentLevel += n
   964  }
   965  
   966  // Printf does an indented fmt.Printf on the format and args.
   967  func (x *expandState) Printf(format string, a ...any) (n int, err error) {
   968  	if x.indentLevel > 0 {
   969  		fmt.Printf("%[1]*s", x.indentLevel, "")
   970  	}
   971  	return fmt.Printf(format, a...)
   972  }
   973  
   974  func (x *expandState) invalidateRecursively(a *Value) {
   975  	var s string
   976  	if x.debug > 0 {
   977  		plus := " "
   978  		if a.Pos.IsStmt() == src.PosIsStmt {
   979  			plus = " +"
   980  		}
   981  		s = a.String() + plus + a.Pos.LineNumber() + " " + a.LongString()
   982  		if x.debug > 1 {
   983  			x.Printf("...marking %v unused\n", s)
   984  		}
   985  	}
   986  	lost := a.invalidateRecursively()
   987  	if x.debug&1 != 0 && lost { // For odd values of x.debug, do this.
   988  		x.Printf("Lost statement marker in %s on former %s\n", base.Ctxt.Pkgpath+"."+x.f.Name, s)
   989  	}
   990  }
   991  

View as plain text