Source file src/cmd/compile/internal/ssa/expand_calls.go

     1  // Copyright 2020 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package ssa
     6  
     7  import (
     8  	"cmd/compile/internal/abi"
     9  	"cmd/compile/internal/base"
    10  	"cmd/compile/internal/ir"
    11  	"cmd/compile/internal/types"
    12  	"cmd/internal/src"
    13  	"fmt"
    14  )
    15  
    16  func postExpandCallsDecompose(f *Func) {
    17  	decomposeUser(f)    // redo user decompose to cleanup after expand calls
    18  	decomposeBuiltIn(f) // handles both regular decomposition and cleanup.
    19  }
    20  
    21  func expandCalls(f *Func) {
    22  	// Convert each aggregate arg to a call into "dismantle aggregate, store/pass parts"
    23  	// Convert each aggregate result from a call into "assemble aggregate from parts"
    24  	// Convert each multivalue exit into "dismantle aggregate, store/return parts"
    25  	// Convert incoming aggregate arg into assembly of parts.
    26  	// Feed modified AST to decompose.
    27  
    28  	sp, _ := f.spSb()
    29  
    30  	x := &expandState{
    31  		f:               f,
    32  		debug:           f.pass.debug,
    33  		regSize:         f.Config.RegSize,
    34  		sp:              sp,
    35  		typs:            &f.Config.Types,
    36  		wideSelects:     make(map[*Value]*Value),
    37  		commonArgs:      make(map[selKey]*Value),
    38  		commonSelectors: make(map[selKey]*Value),
    39  		memForCall:      make(map[ID]*Value),
    40  	}
    41  
    42  	// For 32-bit, need to deal with decomposition of 64-bit integers, which depends on endianness.
    43  	if f.Config.BigEndian {
    44  		x.firstOp = OpInt64Hi
    45  		x.secondOp = OpInt64Lo
    46  		x.firstType = x.typs.Int32
    47  		x.secondType = x.typs.UInt32
    48  	} else {
    49  		x.firstOp = OpInt64Lo
    50  		x.secondOp = OpInt64Hi
    51  		x.firstType = x.typs.UInt32
    52  		x.secondType = x.typs.Int32
    53  	}
    54  
    55  	// Defer select processing until after all calls and selects are seen.
    56  	var selects []*Value
    57  	var calls []*Value
    58  	var args []*Value
    59  	var exitBlocks []*Block
    60  
    61  	var m0 *Value
    62  
    63  	// Accumulate lists of calls, args, selects, and exit blocks to process,
    64  	// note "wide" selects consumed by stores,
    65  	// rewrite mem for each call,
    66  	// rewrite each OpSelectNAddr.
    67  	for _, b := range f.Blocks {
    68  		for _, v := range b.Values {
    69  			switch v.Op {
    70  			case OpInitMem:
    71  				m0 = v
    72  
    73  			case OpClosureLECall, OpInterLECall, OpStaticLECall, OpTailLECall:
    74  				calls = append(calls, v)
    75  
    76  			case OpArg:
    77  				args = append(args, v)
    78  
    79  			case OpStore:
    80  				if a := v.Args[1]; a.Op == OpSelectN && !CanSSA(a.Type) {
    81  					if a.Uses > 1 {
    82  						panic(fmt.Errorf("Saw double use of wide SelectN %s operand of Store %s",
    83  							a.LongString(), v.LongString()))
    84  					}
    85  					x.wideSelects[a] = v
    86  				}
    87  
    88  			case OpSelectN:
    89  				if v.Type == types.TypeMem {
    90  					// rewrite the mem selector in place
    91  					call := v.Args[0]
    92  					aux := call.Aux.(*AuxCall)
    93  					mem := x.memForCall[call.ID]
    94  					if mem == nil {
    95  						v.AuxInt = int64(aux.abiInfo.OutRegistersUsed())
    96  						x.memForCall[call.ID] = v
    97  					} else {
    98  						panic(fmt.Errorf("Saw two memories for call %v, %v and %v", call, mem, v))
    99  					}
   100  				} else {
   101  					selects = append(selects, v)
   102  				}
   103  
   104  			case OpSelectNAddr:
   105  				call := v.Args[0]
   106  				which := v.AuxInt
   107  				aux := call.Aux.(*AuxCall)
   108  				pt := v.Type
   109  				off := x.offsetFrom(x.f.Entry, x.sp, aux.OffsetOfResult(which), pt)
   110  				v.copyOf(off)
   111  			}
   112  		}
   113  
   114  		// rewrite function results from an exit block
   115  		// values returned by function need to be split out into registers.
   116  		if isBlockMultiValueExit(b) {
   117  			exitBlocks = append(exitBlocks, b)
   118  		}
   119  	}
   120  
   121  	// Convert each aggregate arg into Make of its parts (and so on, to primitive types)
   122  	for _, v := range args {
   123  		var rc registerCursor
   124  		a := x.prAssignForArg(v)
   125  		aux := x.f.OwnAux
   126  		regs := a.Registers
   127  		var offset int64
   128  		if len(regs) == 0 {
   129  			offset = a.FrameOffset(aux.abiInfo)
   130  		}
   131  		auxBase := x.offsetFrom(x.f.Entry, x.sp, offset, types.NewPtr(v.Type))
   132  		rc.init(regs, aux.abiInfo, nil, auxBase, 0)
   133  		x.rewriteSelectOrArg(f.Entry.Pos, f.Entry, v, v, m0, v.Type, rc)
   134  	}
   135  
   136  	// Rewrite selects of results (which may be aggregates) into make-aggregates of register/memory-targeted selects
   137  	for _, v := range selects {
   138  		if v.Op == OpInvalid {
   139  			continue
   140  		}
   141  
   142  		call := v.Args[0]
   143  		aux := call.Aux.(*AuxCall)
   144  		mem := x.memForCall[call.ID]
   145  		if mem == nil {
   146  			mem = call.Block.NewValue1I(call.Pos, OpSelectN, types.TypeMem, int64(aux.abiInfo.OutRegistersUsed()), call)
   147  			x.memForCall[call.ID] = mem
   148  		}
   149  
   150  		i := v.AuxInt
   151  		regs := aux.RegsOfResult(i)
   152  
   153  		// If this select cannot fit into SSA and is stored, either disaggregate to register stores, or mem-mem move.
   154  		if store := x.wideSelects[v]; store != nil {
   155  			// Use the mem that comes from the store operation.
   156  			storeAddr := store.Args[0]
   157  			mem := store.Args[2]
   158  			if len(regs) > 0 {
   159  				// Cannot do a rewrite that builds up a result from pieces; instead, copy pieces to the store operation.
   160  				var rc registerCursor
   161  				rc.init(regs, aux.abiInfo, nil, storeAddr, 0)
   162  				mem = x.rewriteWideSelectToStores(call.Pos, call.Block, v, mem, v.Type, rc)
   163  				store.copyOf(mem)
   164  			} else {
   165  				// Move directly from AuxBase to store target; rewrite the store instruction.
   166  				offset := aux.OffsetOfResult(i)
   167  				auxBase := x.offsetFrom(x.f.Entry, x.sp, offset, types.NewPtr(v.Type))
   168  				// was Store dst, v, mem
   169  				// now Move dst, auxBase, mem
   170  				move := store.Block.NewValue3A(store.Pos, OpMove, types.TypeMem, v.Type, storeAddr, auxBase, mem)
   171  				move.AuxInt = v.Type.Size()
   172  				store.copyOf(move)
   173  			}
   174  			continue
   175  		}
   176  
   177  		var auxBase *Value
   178  		if len(regs) == 0 {
   179  			offset := aux.OffsetOfResult(i)
   180  			auxBase = x.offsetFrom(x.f.Entry, x.sp, offset, types.NewPtr(v.Type))
   181  		}
   182  		var rc registerCursor
   183  		rc.init(regs, aux.abiInfo, nil, auxBase, 0)
   184  		x.rewriteSelectOrArg(call.Pos, call.Block, v, v, mem, v.Type, rc)
   185  	}
   186  
   187  	rewriteCall := func(v *Value, newOp Op, argStart int) {
   188  		// Break aggregate args passed to call into smaller pieces.
   189  		x.rewriteCallArgs(v, argStart)
   190  		v.Op = newOp
   191  		rts := abi.RegisterTypes(v.Aux.(*AuxCall).abiInfo.OutParams())
   192  		v.Type = types.NewResults(append(rts, types.TypeMem))
   193  	}
   194  
   195  	// Rewrite calls
   196  	for _, v := range calls {
   197  		switch v.Op {
   198  		case OpStaticLECall:
   199  			rewriteCall(v, OpStaticCall, 0)
   200  		case OpTailLECall:
   201  			rewriteCall(v, OpTailCall, 0)
   202  		case OpClosureLECall:
   203  			rewriteCall(v, OpClosureCall, 2)
   204  		case OpInterLECall:
   205  			rewriteCall(v, OpInterCall, 1)
   206  		}
   207  	}
   208  
   209  	// Rewrite results from exit blocks
   210  	for _, b := range exitBlocks {
   211  		v := b.Controls[0]
   212  		x.rewriteFuncResults(v, b, f.OwnAux)
   213  		b.SetControl(v)
   214  	}
   215  
   216  }
   217  
   218  func (x *expandState) rewriteFuncResults(v *Value, b *Block, aux *AuxCall) {
   219  	// This is very similar to rewriteCallArgs
   220  	// differences:
   221  	// firstArg + preArgs
   222  	// sp vs auxBase
   223  
   224  	m0 := v.MemoryArg()
   225  	mem := m0
   226  
   227  	allResults := []*Value{}
   228  	var oldArgs []*Value
   229  	argsWithoutMem := v.Args[:len(v.Args)-1]
   230  
   231  	for j, a := range argsWithoutMem {
   232  		oldArgs = append(oldArgs, a)
   233  		i := int64(j)
   234  		auxType := aux.TypeOfResult(i)
   235  		auxBase := b.NewValue2A(v.Pos, OpLocalAddr, types.NewPtr(auxType), aux.NameOfResult(i), x.sp, mem)
   236  		auxOffset := int64(0)
   237  		aRegs := aux.RegsOfResult(int64(j))
   238  		if a.Op == OpDereference {
   239  			a.Op = OpLoad
   240  		}
   241  		var rc registerCursor
   242  		var result *[]*Value
   243  		if len(aRegs) > 0 {
   244  			result = &allResults
   245  		} else {
   246  			if a.Op == OpLoad && a.Args[0].Op == OpLocalAddr && a.Args[0].Aux == aux.NameOfResult(i) {
   247  				continue // Self move to output parameter
   248  			}
   249  		}
   250  		rc.init(aRegs, aux.abiInfo, result, auxBase, auxOffset)
   251  		mem = x.decomposeAsNecessary(v.Pos, b, a, mem, rc)
   252  	}
   253  	v.resetArgs()
   254  	v.AddArgs(allResults...)
   255  	v.AddArg(mem)
   256  	for _, a := range oldArgs {
   257  		if a.Uses == 0 {
   258  			if x.debug > 1 {
   259  				x.Printf("...marking %v unused\n", a.LongString())
   260  			}
   261  			x.invalidateRecursively(a)
   262  		}
   263  	}
   264  	v.Type = types.NewResults(append(abi.RegisterTypes(aux.abiInfo.OutParams()), types.TypeMem))
   265  	return
   266  }
   267  
   268  func (x *expandState) rewriteCallArgs(v *Value, firstArg int) {
   269  	if x.debug > 1 {
   270  		x.indent(3)
   271  		defer x.indent(-3)
   272  		x.Printf("rewriteCallArgs(%s; %d)\n", v.LongString(), firstArg)
   273  	}
   274  	// Thread the stores on the memory arg
   275  	aux := v.Aux.(*AuxCall)
   276  	m0 := v.MemoryArg()
   277  	mem := m0
   278  	allResults := []*Value{}
   279  	oldArgs := []*Value{}
   280  	argsWithoutMem := v.Args[firstArg : len(v.Args)-1] // Also strip closure/interface Op-specific args
   281  
   282  	sp := x.sp
   283  	if v.Op == OpTailLECall {
   284  		// For tail call, we unwind the frame before the call so we'll use the caller's
   285  		// SP.
   286  		sp = v.Block.NewValue1(src.NoXPos, OpGetCallerSP, x.typs.Uintptr, mem)
   287  	}
   288  
   289  	for i, a := range argsWithoutMem { // skip leading non-parameter SSA Args and trailing mem SSA Arg.
   290  		oldArgs = append(oldArgs, a)
   291  		auxI := int64(i)
   292  		aRegs := aux.RegsOfArg(auxI)
   293  		aType := aux.TypeOfArg(auxI)
   294  
   295  		if a.Op == OpDereference {
   296  			a.Op = OpLoad
   297  		}
   298  		var rc registerCursor
   299  		var result *[]*Value
   300  		var aOffset int64
   301  		if len(aRegs) > 0 {
   302  			result = &allResults
   303  		} else {
   304  			aOffset = aux.OffsetOfArg(auxI)
   305  		}
   306  		if v.Op == OpTailLECall && a.Op == OpArg && a.AuxInt == 0 {
   307  			// It's common for a tail call passing the same arguments (e.g. method wrapper),
   308  			// so this would be a self copy. Detect this and optimize it out.
   309  			n := a.Aux.(*ir.Name)
   310  			if n.Class == ir.PPARAM && n.FrameOffset()+x.f.Config.ctxt.Arch.FixedFrameSize == aOffset {
   311  				continue
   312  			}
   313  		}
   314  		if x.debug > 1 {
   315  			x.Printf("...storeArg %s, %v, %d\n", a.LongString(), aType, aOffset)
   316  		}
   317  
   318  		rc.init(aRegs, aux.abiInfo, result, sp, aOffset)
   319  		mem = x.decomposeAsNecessary(v.Pos, v.Block, a, mem, rc)
   320  	}
   321  	var preArgStore [2]*Value
   322  	preArgs := append(preArgStore[:0], v.Args[0:firstArg]...)
   323  	v.resetArgs()
   324  	v.AddArgs(preArgs...)
   325  	v.AddArgs(allResults...)
   326  	v.AddArg(mem)
   327  	for _, a := range oldArgs {
   328  		if a.Uses == 0 {
   329  			x.invalidateRecursively(a)
   330  		}
   331  	}
   332  
   333  	return
   334  }
   335  
   336  func (x *expandState) decomposePair(pos src.XPos, b *Block, a, mem *Value, t0, t1 *types.Type, o0, o1 Op, rc *registerCursor) *Value {
   337  	e := b.NewValue1(pos, o0, t0, a)
   338  	pos = pos.WithNotStmt()
   339  	mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(t0))
   340  	e = b.NewValue1(pos, o1, t1, a)
   341  	mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(t1))
   342  	return mem
   343  }
   344  
   345  func (x *expandState) decomposeOne(pos src.XPos, b *Block, a, mem *Value, t0 *types.Type, o0 Op, rc *registerCursor) *Value {
   346  	e := b.NewValue1(pos, o0, t0, a)
   347  	pos = pos.WithNotStmt()
   348  	mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(t0))
   349  	return mem
   350  }
   351  
   352  // decomposeAsNecessary converts a value (perhaps an aggregate) passed to a call or returned by a function,
   353  // into the appropriate sequence of stores and register assignments to transmit that value in a given ABI, and
   354  // returns the current memory after this convert/rewrite (it may be the input memory, perhaps stores were needed.)
   355  // 'pos' is the source position all this is tied to
   356  // 'b' is the enclosing block
   357  // 'a' is the value to decompose
   358  // 'm0' is the input memory arg used for the first store (or returned if there are no stores)
   359  // 'rc' is a registerCursor which identifies the register/memory destination for the value
   360  func (x *expandState) decomposeAsNecessary(pos src.XPos, b *Block, a, m0 *Value, rc registerCursor) *Value {
   361  	if x.debug > 1 {
   362  		x.indent(3)
   363  		defer x.indent(-3)
   364  	}
   365  	at := a.Type
   366  	if at.Size() == 0 {
   367  		return m0
   368  	}
   369  	if a.Op == OpDereference {
   370  		a.Op = OpLoad // For purposes of parameter passing expansion, a Dereference is a Load.
   371  	}
   372  
   373  	if !rc.hasRegs() && !CanSSA(at) {
   374  		dst := x.offsetFrom(b, rc.storeDest, rc.storeOffset, types.NewPtr(at))
   375  		if x.debug > 1 {
   376  			x.Printf("...recur store %s at %s\n", a.LongString(), dst.LongString())
   377  		}
   378  		if a.Op == OpLoad {
   379  			m0 = b.NewValue3A(pos, OpMove, types.TypeMem, at, dst, a.Args[0], m0)
   380  			m0.AuxInt = at.Size()
   381  			return m0
   382  		} else {
   383  			panic(fmt.Errorf("Store of not a load"))
   384  		}
   385  	}
   386  
   387  	mem := m0
   388  	switch at.Kind() {
   389  	case types.TARRAY:
   390  		et := at.Elem()
   391  		for i := int64(0); i < at.NumElem(); i++ {
   392  			e := b.NewValue1I(pos, OpArraySelect, et, i, a)
   393  			pos = pos.WithNotStmt()
   394  			mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(et))
   395  		}
   396  		return mem
   397  
   398  	case types.TSTRUCT:
   399  		for i := 0; i < at.NumFields(); i++ {
   400  			et := at.Field(i).Type // might need to read offsets from the fields
   401  			e := b.NewValue1I(pos, OpStructSelect, et, int64(i), a)
   402  			pos = pos.WithNotStmt()
   403  			if x.debug > 1 {
   404  				x.Printf("...recur decompose %s, %v\n", e.LongString(), et)
   405  			}
   406  			mem = x.decomposeAsNecessary(pos, b, e, mem, rc.next(et))
   407  		}
   408  		return mem
   409  
   410  	case types.TSLICE:
   411  		mem = x.decomposeOne(pos, b, a, mem, at.Elem().PtrTo(), OpSlicePtr, &rc)
   412  		pos = pos.WithNotStmt()
   413  		mem = x.decomposeOne(pos, b, a, mem, x.typs.Int, OpSliceLen, &rc)
   414  		return x.decomposeOne(pos, b, a, mem, x.typs.Int, OpSliceCap, &rc)
   415  
   416  	case types.TSTRING:
   417  		return x.decomposePair(pos, b, a, mem, x.typs.BytePtr, x.typs.Int, OpStringPtr, OpStringLen, &rc)
   418  
   419  	case types.TINTER:
   420  		mem = x.decomposeOne(pos, b, a, mem, x.typs.Uintptr, OpITab, &rc)
   421  		pos = pos.WithNotStmt()
   422  		// Immediate interfaces cause so many headaches.
   423  		if a.Op == OpIMake {
   424  			data := a.Args[1]
   425  			for data.Op == OpStructMake || data.Op == OpArrayMake1 {
   426  				data = data.Args[0]
   427  			}
   428  			return x.decomposeAsNecessary(pos, b, data, mem, rc.next(data.Type))
   429  		}
   430  		return x.decomposeOne(pos, b, a, mem, x.typs.BytePtr, OpIData, &rc)
   431  
   432  	case types.TCOMPLEX64:
   433  		return x.decomposePair(pos, b, a, mem, x.typs.Float32, x.typs.Float32, OpComplexReal, OpComplexImag, &rc)
   434  
   435  	case types.TCOMPLEX128:
   436  		return x.decomposePair(pos, b, a, mem, x.typs.Float64, x.typs.Float64, OpComplexReal, OpComplexImag, &rc)
   437  
   438  	case types.TINT64:
   439  		if at.Size() > x.regSize {
   440  			return x.decomposePair(pos, b, a, mem, x.firstType, x.secondType, x.firstOp, x.secondOp, &rc)
   441  		}
   442  	case types.TUINT64:
   443  		if at.Size() > x.regSize {
   444  			return x.decomposePair(pos, b, a, mem, x.typs.UInt32, x.typs.UInt32, x.firstOp, x.secondOp, &rc)
   445  		}
   446  	}
   447  
   448  	// An atomic type, either record the register or store it and update the memory.
   449  
   450  	if rc.hasRegs() {
   451  		if x.debug > 1 {
   452  			x.Printf("...recur addArg %s\n", a.LongString())
   453  		}
   454  		rc.addArg(a)
   455  	} else {
   456  		dst := x.offsetFrom(b, rc.storeDest, rc.storeOffset, types.NewPtr(at))
   457  		if x.debug > 1 {
   458  			x.Printf("...recur store %s at %s\n", a.LongString(), dst.LongString())
   459  		}
   460  		mem = b.NewValue3A(pos, OpStore, types.TypeMem, at, dst, a, mem)
   461  	}
   462  
   463  	return mem
   464  }
   465  
   466  // Convert scalar OpArg into the proper OpWhateverArg instruction
   467  // Convert scalar OpSelectN into perhaps-differently-indexed OpSelectN
   468  // Convert aggregate OpArg into Make of its parts (which are eventually scalars)
   469  // Convert aggregate OpSelectN into Make of its parts (which are eventually scalars)
   470  // Returns the converted value.
   471  //
   472  //   - "pos" the position for any generated instructions
   473  //   - "b" the block for any generated instructions
   474  //   - "container" the outermost OpArg/OpSelectN
   475  //   - "a" the instruction to overwrite, if any (only the outermost caller)
   476  //   - "m0" the memory arg for any loads that are necessary
   477  //   - "at" the type of the Arg/part
   478  //   - "rc" the register/memory cursor locating the various parts of the Arg.
   479  func (x *expandState) rewriteSelectOrArg(pos src.XPos, b *Block, container, a, m0 *Value, at *types.Type, rc registerCursor) *Value {
   480  
   481  	if at == types.TypeMem {
   482  		a.copyOf(m0)
   483  		return a
   484  	}
   485  
   486  	makeOf := func(a *Value, op Op, args []*Value) *Value {
   487  		if a == nil {
   488  			a = b.NewValue0(pos, op, at)
   489  			a.AddArgs(args...)
   490  		} else {
   491  			a.resetArgs()
   492  			a.Aux, a.AuxInt = nil, 0
   493  			a.Pos, a.Op, a.Type = pos, op, at
   494  			a.AddArgs(args...)
   495  		}
   496  		return a
   497  	}
   498  
   499  	if at.Size() == 0 {
   500  		// For consistency, create these values even though they'll ultimately be unused
   501  		if at.IsArray() {
   502  			return makeOf(a, OpArrayMake0, nil)
   503  		}
   504  		if at.IsStruct() {
   505  			return makeOf(a, OpStructMake, nil)
   506  		}
   507  		return a
   508  	}
   509  
   510  	sk := selKey{from: container, size: 0, offsetOrIndex: rc.storeOffset, typ: at}
   511  	dupe := x.commonSelectors[sk]
   512  	if dupe != nil {
   513  		if a == nil {
   514  			return dupe
   515  		}
   516  		a.copyOf(dupe)
   517  		return a
   518  	}
   519  
   520  	var argStore [10]*Value
   521  	args := argStore[:0]
   522  
   523  	addArg := func(a0 *Value) {
   524  		if a0 == nil {
   525  			as := "<nil>"
   526  			if a != nil {
   527  				as = a.LongString()
   528  			}
   529  			panic(fmt.Errorf("a0 should not be nil, a=%v, container=%v, at=%v", as, container.LongString(), at))
   530  		}
   531  		args = append(args, a0)
   532  	}
   533  
   534  	switch at.Kind() {
   535  	case types.TARRAY:
   536  		et := at.Elem()
   537  		for i := int64(0); i < at.NumElem(); i++ {
   538  			e := x.rewriteSelectOrArg(pos, b, container, nil, m0, et, rc.next(et))
   539  			addArg(e)
   540  		}
   541  		a = makeOf(a, OpArrayMake1, args)
   542  		x.commonSelectors[sk] = a
   543  		return a
   544  
   545  	case types.TSTRUCT:
   546  		// Assume ssagen/ssa.go (in buildssa) spills large aggregates so they won't appear here.
   547  		for i := 0; i < at.NumFields(); i++ {
   548  			et := at.Field(i).Type
   549  			e := x.rewriteSelectOrArg(pos, b, container, nil, m0, et, rc.next(et))
   550  			if e == nil {
   551  				panic(fmt.Errorf("nil e, et=%v, et.Size()=%d, i=%d", et, et.Size(), i))
   552  			}
   553  			addArg(e)
   554  			pos = pos.WithNotStmt()
   555  		}
   556  		if at.NumFields() > 4 {
   557  			panic(fmt.Errorf("Too many fields (%d, %d bytes), container=%s", at.NumFields(), at.Size(), container.LongString()))
   558  		}
   559  		a = makeOf(a, OpStructMake, args)
   560  		x.commonSelectors[sk] = a
   561  		return a
   562  
   563  	case types.TSLICE:
   564  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, at.Elem().PtrTo(), rc.next(x.typs.BytePtr)))
   565  		pos = pos.WithNotStmt()
   566  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Int, rc.next(x.typs.Int)))
   567  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Int, rc.next(x.typs.Int)))
   568  		a = makeOf(a, OpSliceMake, args)
   569  		x.commonSelectors[sk] = a
   570  		return a
   571  
   572  	case types.TSTRING:
   573  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.BytePtr, rc.next(x.typs.BytePtr)))
   574  		pos = pos.WithNotStmt()
   575  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Int, rc.next(x.typs.Int)))
   576  		a = makeOf(a, OpStringMake, args)
   577  		x.commonSelectors[sk] = a
   578  		return a
   579  
   580  	case types.TINTER:
   581  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Uintptr, rc.next(x.typs.Uintptr)))
   582  		pos = pos.WithNotStmt()
   583  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.BytePtr, rc.next(x.typs.BytePtr)))
   584  		a = makeOf(a, OpIMake, args)
   585  		x.commonSelectors[sk] = a
   586  		return a
   587  
   588  	case types.TCOMPLEX64:
   589  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Float32, rc.next(x.typs.Float32)))
   590  		pos = pos.WithNotStmt()
   591  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Float32, rc.next(x.typs.Float32)))
   592  		a = makeOf(a, OpComplexMake, args)
   593  		x.commonSelectors[sk] = a
   594  		return a
   595  
   596  	case types.TCOMPLEX128:
   597  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Float64, rc.next(x.typs.Float64)))
   598  		pos = pos.WithNotStmt()
   599  		addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.Float64, rc.next(x.typs.Float64)))
   600  		a = makeOf(a, OpComplexMake, args)
   601  		x.commonSelectors[sk] = a
   602  		return a
   603  
   604  	case types.TINT64:
   605  		if at.Size() > x.regSize {
   606  			addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.firstType, rc.next(x.firstType)))
   607  			pos = pos.WithNotStmt()
   608  			addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.secondType, rc.next(x.secondType)))
   609  			if !x.f.Config.BigEndian {
   610  				// Int64Make args are big, little
   611  				args[0], args[1] = args[1], args[0]
   612  			}
   613  			a = makeOf(a, OpInt64Make, args)
   614  			x.commonSelectors[sk] = a
   615  			return a
   616  		}
   617  	case types.TUINT64:
   618  		if at.Size() > x.regSize {
   619  			addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.UInt32, rc.next(x.typs.UInt32)))
   620  			pos = pos.WithNotStmt()
   621  			addArg(x.rewriteSelectOrArg(pos, b, container, nil, m0, x.typs.UInt32, rc.next(x.typs.UInt32)))
   622  			if !x.f.Config.BigEndian {
   623  				// Int64Make args are big, little
   624  				args[0], args[1] = args[1], args[0]
   625  			}
   626  			a = makeOf(a, OpInt64Make, args)
   627  			x.commonSelectors[sk] = a
   628  			return a
   629  		}
   630  	}
   631  
   632  	// An atomic type, either record the register or store it and update the memory.
   633  
   634  	// Depending on the container Op, the leaves are either OpSelectN or OpArg{Int,Float}Reg
   635  
   636  	if container.Op == OpArg {
   637  		if rc.hasRegs() {
   638  			op, i := rc.ArgOpAndRegisterFor()
   639  			name := container.Aux.(*ir.Name)
   640  			a = makeOf(a, op, nil)
   641  			a.AuxInt = i
   642  			a.Aux = &AuxNameOffset{name, rc.storeOffset}
   643  		} else {
   644  			key := selKey{container, rc.storeOffset, at.Size(), at}
   645  			w := x.commonArgs[key]
   646  			if w != nil && w.Uses != 0 {
   647  				if a == nil {
   648  					a = w
   649  				} else {
   650  					a.copyOf(w)
   651  				}
   652  			} else {
   653  				if a == nil {
   654  					aux := container.Aux
   655  					auxInt := container.AuxInt + rc.storeOffset
   656  					a = container.Block.NewValue0IA(container.Pos, OpArg, at, auxInt, aux)
   657  				} else {
   658  					// do nothing, the original should be okay.
   659  				}
   660  				x.commonArgs[key] = a
   661  			}
   662  		}
   663  	} else if container.Op == OpSelectN {
   664  		call := container.Args[0]
   665  		aux := call.Aux.(*AuxCall)
   666  		which := container.AuxInt
   667  
   668  		if at == types.TypeMem {
   669  			if a != m0 || a != x.memForCall[call.ID] {
   670  				panic(fmt.Errorf("Memories %s, %s, and %s should all be equal after %s", a.LongString(), m0.LongString(), x.memForCall[call.ID], call.LongString()))
   671  			}
   672  		} else if rc.hasRegs() {
   673  			firstReg := uint32(0)
   674  			for i := 0; i < int(which); i++ {
   675  				firstReg += uint32(len(aux.abiInfo.OutParam(i).Registers))
   676  			}
   677  			reg := int64(rc.nextSlice + Abi1RO(firstReg))
   678  			a = makeOf(a, OpSelectN, []*Value{call})
   679  			a.AuxInt = reg
   680  		} else {
   681  			off := x.offsetFrom(x.f.Entry, x.sp, rc.storeOffset+aux.OffsetOfResult(which), types.NewPtr(at))
   682  			a = makeOf(a, OpLoad, []*Value{off, m0})
   683  		}
   684  
   685  	} else {
   686  		panic(fmt.Errorf("Expected container OpArg or OpSelectN, saw %v instead", container.LongString()))
   687  	}
   688  
   689  	x.commonSelectors[sk] = a
   690  	return a
   691  }
   692  
   693  // rewriteWideSelectToStores handles the case of a SelectN'd result from a function call that is too large for SSA,
   694  // but is transferred in registers.  In this case the register cursor tracks both operands; the register sources and
   695  // the memory destinations.
   696  // This returns the memory flowing out of the last store
   697  func (x *expandState) rewriteWideSelectToStores(pos src.XPos, b *Block, container, m0 *Value, at *types.Type, rc registerCursor) *Value {
   698  
   699  	if at.Size() == 0 {
   700  		return m0
   701  	}
   702  
   703  	switch at.Kind() {
   704  	case types.TARRAY:
   705  		et := at.Elem()
   706  		for i := int64(0); i < at.NumElem(); i++ {
   707  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, et, rc.next(et))
   708  		}
   709  		return m0
   710  
   711  	case types.TSTRUCT:
   712  		// Assume ssagen/ssa.go (in buildssa) spills large aggregates so they won't appear here.
   713  		for i := 0; i < at.NumFields(); i++ {
   714  			et := at.Field(i).Type
   715  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, et, rc.next(et))
   716  			pos = pos.WithNotStmt()
   717  		}
   718  		return m0
   719  
   720  	case types.TSLICE:
   721  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, at.Elem().PtrTo(), rc.next(x.typs.BytePtr))
   722  		pos = pos.WithNotStmt()
   723  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Int, rc.next(x.typs.Int))
   724  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Int, rc.next(x.typs.Int))
   725  		return m0
   726  
   727  	case types.TSTRING:
   728  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.BytePtr, rc.next(x.typs.BytePtr))
   729  		pos = pos.WithNotStmt()
   730  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Int, rc.next(x.typs.Int))
   731  		return m0
   732  
   733  	case types.TINTER:
   734  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Uintptr, rc.next(x.typs.Uintptr))
   735  		pos = pos.WithNotStmt()
   736  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.BytePtr, rc.next(x.typs.BytePtr))
   737  		return m0
   738  
   739  	case types.TCOMPLEX64:
   740  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Float32, rc.next(x.typs.Float32))
   741  		pos = pos.WithNotStmt()
   742  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Float32, rc.next(x.typs.Float32))
   743  		return m0
   744  
   745  	case types.TCOMPLEX128:
   746  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Float64, rc.next(x.typs.Float64))
   747  		pos = pos.WithNotStmt()
   748  		m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.Float64, rc.next(x.typs.Float64))
   749  		return m0
   750  
   751  	case types.TINT64:
   752  		if at.Size() > x.regSize {
   753  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.firstType, rc.next(x.firstType))
   754  			pos = pos.WithNotStmt()
   755  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.secondType, rc.next(x.secondType))
   756  			return m0
   757  		}
   758  	case types.TUINT64:
   759  		if at.Size() > x.regSize {
   760  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.UInt32, rc.next(x.typs.UInt32))
   761  			pos = pos.WithNotStmt()
   762  			m0 = x.rewriteWideSelectToStores(pos, b, container, m0, x.typs.UInt32, rc.next(x.typs.UInt32))
   763  			return m0
   764  		}
   765  	}
   766  
   767  	// TODO could change treatment of too-large OpArg, would deal with it here.
   768  	if container.Op == OpSelectN {
   769  		call := container.Args[0]
   770  		aux := call.Aux.(*AuxCall)
   771  		which := container.AuxInt
   772  
   773  		if rc.hasRegs() {
   774  			firstReg := uint32(0)
   775  			for i := 0; i < int(which); i++ {
   776  				firstReg += uint32(len(aux.abiInfo.OutParam(i).Registers))
   777  			}
   778  			reg := int64(rc.nextSlice + Abi1RO(firstReg))
   779  			a := b.NewValue1I(pos, OpSelectN, at, reg, call)
   780  			dst := x.offsetFrom(b, rc.storeDest, rc.storeOffset, types.NewPtr(at))
   781  			m0 = b.NewValue3A(pos, OpStore, types.TypeMem, at, dst, a, m0)
   782  		} else {
   783  			panic(fmt.Errorf("Expected rc to have registers"))
   784  		}
   785  	} else {
   786  		panic(fmt.Errorf("Expected container OpSelectN, saw %v instead", container.LongString()))
   787  	}
   788  	return m0
   789  }
   790  
   791  func isBlockMultiValueExit(b *Block) bool {
   792  	return (b.Kind == BlockRet || b.Kind == BlockRetJmp) && b.Controls[0] != nil && b.Controls[0].Op == OpMakeResult
   793  }
   794  
   795  type Abi1RO uint8 // An offset within a parameter's slice of register indices, for abi1.
   796  
   797  // A registerCursor tracks which register is used for an Arg or regValues, or a piece of such.
   798  type registerCursor struct {
   799  	storeDest   *Value // if there are no register targets, then this is the base of the store.
   800  	storeOffset int64
   801  	regs        []abi.RegIndex // the registers available for this Arg/result (which is all in registers or not at all)
   802  	nextSlice   Abi1RO         // the next register/register-slice offset
   803  	config      *abi.ABIConfig
   804  	regValues   *[]*Value // values assigned to registers accumulate here
   805  }
   806  
   807  func (c *registerCursor) String() string {
   808  	dest := "<none>"
   809  	if c.storeDest != nil {
   810  		dest = fmt.Sprintf("%s+%d", c.storeDest.String(), c.storeOffset)
   811  	}
   812  	regs := "<none>"
   813  	if c.regValues != nil {
   814  		regs = ""
   815  		for i, x := range *c.regValues {
   816  			if i > 0 {
   817  				regs = regs + "; "
   818  			}
   819  			regs = regs + x.LongString()
   820  		}
   821  	}
   822  
   823  	// not printing the config because that has not been useful
   824  	return fmt.Sprintf("RCSR{storeDest=%v, regsLen=%d, nextSlice=%d, regValues=[%s]}", dest, len(c.regs), c.nextSlice, regs)
   825  }
   826  
   827  // next effectively post-increments the register cursor; the receiver is advanced,
   828  // the (aligned) old value is returned.
   829  func (c *registerCursor) next(t *types.Type) registerCursor {
   830  	c.storeOffset = types.RoundUp(c.storeOffset, t.Alignment())
   831  	rc := *c
   832  	c.storeOffset = types.RoundUp(c.storeOffset+t.Size(), t.Alignment())
   833  	if int(c.nextSlice) < len(c.regs) {
   834  		w := c.config.NumParamRegs(t)
   835  		c.nextSlice += Abi1RO(w)
   836  	}
   837  	return rc
   838  }
   839  
   840  // plus returns a register cursor offset from the original, without modifying the original.
   841  func (c *registerCursor) plus(regWidth Abi1RO) registerCursor {
   842  	rc := *c
   843  	rc.nextSlice += regWidth
   844  	return rc
   845  }
   846  
   847  func (c *registerCursor) init(regs []abi.RegIndex, info *abi.ABIParamResultInfo, result *[]*Value, storeDest *Value, storeOffset int64) {
   848  	c.regs = regs
   849  	c.nextSlice = 0
   850  	c.storeOffset = storeOffset
   851  	c.storeDest = storeDest
   852  	c.config = info.Config()
   853  	c.regValues = result
   854  }
   855  
   856  func (c *registerCursor) addArg(v *Value) {
   857  	*c.regValues = append(*c.regValues, v)
   858  }
   859  
   860  func (c *registerCursor) hasRegs() bool {
   861  	return len(c.regs) > 0
   862  }
   863  
   864  func (c *registerCursor) ArgOpAndRegisterFor() (Op, int64) {
   865  	r := c.regs[c.nextSlice]
   866  	return ArgOpAndRegisterFor(r, c.config)
   867  }
   868  
   869  // ArgOpAndRegisterFor converts an abi register index into an ssa Op and corresponding
   870  // arg register index.
   871  func ArgOpAndRegisterFor(r abi.RegIndex, abiConfig *abi.ABIConfig) (Op, int64) {
   872  	i := abiConfig.FloatIndexFor(r)
   873  	if i >= 0 { // float PR
   874  		return OpArgFloatReg, i
   875  	}
   876  	return OpArgIntReg, int64(r)
   877  }
   878  
   879  type selKey struct {
   880  	from          *Value // what is selected from
   881  	offsetOrIndex int64  // whatever is appropriate for the selector
   882  	size          int64
   883  	typ           *types.Type
   884  }
   885  
   886  type expandState struct {
   887  	f       *Func
   888  	debug   int // odd values log lost statement markers, so likely settings are 1 (stmts), 2 (expansion), and 3 (both)
   889  	regSize int64
   890  	sp      *Value
   891  	typs    *Types
   892  
   893  	firstOp    Op          // for 64-bit integers on 32-bit machines, first word in memory
   894  	secondOp   Op          // for 64-bit integers on 32-bit machines, second word in memory
   895  	firstType  *types.Type // first half type, for Int64
   896  	secondType *types.Type // second half type, for Int64
   897  
   898  	wideSelects     map[*Value]*Value // Selects that are not SSA-able, mapped to consuming stores.
   899  	commonSelectors map[selKey]*Value // used to de-dupe selectors
   900  	commonArgs      map[selKey]*Value // used to de-dupe OpArg/OpArgIntReg/OpArgFloatReg
   901  	memForCall      map[ID]*Value     // For a call, need to know the unique selector that gets the mem.
   902  	indentLevel     int               // Indentation for debugging recursion
   903  }
   904  
   905  // offsetFrom creates an offset from a pointer, simplifying chained offsets and offsets from SP
   906  func (x *expandState) offsetFrom(b *Block, from *Value, offset int64, pt *types.Type) *Value {
   907  	ft := from.Type
   908  	if offset == 0 {
   909  		if ft == pt {
   910  			return from
   911  		}
   912  		// This captures common, (apparently) safe cases.  The unsafe cases involve ft == uintptr
   913  		if (ft.IsPtr() || ft.IsUnsafePtr()) && pt.IsPtr() {
   914  			return from
   915  		}
   916  	}
   917  	// Simplify, canonicalize
   918  	for from.Op == OpOffPtr {
   919  		offset += from.AuxInt
   920  		from = from.Args[0]
   921  	}
   922  	if from == x.sp {
   923  		return x.f.ConstOffPtrSP(pt, offset, x.sp)
   924  	}
   925  	return b.NewValue1I(from.Pos.WithNotStmt(), OpOffPtr, pt, offset, from)
   926  }
   927  
   928  // prAssignForArg returns the ABIParamAssignment for v, assumed to be an OpArg.
   929  func (x *expandState) prAssignForArg(v *Value) *abi.ABIParamAssignment {
   930  	if v.Op != OpArg {
   931  		panic(fmt.Errorf("Wanted OpArg, instead saw %s", v.LongString()))
   932  	}
   933  	return ParamAssignmentForArgName(x.f, v.Aux.(*ir.Name))
   934  }
   935  
   936  // ParamAssignmentForArgName returns the ABIParamAssignment for f's arg with matching name.
   937  func ParamAssignmentForArgName(f *Func, name *ir.Name) *abi.ABIParamAssignment {
   938  	abiInfo := f.OwnAux.abiInfo
   939  	ip := abiInfo.InParams()
   940  	for i, a := range ip {
   941  		if a.Name == name {
   942  			return &ip[i]
   943  		}
   944  	}
   945  	panic(fmt.Errorf("Did not match param %v in prInfo %+v", name, abiInfo.InParams()))
   946  }
   947  
   948  // indent increments (or decrements) the indentation.
   949  func (x *expandState) indent(n int) {
   950  	x.indentLevel += n
   951  }
   952  
   953  // Printf does an indented fmt.Printf on the format and args.
   954  func (x *expandState) Printf(format string, a ...interface{}) (n int, err error) {
   955  	if x.indentLevel > 0 {
   956  		fmt.Printf("%[1]*s", x.indentLevel, "")
   957  	}
   958  	return fmt.Printf(format, a...)
   959  }
   960  
   961  func (x *expandState) invalidateRecursively(a *Value) {
   962  	var s string
   963  	if x.debug > 0 {
   964  		plus := " "
   965  		if a.Pos.IsStmt() == src.PosIsStmt {
   966  			plus = " +"
   967  		}
   968  		s = a.String() + plus + a.Pos.LineNumber() + " " + a.LongString()
   969  		if x.debug > 1 {
   970  			x.Printf("...marking %v unused\n", s)
   971  		}
   972  	}
   973  	lost := a.invalidateRecursively()
   974  	if x.debug&1 != 0 && lost { // For odd values of x.debug, do this.
   975  		x.Printf("Lost statement marker in %s on former %s\n", base.Ctxt.Pkgpath+"."+x.f.Name, s)
   976  	}
   977  }
   978  

View as plain text