Source file src/cmd/compile/internal/inline/inl.go

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  //
     5  // The inlining facility makes 2 passes: first CanInline determines which
     6  // functions are suitable for inlining, and for those that are it
     7  // saves a copy of the body. Then InlineCalls walks each function body to
     8  // expand calls to inlinable functions.
     9  //
    10  // The Debug.l flag controls the aggressiveness. Note that main() swaps level 0 and 1,
    11  // making 1 the default and -l disable. Additional levels (beyond -l) may be buggy and
    12  // are not supported.
    13  //      0: disabled
    14  //      1: 80-nodes leaf functions, oneliners, panic, lazy typechecking (default)
    15  //      2: (unassigned)
    16  //      3: (unassigned)
    17  //      4: allow non-leaf functions
    18  //
    19  // At some point this may get another default and become switch-offable with -N.
    20  //
    21  // The -d typcheckinl flag enables early typechecking of all imported bodies,
    22  // which is useful to flush out bugs.
    23  //
    24  // The Debug.m flag enables diagnostic output.  a single -m is useful for verifying
    25  // which calls get inlined or not, more is for debugging, and may go away at any point.
    26  
    27  package inline
    28  
    29  import (
    30  	"fmt"
    31  	"go/constant"
    32  	"internal/buildcfg"
    33  	"strconv"
    34  
    35  	"cmd/compile/internal/base"
    36  	"cmd/compile/internal/inline/inlheur"
    37  	"cmd/compile/internal/ir"
    38  	"cmd/compile/internal/logopt"
    39  	"cmd/compile/internal/pgoir"
    40  	"cmd/compile/internal/typecheck"
    41  	"cmd/compile/internal/types"
    42  	"cmd/internal/obj"
    43  	"cmd/internal/pgo"
    44  )
    45  
    46  // Inlining budget parameters, gathered in one place
    47  const (
    48  	inlineMaxBudget       = 80
    49  	inlineExtraAppendCost = 0
    50  	// default is to inline if there's at most one call. -l=4 overrides this by using 1 instead.
    51  	inlineExtraCallCost  = 57              // 57 was benchmarked to provided most benefit with no bad surprises; see https://github.com/golang/go/issues/19348#issuecomment-439370742
    52  	inlineParamCallCost  = 17              // calling a parameter only costs this much extra (inlining might expose a constant function)
    53  	inlineExtraPanicCost = 1               // do not penalize inlining panics.
    54  	inlineExtraThrowCost = inlineMaxBudget // with current (2018-05/1.11) code, inlining runtime.throw does not help.
    55  
    56  	inlineBigFunctionNodes   = 5000 // Functions with this many nodes are considered "big".
    57  	inlineBigFunctionMaxCost = 20   // Max cost of inlinee when inlining into a "big" function.
    58  )
    59  
    60  var (
    61  	// List of all hot callee nodes.
    62  	// TODO(prattmic): Make this non-global.
    63  	candHotCalleeMap = make(map[*pgoir.IRNode]struct{})
    64  
    65  	// Set of functions that contain hot call sites.
    66  	hasHotCall = make(map[*ir.Func]struct{})
    67  
    68  	// List of all hot call sites. CallSiteInfo.Callee is always nil.
    69  	// TODO(prattmic): Make this non-global.
    70  	candHotEdgeMap = make(map[pgoir.CallSiteInfo]struct{})
    71  
    72  	// Threshold in percentage for hot callsite inlining.
    73  	inlineHotCallSiteThresholdPercent float64
    74  
    75  	// Threshold in CDF percentage for hot callsite inlining,
    76  	// that is, for a threshold of X the hottest callsites that
    77  	// make up the top X% of total edge weight will be
    78  	// considered hot for inlining candidates.
    79  	inlineCDFHotCallSiteThresholdPercent = float64(99)
    80  
    81  	// Budget increased due to hotness.
    82  	inlineHotMaxBudget int32 = 2000
    83  )
    84  
    85  func IsPgoHotFunc(fn *ir.Func, profile *pgoir.Profile) bool {
    86  	if profile == nil {
    87  		return false
    88  	}
    89  	if n, ok := profile.WeightedCG.IRNodes[ir.LinkFuncName(fn)]; ok {
    90  		_, ok := candHotCalleeMap[n]
    91  		return ok
    92  	}
    93  	return false
    94  }
    95  
    96  func HasPgoHotInline(fn *ir.Func) bool {
    97  	_, has := hasHotCall[fn]
    98  	return has
    99  }
   100  
   101  // PGOInlinePrologue records the hot callsites from ir-graph.
   102  func PGOInlinePrologue(p *pgoir.Profile) {
   103  	if base.Debug.PGOInlineCDFThreshold != "" {
   104  		if s, err := strconv.ParseFloat(base.Debug.PGOInlineCDFThreshold, 64); err == nil && s >= 0 && s <= 100 {
   105  			inlineCDFHotCallSiteThresholdPercent = s
   106  		} else {
   107  			base.Fatalf("invalid PGOInlineCDFThreshold, must be between 0 and 100")
   108  		}
   109  	}
   110  	var hotCallsites []pgo.NamedCallEdge
   111  	inlineHotCallSiteThresholdPercent, hotCallsites = hotNodesFromCDF(p)
   112  	if base.Debug.PGODebug > 0 {
   113  		fmt.Printf("hot-callsite-thres-from-CDF=%v\n", inlineHotCallSiteThresholdPercent)
   114  	}
   115  
   116  	if x := base.Debug.PGOInlineBudget; x != 0 {
   117  		inlineHotMaxBudget = int32(x)
   118  	}
   119  
   120  	for _, n := range hotCallsites {
   121  		// mark inlineable callees from hot edges
   122  		if callee := p.WeightedCG.IRNodes[n.CalleeName]; callee != nil {
   123  			candHotCalleeMap[callee] = struct{}{}
   124  		}
   125  		// mark hot call sites
   126  		if caller := p.WeightedCG.IRNodes[n.CallerName]; caller != nil && caller.AST != nil {
   127  			csi := pgoir.CallSiteInfo{LineOffset: n.CallSiteOffset, Caller: caller.AST}
   128  			candHotEdgeMap[csi] = struct{}{}
   129  		}
   130  	}
   131  
   132  	if base.Debug.PGODebug >= 3 {
   133  		fmt.Printf("hot-cg before inline in dot format:")
   134  		p.PrintWeightedCallGraphDOT(inlineHotCallSiteThresholdPercent)
   135  	}
   136  }
   137  
   138  // hotNodesFromCDF computes an edge weight threshold and the list of hot
   139  // nodes that make up the given percentage of the CDF. The threshold, as
   140  // a percent, is the lower bound of weight for nodes to be considered hot
   141  // (currently only used in debug prints) (in case of equal weights,
   142  // comparing with the threshold may not accurately reflect which nodes are
   143  // considered hot).
   144  func hotNodesFromCDF(p *pgoir.Profile) (float64, []pgo.NamedCallEdge) {
   145  	cum := int64(0)
   146  	for i, n := range p.NamedEdgeMap.ByWeight {
   147  		w := p.NamedEdgeMap.Weight[n]
   148  		cum += w
   149  		if pgo.WeightInPercentage(cum, p.TotalWeight) > inlineCDFHotCallSiteThresholdPercent {
   150  			// nodes[:i+1] to include the very last node that makes it to go over the threshold.
   151  			// (Say, if the CDF threshold is 50% and one hot node takes 60% of weight, we want to
   152  			// include that node instead of excluding it.)
   153  			return pgo.WeightInPercentage(w, p.TotalWeight), p.NamedEdgeMap.ByWeight[:i+1]
   154  		}
   155  	}
   156  	return 0, p.NamedEdgeMap.ByWeight
   157  }
   158  
   159  // CanInlineFuncs computes whether a batch of functions are inlinable.
   160  func CanInlineFuncs(funcs []*ir.Func, profile *pgoir.Profile) {
   161  	if profile != nil {
   162  		PGOInlinePrologue(profile)
   163  	}
   164  
   165  	if base.Flag.LowerL == 0 {
   166  		return
   167  	}
   168  
   169  	ir.VisitFuncsBottomUp(funcs, func(funcs []*ir.Func, recursive bool) {
   170  		numfns := numNonClosures(funcs)
   171  
   172  		for _, fn := range funcs {
   173  			if !recursive || numfns > 1 {
   174  				// We allow inlining if there is no
   175  				// recursion, or the recursion cycle is
   176  				// across more than one function.
   177  				CanInline(fn, profile)
   178  			} else {
   179  				if base.Flag.LowerM > 1 && fn.OClosure == nil {
   180  					fmt.Printf("%v: cannot inline %v: recursive\n", ir.Line(fn), fn.Nname)
   181  				}
   182  			}
   183  			if inlheur.Enabled() {
   184  				analyzeFuncProps(fn, profile)
   185  			}
   186  		}
   187  	})
   188  }
   189  
   190  // inlineBudget determines the max budget for function 'fn' prior to
   191  // analyzing the hairiness of the body of 'fn'. We pass in the pgo
   192  // profile if available (which can change the budget), also a
   193  // 'relaxed' flag, which expands the budget slightly to allow for the
   194  // possibility that a call to the function might have its score
   195  // adjusted downwards. If 'verbose' is set, then print a remark where
   196  // we boost the budget due to PGO.
   197  func inlineBudget(fn *ir.Func, profile *pgoir.Profile, relaxed bool, verbose bool) int32 {
   198  	// Update the budget for profile-guided inlining.
   199  	budget := int32(inlineMaxBudget)
   200  	if IsPgoHotFunc(fn, profile) {
   201  		budget = inlineHotMaxBudget
   202  		if verbose {
   203  			fmt.Printf("hot-node enabled increased budget=%v for func=%v\n", budget, ir.PkgFuncName(fn))
   204  		}
   205  	}
   206  	if relaxed {
   207  		budget += inlheur.BudgetExpansion(inlineMaxBudget)
   208  	}
   209  	return budget
   210  }
   211  
   212  // CanInline determines whether fn is inlineable.
   213  // If so, CanInline saves copies of fn.Body and fn.Dcl in fn.Inl.
   214  // fn and fn.Body will already have been typechecked.
   215  func CanInline(fn *ir.Func, profile *pgoir.Profile) {
   216  	if fn.Nname == nil {
   217  		base.Fatalf("CanInline no nname %+v", fn)
   218  	}
   219  
   220  	var reason string // reason, if any, that the function was not inlined
   221  	if base.Flag.LowerM > 1 || logopt.Enabled() {
   222  		defer func() {
   223  			if reason != "" {
   224  				if base.Flag.LowerM > 1 {
   225  					fmt.Printf("%v: cannot inline %v: %s\n", ir.Line(fn), fn.Nname, reason)
   226  				}
   227  				if logopt.Enabled() {
   228  					logopt.LogOpt(fn.Pos(), "cannotInlineFunction", "inline", ir.FuncName(fn), reason)
   229  				}
   230  			}
   231  		}()
   232  	}
   233  
   234  	reason = InlineImpossible(fn)
   235  	if reason != "" {
   236  		return
   237  	}
   238  	if fn.Typecheck() == 0 {
   239  		base.Fatalf("CanInline on non-typechecked function %v", fn)
   240  	}
   241  
   242  	n := fn.Nname
   243  	if n.Func.InlinabilityChecked() {
   244  		return
   245  	}
   246  	defer n.Func.SetInlinabilityChecked(true)
   247  
   248  	cc := int32(inlineExtraCallCost)
   249  	if base.Flag.LowerL == 4 {
   250  		cc = 1 // this appears to yield better performance than 0.
   251  	}
   252  
   253  	// Used a "relaxed" inline budget if the new inliner is enabled.
   254  	relaxed := inlheur.Enabled()
   255  
   256  	// Compute the inline budget for this func.
   257  	budget := inlineBudget(fn, profile, relaxed, base.Debug.PGODebug > 0)
   258  
   259  	// At this point in the game the function we're looking at may
   260  	// have "stale" autos, vars that still appear in the Dcl list, but
   261  	// which no longer have any uses in the function body (due to
   262  	// elimination by deadcode). We'd like to exclude these dead vars
   263  	// when creating the "Inline.Dcl" field below; to accomplish this,
   264  	// the hairyVisitor below builds up a map of used/referenced
   265  	// locals, and we use this map to produce a pruned Inline.Dcl
   266  	// list. See issue 25459 for more context.
   267  
   268  	visitor := hairyVisitor{
   269  		curFunc:       fn,
   270  		isBigFunc:     IsBigFunc(fn),
   271  		budget:        budget,
   272  		maxBudget:     budget,
   273  		extraCallCost: cc,
   274  		profile:       profile,
   275  	}
   276  	if visitor.tooHairy(fn) {
   277  		reason = visitor.reason
   278  		return
   279  	}
   280  
   281  	n.Func.Inl = &ir.Inline{
   282  		Cost:            budget - visitor.budget,
   283  		Dcl:             pruneUnusedAutos(n.Func.Dcl, &visitor),
   284  		HaveDcl:         true,
   285  		CanDelayResults: canDelayResults(fn),
   286  	}
   287  	if base.Flag.LowerM != 0 || logopt.Enabled() {
   288  		noteInlinableFunc(n, fn, budget-visitor.budget)
   289  	}
   290  }
   291  
   292  // noteInlinableFunc issues a message to the user that the specified
   293  // function is inlinable.
   294  func noteInlinableFunc(n *ir.Name, fn *ir.Func, cost int32) {
   295  	if base.Flag.LowerM > 1 {
   296  		fmt.Printf("%v: can inline %v with cost %d as: %v { %v }\n", ir.Line(fn), n, cost, fn.Type(), ir.Nodes(fn.Body))
   297  	} else if base.Flag.LowerM != 0 {
   298  		fmt.Printf("%v: can inline %v\n", ir.Line(fn), n)
   299  	}
   300  	// JSON optimization log output.
   301  	if logopt.Enabled() {
   302  		logopt.LogOpt(fn.Pos(), "canInlineFunction", "inline", ir.FuncName(fn), fmt.Sprintf("cost: %d", cost))
   303  	}
   304  }
   305  
   306  // InlineImpossible returns a non-empty reason string if fn is impossible to
   307  // inline regardless of cost or contents.
   308  func InlineImpossible(fn *ir.Func) string {
   309  	var reason string // reason, if any, that the function can not be inlined.
   310  	if fn.Nname == nil {
   311  		reason = "no name"
   312  		return reason
   313  	}
   314  
   315  	// If marked "go:noinline", don't inline.
   316  	if fn.Pragma&ir.Noinline != 0 {
   317  		reason = "marked go:noinline"
   318  		return reason
   319  	}
   320  
   321  	// If marked "go:norace" and -race compilation, don't inline.
   322  	if base.Flag.Race && fn.Pragma&ir.Norace != 0 {
   323  		reason = "marked go:norace with -race compilation"
   324  		return reason
   325  	}
   326  
   327  	// If marked "go:nocheckptr" and -d checkptr compilation, don't inline.
   328  	if base.Debug.Checkptr != 0 && fn.Pragma&ir.NoCheckPtr != 0 {
   329  		reason = "marked go:nocheckptr"
   330  		return reason
   331  	}
   332  
   333  	// If marked "go:cgo_unsafe_args", don't inline, since the function
   334  	// makes assumptions about its argument frame layout.
   335  	if fn.Pragma&ir.CgoUnsafeArgs != 0 {
   336  		reason = "marked go:cgo_unsafe_args"
   337  		return reason
   338  	}
   339  
   340  	// If marked as "go:uintptrkeepalive", don't inline, since the keep
   341  	// alive information is lost during inlining.
   342  	//
   343  	// TODO(prattmic): This is handled on calls during escape analysis,
   344  	// which is after inlining. Move prior to inlining so the keep-alive is
   345  	// maintained after inlining.
   346  	if fn.Pragma&ir.UintptrKeepAlive != 0 {
   347  		reason = "marked as having a keep-alive uintptr argument"
   348  		return reason
   349  	}
   350  
   351  	// If marked as "go:uintptrescapes", don't inline, since the escape
   352  	// information is lost during inlining.
   353  	if fn.Pragma&ir.UintptrEscapes != 0 {
   354  		reason = "marked as having an escaping uintptr argument"
   355  		return reason
   356  	}
   357  
   358  	// The nowritebarrierrec checker currently works at function
   359  	// granularity, so inlining yeswritebarrierrec functions can confuse it
   360  	// (#22342). As a workaround, disallow inlining them for now.
   361  	if fn.Pragma&ir.Yeswritebarrierrec != 0 {
   362  		reason = "marked go:yeswritebarrierrec"
   363  		return reason
   364  	}
   365  
   366  	// If a local function has no fn.Body (is defined outside of Go), cannot inline it.
   367  	// Imported functions don't have fn.Body but might have inline body in fn.Inl.
   368  	if len(fn.Body) == 0 && !typecheck.HaveInlineBody(fn) {
   369  		reason = "no function body"
   370  		return reason
   371  	}
   372  
   373  	return ""
   374  }
   375  
   376  // canDelayResults reports whether inlined calls to fn can delay
   377  // declaring the result parameter until the "return" statement.
   378  func canDelayResults(fn *ir.Func) bool {
   379  	// We can delay declaring+initializing result parameters if:
   380  	// (1) there's exactly one "return" statement in the inlined function;
   381  	// (2) it's not an empty return statement (#44355); and
   382  	// (3) the result parameters aren't named.
   383  
   384  	nreturns := 0
   385  	ir.VisitList(fn.Body, func(n ir.Node) {
   386  		if n, ok := n.(*ir.ReturnStmt); ok {
   387  			nreturns++
   388  			if len(n.Results) == 0 {
   389  				nreturns++ // empty return statement (case 2)
   390  			}
   391  		}
   392  	})
   393  
   394  	if nreturns != 1 {
   395  		return false // not exactly one return statement (case 1)
   396  	}
   397  
   398  	// temporaries for return values.
   399  	for _, param := range fn.Type().Results() {
   400  		if sym := param.Sym; sym != nil && !sym.IsBlank() {
   401  			return false // found a named result parameter (case 3)
   402  		}
   403  	}
   404  
   405  	return true
   406  }
   407  
   408  // hairyVisitor visits a function body to determine its inlining
   409  // hairiness and whether or not it can be inlined.
   410  type hairyVisitor struct {
   411  	// This is needed to access the current caller in the doNode function.
   412  	curFunc       *ir.Func
   413  	isBigFunc     bool
   414  	budget        int32
   415  	maxBudget     int32
   416  	reason        string
   417  	extraCallCost int32
   418  	usedLocals    ir.NameSet
   419  	do            func(ir.Node) bool
   420  	profile       *pgoir.Profile
   421  }
   422  
   423  func (v *hairyVisitor) tooHairy(fn *ir.Func) bool {
   424  	v.do = v.doNode // cache closure
   425  	if ir.DoChildren(fn, v.do) {
   426  		return true
   427  	}
   428  	if v.budget < 0 {
   429  		v.reason = fmt.Sprintf("function too complex: cost %d exceeds budget %d", v.maxBudget-v.budget, v.maxBudget)
   430  		return true
   431  	}
   432  	return false
   433  }
   434  
   435  // doNode visits n and its children, updates the state in v, and returns true if
   436  // n makes the current function too hairy for inlining.
   437  func (v *hairyVisitor) doNode(n ir.Node) bool {
   438  	if n == nil {
   439  		return false
   440  	}
   441  opSwitch:
   442  	switch n.Op() {
   443  	// Call is okay if inlinable and we have the budget for the body.
   444  	case ir.OCALLFUNC:
   445  		n := n.(*ir.CallExpr)
   446  		var cheap bool
   447  		if n.Fun.Op() == ir.ONAME {
   448  			name := n.Fun.(*ir.Name)
   449  			if name.Class == ir.PFUNC {
   450  				s := name.Sym()
   451  				fn := s.Name
   452  				switch s.Pkg.Path {
   453  				case "internal/abi":
   454  					switch fn {
   455  					case "NoEscape":
   456  						// Special case for internal/abi.NoEscape. It does just type
   457  						// conversions to appease the escape analysis, and doesn't
   458  						// generate code.
   459  						cheap = true
   460  					}
   461  				case "internal/runtime/sys":
   462  					switch fn {
   463  					case "GetCallerPC", "GetCallerSP":
   464  						// Functions that call GetCallerPC/SP can not be inlined
   465  						// because users expect the PC/SP of the logical caller,
   466  						// but GetCallerPC/SP returns the physical caller.
   467  						v.reason = "call to " + fn
   468  						return true
   469  					}
   470  				case "go.runtime":
   471  					switch fn {
   472  					case "throw":
   473  						// runtime.throw is a "cheap call" like panic in normal code.
   474  						v.budget -= inlineExtraThrowCost
   475  						break opSwitch
   476  					case "panicrangestate":
   477  						cheap = true
   478  					}
   479  				}
   480  			}
   481  			// Special case for coverage counter updates; although
   482  			// these correspond to real operations, we treat them as
   483  			// zero cost for the moment. This is due to the existence
   484  			// of tests that are sensitive to inlining-- if the
   485  			// insertion of coverage instrumentation happens to tip a
   486  			// given function over the threshold and move it from
   487  			// "inlinable" to "not-inlinable", this can cause changes
   488  			// in allocation behavior, which can then result in test
   489  			// failures (a good example is the TestAllocations in
   490  			// crypto/ed25519).
   491  			if isAtomicCoverageCounterUpdate(n) {
   492  				return false
   493  			}
   494  		}
   495  		if n.Fun.Op() == ir.OMETHEXPR {
   496  			if meth := ir.MethodExprName(n.Fun); meth != nil {
   497  				if fn := meth.Func; fn != nil {
   498  					s := fn.Sym()
   499  					if types.RuntimeSymName(s) == "heapBits.nextArena" {
   500  						// Special case: explicitly allow mid-stack inlining of
   501  						// runtime.heapBits.next even though it calls slow-path
   502  						// runtime.heapBits.nextArena.
   503  						cheap = true
   504  					}
   505  					// Special case: on architectures that can do unaligned loads,
   506  					// explicitly mark encoding/binary methods as cheap,
   507  					// because in practice they are, even though our inlining
   508  					// budgeting system does not see that. See issue 42958.
   509  					if base.Ctxt.Arch.CanMergeLoads && s.Pkg.Path == "encoding/binary" {
   510  						switch s.Name {
   511  						case "littleEndian.Uint64", "littleEndian.Uint32", "littleEndian.Uint16",
   512  							"bigEndian.Uint64", "bigEndian.Uint32", "bigEndian.Uint16",
   513  							"littleEndian.PutUint64", "littleEndian.PutUint32", "littleEndian.PutUint16",
   514  							"bigEndian.PutUint64", "bigEndian.PutUint32", "bigEndian.PutUint16",
   515  							"littleEndian.AppendUint64", "littleEndian.AppendUint32", "littleEndian.AppendUint16",
   516  							"bigEndian.AppendUint64", "bigEndian.AppendUint32", "bigEndian.AppendUint16":
   517  							cheap = true
   518  						}
   519  					}
   520  				}
   521  			}
   522  		}
   523  
   524  		// A call to a parameter is optimistically a cheap call, if it's a constant function
   525  		// perhaps it will inline, it also can simplify escape analysis.
   526  		extraCost := v.extraCallCost
   527  
   528  		if n.Fun.Op() == ir.ONAME {
   529  			name := n.Fun.(*ir.Name)
   530  			if name.Class == ir.PFUNC {
   531  				// Special case: on architectures that can do unaligned loads,
   532  				// explicitly mark internal/byteorder methods as cheap,
   533  				// because in practice they are, even though our inlining
   534  				// budgeting system does not see that. See issue 42958.
   535  				if base.Ctxt.Arch.CanMergeLoads && name.Sym().Pkg.Path == "internal/byteorder" {
   536  					switch name.Sym().Name {
   537  					case "LeUint64", "LeUint32", "LeUint16",
   538  						"BeUint64", "BeUint32", "BeUint16",
   539  						"LePutUint64", "LePutUint32", "LePutUint16",
   540  						"BePutUint64", "BePutUint32", "BePutUint16",
   541  						"LeAppendUint64", "LeAppendUint32", "LeAppendUint16",
   542  						"BeAppendUint64", "BeAppendUint32", "BeAppendUint16":
   543  						cheap = true
   544  					}
   545  				}
   546  			}
   547  			if name.Class == ir.PPARAM || name.Class == ir.PAUTOHEAP && name.IsClosureVar() {
   548  				extraCost = min(extraCost, inlineParamCallCost)
   549  			}
   550  		}
   551  
   552  		if cheap {
   553  			break // treat like any other node, that is, cost of 1
   554  		}
   555  
   556  		if ir.IsIntrinsicCall(n) {
   557  			// Treat like any other node.
   558  			break
   559  		}
   560  
   561  		if callee := inlCallee(v.curFunc, n.Fun, v.profile); callee != nil && typecheck.HaveInlineBody(callee) {
   562  			// Check whether we'd actually inline this call. Set
   563  			// log == false since we aren't actually doing inlining
   564  			// yet.
   565  			if ok, _, _ := canInlineCallExpr(v.curFunc, n, callee, v.isBigFunc, false); ok {
   566  				// mkinlcall would inline this call [1], so use
   567  				// the cost of the inline body as the cost of
   568  				// the call, as that is what will actually
   569  				// appear in the code.
   570  				//
   571  				// [1] This is almost a perfect match to the
   572  				// mkinlcall logic, except that
   573  				// canInlineCallExpr considers inlining cycles
   574  				// by looking at what has already been inlined.
   575  				// Since we haven't done any inlining yet we
   576  				// will miss those.
   577  				v.budget -= callee.Inl.Cost
   578  				break
   579  			}
   580  		}
   581  
   582  		// Call cost for non-leaf inlining.
   583  		v.budget -= extraCost
   584  
   585  	case ir.OCALLMETH:
   586  		base.FatalfAt(n.Pos(), "OCALLMETH missed by typecheck")
   587  
   588  	// Things that are too hairy, irrespective of the budget
   589  	case ir.OCALL, ir.OCALLINTER:
   590  		// Call cost for non-leaf inlining.
   591  		v.budget -= v.extraCallCost
   592  
   593  	case ir.OPANIC:
   594  		n := n.(*ir.UnaryExpr)
   595  		if n.X.Op() == ir.OCONVIFACE && n.X.(*ir.ConvExpr).Implicit() {
   596  			// Hack to keep reflect.flag.mustBe inlinable for TestIntendedInlining.
   597  			// Before CL 284412, these conversions were introduced later in the
   598  			// compiler, so they didn't count against inlining budget.
   599  			v.budget++
   600  		}
   601  		v.budget -= inlineExtraPanicCost
   602  
   603  	case ir.ORECOVER:
   604  		base.FatalfAt(n.Pos(), "ORECOVER missed typecheck")
   605  	case ir.ORECOVERFP:
   606  		// recover matches the argument frame pointer to find
   607  		// the right panic value, so it needs an argument frame.
   608  		v.reason = "call to recover"
   609  		return true
   610  
   611  	case ir.OCLOSURE:
   612  		if base.Debug.InlFuncsWithClosures == 0 {
   613  			v.reason = "not inlining functions with closures"
   614  			return true
   615  		}
   616  
   617  		// TODO(danscales): Maybe make budget proportional to number of closure
   618  		// variables, e.g.:
   619  		//v.budget -= int32(len(n.(*ir.ClosureExpr).Func.ClosureVars) * 3)
   620  		// TODO(austin): However, if we're able to inline this closure into
   621  		// v.curFunc, then we actually pay nothing for the closure captures. We
   622  		// should try to account for that if we're going to account for captures.
   623  		v.budget -= 15
   624  
   625  	case ir.OGO, ir.ODEFER, ir.OTAILCALL:
   626  		v.reason = "unhandled op " + n.Op().String()
   627  		return true
   628  
   629  	case ir.OAPPEND:
   630  		v.budget -= inlineExtraAppendCost
   631  
   632  	case ir.OADDR:
   633  		n := n.(*ir.AddrExpr)
   634  		// Make "&s.f" cost 0 when f's offset is zero.
   635  		if dot, ok := n.X.(*ir.SelectorExpr); ok && (dot.Op() == ir.ODOT || dot.Op() == ir.ODOTPTR) {
   636  			if _, ok := dot.X.(*ir.Name); ok && dot.Selection.Offset == 0 {
   637  				v.budget += 2 // undo ir.OADDR+ir.ODOT/ir.ODOTPTR
   638  			}
   639  		}
   640  
   641  	case ir.ODEREF:
   642  		// *(*X)(unsafe.Pointer(&x)) is low-cost
   643  		n := n.(*ir.StarExpr)
   644  
   645  		ptr := n.X
   646  		for ptr.Op() == ir.OCONVNOP {
   647  			ptr = ptr.(*ir.ConvExpr).X
   648  		}
   649  		if ptr.Op() == ir.OADDR {
   650  			v.budget += 1 // undo half of default cost of ir.ODEREF+ir.OADDR
   651  		}
   652  
   653  	case ir.OCONVNOP:
   654  		// This doesn't produce code, but the children might.
   655  		v.budget++ // undo default cost
   656  
   657  	case ir.OFALL, ir.OTYPE:
   658  		// These nodes don't produce code; omit from inlining budget.
   659  		return false
   660  
   661  	case ir.OIF:
   662  		n := n.(*ir.IfStmt)
   663  		if ir.IsConst(n.Cond, constant.Bool) {
   664  			// This if and the condition cost nothing.
   665  			if doList(n.Init(), v.do) {
   666  				return true
   667  			}
   668  			if ir.BoolVal(n.Cond) {
   669  				return doList(n.Body, v.do)
   670  			} else {
   671  				return doList(n.Else, v.do)
   672  			}
   673  		}
   674  
   675  	case ir.ONAME:
   676  		n := n.(*ir.Name)
   677  		if n.Class == ir.PAUTO {
   678  			v.usedLocals.Add(n)
   679  		}
   680  
   681  	case ir.OBLOCK:
   682  		// The only OBLOCK we should see at this point is an empty one.
   683  		// In any event, let the visitList(n.List()) below take care of the statements,
   684  		// and don't charge for the OBLOCK itself. The ++ undoes the -- below.
   685  		v.budget++
   686  
   687  	case ir.OMETHVALUE, ir.OSLICELIT:
   688  		v.budget-- // Hack for toolstash -cmp.
   689  
   690  	case ir.OMETHEXPR:
   691  		v.budget++ // Hack for toolstash -cmp.
   692  
   693  	case ir.OAS2:
   694  		n := n.(*ir.AssignListStmt)
   695  
   696  		// Unified IR unconditionally rewrites:
   697  		//
   698  		//	a, b = f()
   699  		//
   700  		// into:
   701  		//
   702  		//	DCL tmp1
   703  		//	DCL tmp2
   704  		//	tmp1, tmp2 = f()
   705  		//	a, b = tmp1, tmp2
   706  		//
   707  		// so that it can insert implicit conversions as necessary. To
   708  		// minimize impact to the existing inlining heuristics (in
   709  		// particular, to avoid breaking the existing inlinability regress
   710  		// tests), we need to compensate for this here.
   711  		//
   712  		// See also identical logic in IsBigFunc.
   713  		if len(n.Rhs) > 0 {
   714  			if init := n.Rhs[0].Init(); len(init) == 1 {
   715  				if _, ok := init[0].(*ir.AssignListStmt); ok {
   716  					// 4 for each value, because each temporary variable now
   717  					// appears 3 times (DCL, LHS, RHS), plus an extra DCL node.
   718  					//
   719  					// 1 for the extra "tmp1, tmp2 = f()" assignment statement.
   720  					v.budget += 4*int32(len(n.Lhs)) + 1
   721  				}
   722  			}
   723  		}
   724  
   725  	case ir.OAS:
   726  		// Special case for coverage counter updates and coverage
   727  		// function registrations. Although these correspond to real
   728  		// operations, we treat them as zero cost for the moment. This
   729  		// is primarily due to the existence of tests that are
   730  		// sensitive to inlining-- if the insertion of coverage
   731  		// instrumentation happens to tip a given function over the
   732  		// threshold and move it from "inlinable" to "not-inlinable",
   733  		// this can cause changes in allocation behavior, which can
   734  		// then result in test failures (a good example is the
   735  		// TestAllocations in crypto/ed25519).
   736  		n := n.(*ir.AssignStmt)
   737  		if n.X.Op() == ir.OINDEX && isIndexingCoverageCounter(n.X) {
   738  			return false
   739  		}
   740  	}
   741  
   742  	v.budget--
   743  
   744  	// When debugging, don't stop early, to get full cost of inlining this function
   745  	if v.budget < 0 && base.Flag.LowerM < 2 && !logopt.Enabled() {
   746  		v.reason = "too expensive"
   747  		return true
   748  	}
   749  
   750  	return ir.DoChildren(n, v.do)
   751  }
   752  
   753  // IsBigFunc reports whether fn is a "big" function.
   754  //
   755  // Note: The criteria for "big" is heuristic and subject to change.
   756  func IsBigFunc(fn *ir.Func) bool {
   757  	budget := inlineBigFunctionNodes
   758  	return ir.Any(fn, func(n ir.Node) bool {
   759  		// See logic in hairyVisitor.doNode, explaining unified IR's
   760  		// handling of "a, b = f()" assignments.
   761  		if n, ok := n.(*ir.AssignListStmt); ok && n.Op() == ir.OAS2 && len(n.Rhs) > 0 {
   762  			if init := n.Rhs[0].Init(); len(init) == 1 {
   763  				if _, ok := init[0].(*ir.AssignListStmt); ok {
   764  					budget += 4*len(n.Lhs) + 1
   765  				}
   766  			}
   767  		}
   768  
   769  		budget--
   770  		return budget <= 0
   771  	})
   772  }
   773  
   774  // TryInlineCall returns an inlined call expression for call, or nil
   775  // if inlining is not possible.
   776  func TryInlineCall(callerfn *ir.Func, call *ir.CallExpr, bigCaller bool, profile *pgoir.Profile) *ir.InlinedCallExpr {
   777  	if base.Flag.LowerL == 0 {
   778  		return nil
   779  	}
   780  	if call.Op() != ir.OCALLFUNC {
   781  		return nil
   782  	}
   783  	if call.GoDefer || call.NoInline {
   784  		return nil
   785  	}
   786  
   787  	// Prevent inlining some reflect.Value methods when using checkptr,
   788  	// even when package reflect was compiled without it (#35073).
   789  	if base.Debug.Checkptr != 0 && call.Fun.Op() == ir.OMETHEXPR {
   790  		if method := ir.MethodExprName(call.Fun); method != nil {
   791  			switch types.ReflectSymName(method.Sym()) {
   792  			case "Value.UnsafeAddr", "Value.Pointer":
   793  				return nil
   794  			}
   795  		}
   796  	}
   797  
   798  	if base.Flag.LowerM > 3 {
   799  		fmt.Printf("%v:call to func %+v\n", ir.Line(call), call.Fun)
   800  	}
   801  	if ir.IsIntrinsicCall(call) {
   802  		return nil
   803  	}
   804  	if fn := inlCallee(callerfn, call.Fun, profile); fn != nil && typecheck.HaveInlineBody(fn) {
   805  		return mkinlcall(callerfn, call, fn, bigCaller)
   806  	}
   807  	return nil
   808  }
   809  
   810  // inlCallee takes a function-typed expression and returns the underlying function ONAME
   811  // that it refers to if statically known. Otherwise, it returns nil.
   812  func inlCallee(caller *ir.Func, fn ir.Node, profile *pgoir.Profile) (res *ir.Func) {
   813  	fn = ir.StaticValue(fn)
   814  	switch fn.Op() {
   815  	case ir.OMETHEXPR:
   816  		fn := fn.(*ir.SelectorExpr)
   817  		n := ir.MethodExprName(fn)
   818  		// Check that receiver type matches fn.X.
   819  		// TODO(mdempsky): Handle implicit dereference
   820  		// of pointer receiver argument?
   821  		if n == nil || !types.Identical(n.Type().Recv().Type, fn.X.Type()) {
   822  			return nil
   823  		}
   824  		return n.Func
   825  	case ir.ONAME:
   826  		fn := fn.(*ir.Name)
   827  		if fn.Class == ir.PFUNC {
   828  			return fn.Func
   829  		}
   830  	case ir.OCLOSURE:
   831  		fn := fn.(*ir.ClosureExpr)
   832  		c := fn.Func
   833  		if len(c.ClosureVars) != 0 && c.ClosureVars[0].Outer.Curfn != caller {
   834  			return nil // inliner doesn't support inlining across closure frames
   835  		}
   836  		CanInline(c, profile)
   837  		return c
   838  	}
   839  	return nil
   840  }
   841  
   842  var inlgen int
   843  
   844  // SSADumpInline gives the SSA back end a chance to dump the function
   845  // when producing output for debugging the compiler itself.
   846  var SSADumpInline = func(*ir.Func) {}
   847  
   848  // InlineCall allows the inliner implementation to be overridden.
   849  // If it returns nil, the function will not be inlined.
   850  var InlineCall = func(callerfn *ir.Func, call *ir.CallExpr, fn *ir.Func, inlIndex int) *ir.InlinedCallExpr {
   851  	base.Fatalf("inline.InlineCall not overridden")
   852  	panic("unreachable")
   853  }
   854  
   855  // inlineCostOK returns true if call n from caller to callee is cheap enough to
   856  // inline. bigCaller indicates that caller is a big function.
   857  //
   858  // In addition to the "cost OK" boolean, it also returns
   859  //   - the "max cost" limit used to make the decision (which may differ depending on func size)
   860  //   - the score assigned to this specific callsite
   861  //   - whether the inlined function is "hot" according to PGO.
   862  func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller bool) (bool, int32, int32, bool) {
   863  	maxCost := int32(inlineMaxBudget)
   864  	if bigCaller {
   865  		// We use this to restrict inlining into very big functions.
   866  		// See issue 26546 and 17566.
   867  		maxCost = inlineBigFunctionMaxCost
   868  	}
   869  
   870  	metric := callee.Inl.Cost
   871  	if inlheur.Enabled() {
   872  		score, ok := inlheur.GetCallSiteScore(caller, n)
   873  		if ok {
   874  			metric = int32(score)
   875  		}
   876  	}
   877  
   878  	lineOffset := pgoir.NodeLineOffset(n, caller)
   879  	csi := pgoir.CallSiteInfo{LineOffset: lineOffset, Caller: caller}
   880  	_, hot := candHotEdgeMap[csi]
   881  
   882  	if metric <= maxCost {
   883  		// Simple case. Function is already cheap enough.
   884  		return true, 0, metric, hot
   885  	}
   886  
   887  	// We'll also allow inlining of hot functions below inlineHotMaxBudget,
   888  	// but only in small functions.
   889  
   890  	if !hot {
   891  		// Cold
   892  		return false, maxCost, metric, false
   893  	}
   894  
   895  	// Hot
   896  
   897  	if bigCaller {
   898  		if base.Debug.PGODebug > 0 {
   899  			fmt.Printf("hot-big check disallows inlining for call %s (cost %d) at %v in big function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller))
   900  		}
   901  		return false, maxCost, metric, false
   902  	}
   903  
   904  	if metric > inlineHotMaxBudget {
   905  		return false, inlineHotMaxBudget, metric, false
   906  	}
   907  
   908  	if !base.PGOHash.MatchPosWithInfo(n.Pos(), "inline", nil) {
   909  		// De-selected by PGO Hash.
   910  		return false, maxCost, metric, false
   911  	}
   912  
   913  	if base.Debug.PGODebug > 0 {
   914  		fmt.Printf("hot-budget check allows inlining for call %s (cost %d) at %v in function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller))
   915  	}
   916  
   917  	return true, 0, metric, hot
   918  }
   919  
   920  // canInlineCallExpr returns true if the call n from caller to callee
   921  // can be inlined, plus the score computed for the call expr in question,
   922  // and whether the callee is hot according to PGO.
   923  // bigCaller indicates that caller is a big function. log
   924  // indicates that the 'cannot inline' reason should be logged.
   925  //
   926  // Preconditions: CanInline(callee) has already been called.
   927  func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCaller bool, log bool) (bool, int32, bool) {
   928  	if callee.Inl == nil {
   929  		// callee is never inlinable.
   930  		if log && logopt.Enabled() {
   931  			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
   932  				fmt.Sprintf("%s cannot be inlined", ir.PkgFuncName(callee)))
   933  		}
   934  		return false, 0, false
   935  	}
   936  
   937  	ok, maxCost, callSiteScore, hot := inlineCostOK(n, callerfn, callee, bigCaller)
   938  	if !ok {
   939  		// callee cost too high for this call site.
   940  		if log && logopt.Enabled() {
   941  			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
   942  				fmt.Sprintf("cost %d of %s exceeds max caller cost %d", callee.Inl.Cost, ir.PkgFuncName(callee), maxCost))
   943  		}
   944  		return false, 0, false
   945  	}
   946  
   947  	if callee == callerfn {
   948  		// Can't recursively inline a function into itself.
   949  		if log && logopt.Enabled() {
   950  			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to %s", ir.FuncName(callerfn)))
   951  		}
   952  		return false, 0, false
   953  	}
   954  
   955  	if base.Flag.Cfg.Instrumenting && types.IsNoInstrumentPkg(callee.Sym().Pkg) {
   956  		// Runtime package must not be instrumented.
   957  		// Instrument skips runtime package. However, some runtime code can be
   958  		// inlined into other packages and instrumented there. To avoid this,
   959  		// we disable inlining of runtime functions when instrumenting.
   960  		// The example that we observed is inlining of LockOSThread,
   961  		// which lead to false race reports on m contents.
   962  		if log && logopt.Enabled() {
   963  			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
   964  				fmt.Sprintf("call to runtime function %s in instrumented build", ir.PkgFuncName(callee)))
   965  		}
   966  		return false, 0, false
   967  	}
   968  
   969  	if base.Flag.Race && types.IsNoRacePkg(callee.Sym().Pkg) {
   970  		if log && logopt.Enabled() {
   971  			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
   972  				fmt.Sprintf(`call to into "no-race" package function %s in race build`, ir.PkgFuncName(callee)))
   973  		}
   974  		return false, 0, false
   975  	}
   976  
   977  	if base.Debug.Checkptr != 0 && types.IsRuntimePkg(callee.Sym().Pkg) {
   978  		// We don't instrument runtime packages for checkptr (see base/flag.go).
   979  		if log && logopt.Enabled() {
   980  			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
   981  				fmt.Sprintf(`call to into runtime package function %s in -d=checkptr build`, ir.PkgFuncName(callee)))
   982  		}
   983  		return false, 0, false
   984  	}
   985  
   986  	// Check if we've already inlined this function at this particular
   987  	// call site, in order to stop inlining when we reach the beginning
   988  	// of a recursion cycle again. We don't inline immediately recursive
   989  	// functions, but allow inlining if there is a recursion cycle of
   990  	// many functions. Most likely, the inlining will stop before we
   991  	// even hit the beginning of the cycle again, but this catches the
   992  	// unusual case.
   993  	parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()
   994  	sym := callee.Linksym()
   995  	for inlIndex := parent; inlIndex >= 0; inlIndex = base.Ctxt.InlTree.Parent(inlIndex) {
   996  		if base.Ctxt.InlTree.InlinedFunction(inlIndex) == sym {
   997  			if log {
   998  				if base.Flag.LowerM > 1 {
   999  					fmt.Printf("%v: cannot inline %v into %v: repeated recursive cycle\n", ir.Line(n), callee, ir.FuncName(callerfn))
  1000  				}
  1001  				if logopt.Enabled() {
  1002  					logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
  1003  						fmt.Sprintf("repeated recursive cycle to %s", ir.PkgFuncName(callee)))
  1004  				}
  1005  			}
  1006  			return false, 0, false
  1007  		}
  1008  	}
  1009  
  1010  	return true, callSiteScore, hot
  1011  }
  1012  
  1013  // mkinlcall returns an OINLCALL node that can replace OCALLFUNC n, or
  1014  // nil if it cannot be inlined. callerfn is the function that contains
  1015  // n, and fn is the function being called.
  1016  //
  1017  // The result of mkinlcall MUST be assigned back to n, e.g.
  1018  //
  1019  //	n.Left = mkinlcall(n.Left, fn, isddd)
  1020  func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller bool) *ir.InlinedCallExpr {
  1021  	ok, score, hot := canInlineCallExpr(callerfn, n, fn, bigCaller, true)
  1022  	if !ok {
  1023  		return nil
  1024  	}
  1025  	if hot {
  1026  		hasHotCall[callerfn] = struct{}{}
  1027  	}
  1028  	typecheck.AssertFixedCall(n)
  1029  
  1030  	parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()
  1031  	sym := fn.Linksym()
  1032  	inlIndex := base.Ctxt.InlTree.Add(parent, n.Pos(), sym, ir.FuncName(fn))
  1033  
  1034  	closureInitLSym := func(n *ir.CallExpr, fn *ir.Func) {
  1035  		// The linker needs FuncInfo metadata for all inlined
  1036  		// functions. This is typically handled by gc.enqueueFunc
  1037  		// calling ir.InitLSym for all function declarations in
  1038  		// typecheck.Target.Decls (ir.UseClosure adds all closures to
  1039  		// Decls).
  1040  		//
  1041  		// However, closures in Decls are ignored, and are
  1042  		// instead enqueued when walk of the calling function
  1043  		// discovers them.
  1044  		//
  1045  		// This presents a problem for direct calls to closures.
  1046  		// Inlining will replace the entire closure definition with its
  1047  		// body, which hides the closure from walk and thus suppresses
  1048  		// symbol creation.
  1049  		//
  1050  		// Explicitly create a symbol early in this edge case to ensure
  1051  		// we keep this metadata.
  1052  		//
  1053  		// TODO: Refactor to keep a reference so this can all be done
  1054  		// by enqueueFunc.
  1055  
  1056  		if n.Op() != ir.OCALLFUNC {
  1057  			// Not a standard call.
  1058  			return
  1059  		}
  1060  		if n.Fun.Op() != ir.OCLOSURE {
  1061  			// Not a direct closure call.
  1062  			return
  1063  		}
  1064  
  1065  		clo := n.Fun.(*ir.ClosureExpr)
  1066  		if !clo.Func.IsClosure() {
  1067  			// enqueueFunc will handle non closures anyways.
  1068  			return
  1069  		}
  1070  
  1071  		ir.InitLSym(fn, true)
  1072  	}
  1073  
  1074  	closureInitLSym(n, fn)
  1075  
  1076  	if base.Flag.GenDwarfInl > 0 {
  1077  		if !sym.WasInlined() {
  1078  			base.Ctxt.DwFixups.SetPrecursorFunc(sym, fn)
  1079  			sym.Set(obj.AttrWasInlined, true)
  1080  		}
  1081  	}
  1082  
  1083  	if base.Flag.LowerM != 0 {
  1084  		if buildcfg.Experiment.NewInliner {
  1085  			fmt.Printf("%v: inlining call to %v with score %d\n",
  1086  				ir.Line(n), fn, score)
  1087  		} else {
  1088  			fmt.Printf("%v: inlining call to %v\n", ir.Line(n), fn)
  1089  		}
  1090  	}
  1091  	if base.Flag.LowerM > 2 {
  1092  		fmt.Printf("%v: Before inlining: %+v\n", ir.Line(n), n)
  1093  	}
  1094  
  1095  	res := InlineCall(callerfn, n, fn, inlIndex)
  1096  
  1097  	if res == nil {
  1098  		base.FatalfAt(n.Pos(), "inlining call to %v failed", fn)
  1099  	}
  1100  
  1101  	if base.Flag.LowerM > 2 {
  1102  		fmt.Printf("%v: After inlining %+v\n\n", ir.Line(res), res)
  1103  	}
  1104  
  1105  	if inlheur.Enabled() {
  1106  		inlheur.UpdateCallsiteTable(callerfn, n, res)
  1107  	}
  1108  
  1109  	return res
  1110  }
  1111  
  1112  // CalleeEffects appends any side effects from evaluating callee to init.
  1113  func CalleeEffects(init *ir.Nodes, callee ir.Node) {
  1114  	for {
  1115  		init.Append(ir.TakeInit(callee)...)
  1116  
  1117  		switch callee.Op() {
  1118  		case ir.ONAME, ir.OCLOSURE, ir.OMETHEXPR:
  1119  			return // done
  1120  
  1121  		case ir.OCONVNOP:
  1122  			conv := callee.(*ir.ConvExpr)
  1123  			callee = conv.X
  1124  
  1125  		case ir.OINLCALL:
  1126  			ic := callee.(*ir.InlinedCallExpr)
  1127  			init.Append(ic.Body.Take()...)
  1128  			callee = ic.SingleResult()
  1129  
  1130  		default:
  1131  			base.FatalfAt(callee.Pos(), "unexpected callee expression: %v", callee)
  1132  		}
  1133  	}
  1134  }
  1135  
  1136  func pruneUnusedAutos(ll []*ir.Name, vis *hairyVisitor) []*ir.Name {
  1137  	s := make([]*ir.Name, 0, len(ll))
  1138  	for _, n := range ll {
  1139  		if n.Class == ir.PAUTO {
  1140  			if !vis.usedLocals.Has(n) {
  1141  				// TODO(mdempsky): Simplify code after confident that this
  1142  				// never happens anymore.
  1143  				base.FatalfAt(n.Pos(), "unused auto: %v", n)
  1144  				continue
  1145  			}
  1146  		}
  1147  		s = append(s, n)
  1148  	}
  1149  	return s
  1150  }
  1151  
  1152  // numNonClosures returns the number of functions in list which are not closures.
  1153  func numNonClosures(list []*ir.Func) int {
  1154  	count := 0
  1155  	for _, fn := range list {
  1156  		if fn.OClosure == nil {
  1157  			count++
  1158  		}
  1159  	}
  1160  	return count
  1161  }
  1162  
  1163  func doList(list []ir.Node, do func(ir.Node) bool) bool {
  1164  	for _, x := range list {
  1165  		if x != nil {
  1166  			if do(x) {
  1167  				return true
  1168  			}
  1169  		}
  1170  	}
  1171  	return false
  1172  }
  1173  
  1174  // isIndexingCoverageCounter returns true if the specified node 'n' is indexing
  1175  // into a coverage counter array.
  1176  func isIndexingCoverageCounter(n ir.Node) bool {
  1177  	if n.Op() != ir.OINDEX {
  1178  		return false
  1179  	}
  1180  	ixn := n.(*ir.IndexExpr)
  1181  	if ixn.X.Op() != ir.ONAME || !ixn.X.Type().IsArray() {
  1182  		return false
  1183  	}
  1184  	nn := ixn.X.(*ir.Name)
  1185  	// CoverageAuxVar implies either a coverage counter or a package
  1186  	// ID; since the cover tool never emits code to index into ID vars
  1187  	// this is effectively testing whether nn is a coverage counter.
  1188  	return nn.CoverageAuxVar()
  1189  }
  1190  
  1191  // isAtomicCoverageCounterUpdate examines the specified node to
  1192  // determine whether it represents a call to sync/atomic.AddUint32 to
  1193  // increment a coverage counter.
  1194  func isAtomicCoverageCounterUpdate(cn *ir.CallExpr) bool {
  1195  	if cn.Fun.Op() != ir.ONAME {
  1196  		return false
  1197  	}
  1198  	name := cn.Fun.(*ir.Name)
  1199  	if name.Class != ir.PFUNC {
  1200  		return false
  1201  	}
  1202  	fn := name.Sym().Name
  1203  	if name.Sym().Pkg.Path != "sync/atomic" ||
  1204  		(fn != "AddUint32" && fn != "StoreUint32") {
  1205  		return false
  1206  	}
  1207  	if len(cn.Args) != 2 || cn.Args[0].Op() != ir.OADDR {
  1208  		return false
  1209  	}
  1210  	adn := cn.Args[0].(*ir.AddrExpr)
  1211  	v := isIndexingCoverageCounter(adn.X)
  1212  	return v
  1213  }
  1214  
  1215  func PostProcessCallSites(profile *pgoir.Profile) {
  1216  	if base.Debug.DumpInlCallSiteScores != 0 {
  1217  		budgetCallback := func(fn *ir.Func, prof *pgoir.Profile) (int32, bool) {
  1218  			v := inlineBudget(fn, prof, false, false)
  1219  			return v, v == inlineHotMaxBudget
  1220  		}
  1221  		inlheur.DumpInlCallSiteScores(profile, budgetCallback)
  1222  	}
  1223  }
  1224  
  1225  func analyzeFuncProps(fn *ir.Func, p *pgoir.Profile) {
  1226  	canInline := func(fn *ir.Func) { CanInline(fn, p) }
  1227  	budgetForFunc := func(fn *ir.Func) int32 {
  1228  		return inlineBudget(fn, p, true, false)
  1229  	}
  1230  	inlheur.AnalyzeFunc(fn, canInline, budgetForFunc, inlineMaxBudget)
  1231  }
  1232  

View as plain text