unified.go

     1  // Copyright 2021 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package noder
     6  
     7  import (
     8  	"cmp"
     9  	"fmt"
    10  	"internal/buildcfg"
    11  	"internal/pkgbits"
    12  	"internal/types/errors"
    13  	"io"
    14  	"runtime"
    15  	"slices"
    16  	"strings"
    17  
    18  	"cmd/compile/internal/base"
    19  	"cmd/compile/internal/inline"
    20  	"cmd/compile/internal/ir"
    21  	"cmd/compile/internal/pgoir"
    22  	"cmd/compile/internal/typecheck"
    23  	"cmd/compile/internal/types"
    24  	"cmd/compile/internal/types2"
    25  	"cmd/internal/src"
    26  )
    27  
    28  // localPkgReader holds the package reader used for reading the local
    29  // package. It exists so the unified IR linker can refer back to it
    30  // later.
    31  var localPkgReader *pkgReader
    32  
    33  // LookupFunc returns the ir.Func for an arbitrary full symbol name if
    34  // that function exists in the set of available export data.
    35  //
    36  // This allows lookup of arbitrary functions and methods that aren't otherwise
    37  // referenced by the local package and thus haven't been read yet.
    38  //
    39  // TODO(prattmic): Does not handle instantiation of generic types. Currently
    40  // profiles don't contain the original type arguments, so we won't be able to
    41  // create the runtime dictionaries.
    42  //
    43  // TODO(prattmic): Hit rate of this function is usually fairly low, and errors
    44  // are only used when debug logging is enabled. Consider constructing cheaper
    45  // errors by default.
    46  func LookupFunc(fullName string) (*ir.Func, error) {
    47  	pkgPath, symName, err := ir.ParseLinkFuncName(fullName)
    48  	if err != nil {
    49  		return nil, fmt.Errorf("error parsing symbol name %q: %v", fullName, err)
    50  	}
    51  
    52  	pkg, ok := types.PkgMap()[pkgPath]
    53  	if !ok {
    54  		return nil, fmt.Errorf("pkg %s doesn't exist in %v", pkgPath, types.PkgMap())
    55  	}
    56  
    57  	// Symbol naming is ambiguous. We can't necessarily distinguish between
    58  	// a method and a closure. e.g., is foo.Bar.func1 a closure defined in
    59  	// function Bar, or a method on type Bar? Thus we must simply attempt
    60  	// to lookup both.
    61  
    62  	fn, err := lookupFunction(pkg, symName)
    63  	if err == nil {
    64  		return fn, nil
    65  	}
    66  
    67  	fn, mErr := lookupMethod(pkg, symName)
    68  	if mErr == nil {
    69  		return fn, nil
    70  	}
    71  
    72  	return nil, fmt.Errorf("%s is not a function (%v) or method (%v)", fullName, err, mErr)
    73  }
    74  
    75  // PostLookupCleanup performs cleanup operations needed
    76  // after a series of calls to LookupFunc, specifically invoking
    77  // readBodies to post-process any funcs on the "todoBodies" list
    78  // that were added as a result of the lookup operations.
    79  func PostLookupCleanup() {
    80  	readBodies(typecheck.Target, false)
    81  }
    82  
    83  func lookupFunction(pkg *types.Pkg, symName string) (*ir.Func, error) {
    84  	sym := pkg.Lookup(symName)
    85  
    86  	// TODO(prattmic): Enclosed functions (e.g., foo.Bar.func1) are not
    87  	// present in objReader, only as OCLOSURE nodes in the enclosing
    88  	// function.
    89  	pri, ok := objReader[sym]
    90  	if !ok {
    91  		return nil, fmt.Errorf("func sym %v missing objReader", sym)
    92  	}
    93  
    94  	node, err := pri.pr.objIdxMayFail(pri.idx, nil, nil, false)
    95  	if err != nil {
    96  		return nil, fmt.Errorf("func sym %v lookup error: %w", sym, err)
    97  	}
    98  	name := node.(*ir.Name)
    99  	if name.Op() != ir.ONAME || name.Class != ir.PFUNC {
   100  		return nil, fmt.Errorf("func sym %v refers to non-function name: %v", sym, name)
   101  	}
   102  	return name.Func, nil
   103  }
   104  
   105  func lookupMethod(pkg *types.Pkg, symName string) (*ir.Func, error) {
   106  	// N.B. readPackage creates a Sym for every object in the package to
   107  	// initialize objReader and importBodyReader, even if the object isn't
   108  	// read.
   109  	//
   110  	// However, objReader is only initialized for top-level objects, so we
   111  	// must first lookup the type and use that to find the method rather
   112  	// than looking for the method directly.
   113  	typ, meth, err := ir.LookupMethodSelector(pkg, symName)
   114  	if err != nil {
   115  		return nil, fmt.Errorf("error looking up method symbol %q: %v", symName, err)
   116  	}
   117  
   118  	pri, ok := objReader[typ]
   119  	if !ok {
   120  		return nil, fmt.Errorf("type sym %v missing objReader", typ)
   121  	}
   122  
   123  	node, err := pri.pr.objIdxMayFail(pri.idx, nil, nil, false)
   124  	if err != nil {
   125  		return nil, fmt.Errorf("func sym %v lookup error: %w", typ, err)
   126  	}
   127  	name := node.(*ir.Name)
   128  	if name.Op() != ir.OTYPE {
   129  		return nil, fmt.Errorf("type sym %v refers to non-type name: %v", typ, name)
   130  	}
   131  	if name.Alias() {
   132  		return nil, fmt.Errorf("type sym %v refers to alias", typ)
   133  	}
   134  	if name.Type().IsInterface() {
   135  		return nil, fmt.Errorf("type sym %v refers to interface type", typ)
   136  	}
   137  
   138  	for _, m := range name.Type().Methods() {
   139  		if m.Sym == meth {
   140  			fn := m.Nname.(*ir.Name).Func
   141  			return fn, nil
   142  		}
   143  	}
   144  
   145  	return nil, fmt.Errorf("method %s missing from method set of %v", symName, typ)
   146  }
   147  
   148  // unified constructs the local package's Internal Representation (IR)
   149  // from its syntax tree (AST).
   150  //
   151  // The pipeline contains 2 steps:
   152  //
   153  //  1. Generate the export data "stub".
   154  //
   155  //  2. Generate the IR from the export data above.
   156  //
   157  // The package data "stub" at step (1) contains everything from the local package,
   158  // but nothing that has been imported. When we're actually writing out export data
   159  // to the output files (see writeNewExport), we run the "linker", which:
   160  //
   161  //   - Updates compiler extensions data (e.g. inlining cost, escape analysis results).
   162  //
   163  //   - Handles re-exporting any transitive dependencies.
   164  //
   165  //   - Prunes out any unnecessary details (e.g. non-inlineable functions, because any
   166  //     downstream importers only care about inlinable functions).
   167  //
   168  // The source files are typechecked twice: once before writing the export data
   169  // using types2, and again after reading the export data using gc/typecheck.
   170  // The duplication of work will go away once we only use the types2 type checker,
   171  // removing the gc/typecheck step. For now, it is kept because:
   172  //
   173  //   - It reduces the engineering costs in maintaining a fork of typecheck
   174  //     (e.g. no need to backport fixes like CL 327651).
   175  //
   176  //   - It makes it easier to pass toolstash -cmp.
   177  //
   178  //   - Historically, we would always re-run the typechecker after importing a package,
   179  //     even though we know the imported data is valid. It's not ideal, but it's
   180  //     not causing any problems either.
   181  //
   182  //   - gc/typecheck is still in charge of some transformations, such as rewriting
   183  //     multi-valued function calls or transforming ir.OINDEX to ir.OINDEXMAP.
   184  //
   185  // Using the syntax tree with types2, which has a complete representation of generics,
   186  // the unified IR has the full typed AST needed for introspection during step (1).
   187  // In other words, we have all the necessary information to build the generic IR form
   188  // (see writer.captureVars for an example).
   189  func unified(m posMap, noders []*noder) {
   190  	inline.InlineCall = unifiedInlineCall
   191  	typecheck.HaveInlineBody = unifiedHaveInlineBody
   192  	pgoir.LookupFunc = LookupFunc
   193  	pgoir.PostLookupCleanup = PostLookupCleanup
   194  
   195  	data := writePkgStub(m, noders)
   196  
   197  	target := typecheck.Target
   198  
   199  	localPkgReader = newPkgReader(pkgbits.NewPkgDecoder(types.LocalPkg.Path, data))
   200  	readPackage(localPkgReader, types.LocalPkg, true)
   201  
   202  	r := localPkgReader.newReader(pkgbits.RelocMeta, pkgbits.PrivateRootIdx, pkgbits.SyncPrivate)
   203  	r.pkgInit(types.LocalPkg, target)
   204  
   205  	readBodies(target, false)
   206  
   207  	// Check that nothing snuck past typechecking.
   208  	for _, fn := range target.Funcs {
   209  		if fn.Typecheck() == 0 {
   210  			base.FatalfAt(fn.Pos(), "missed typecheck: %v", fn)
   211  		}
   212  
   213  		// For functions, check that at least their first statement (if
   214  		// any) was typechecked too.
   215  		if len(fn.Body) != 0 {
   216  			if stmt := fn.Body[0]; stmt.Typecheck() == 0 {
   217  				base.FatalfAt(stmt.Pos(), "missed typecheck: %v", stmt)
   218  			}
   219  		}
   220  	}
   221  
   222  	// For functions originally came from package runtime,
   223  	// mark as norace to prevent instrumenting, see issue #60439.
   224  	for _, fn := range target.Funcs {
   225  		if !base.Flag.CompilingRuntime && types.RuntimeSymName(fn.Sym()) != "" {
   226  			fn.Pragma |= ir.Norace
   227  		}
   228  	}
   229  
   230  	base.ExitIfErrors() // just in case
   231  }
   232  
   233  // readBodies iteratively expands all pending dictionaries and
   234  // function bodies.
   235  //
   236  // If duringInlining is true, then the inline.InlineDecls is called as
   237  // necessary on instantiations of imported generic functions, so their
   238  // inlining costs can be computed.
   239  func readBodies(target *ir.Package, duringInlining bool) {
   240  	var inlDecls []*ir.Func
   241  
   242  	// Don't use range--bodyIdx can add closures to todoBodies.
   243  	for {
   244  		// The order we expand dictionaries and bodies doesn't matter, so
   245  		// pop from the end to reduce todoBodies reallocations if it grows
   246  		// further.
   247  		//
   248  		// However, we do at least need to flush any pending dictionaries
   249  		// before reading bodies, because bodies might reference the
   250  		// dictionaries.
   251  
   252  		if len(todoDicts) > 0 {
   253  			fn := todoDicts[len(todoDicts)-1]
   254  			todoDicts = todoDicts[:len(todoDicts)-1]
   255  			fn()
   256  			continue
   257  		}
   258  
   259  		if len(todoBodies) > 0 {
   260  			fn := todoBodies[len(todoBodies)-1]
   261  			todoBodies = todoBodies[:len(todoBodies)-1]
   262  
   263  			pri, ok := bodyReader[fn]
   264  			assert(ok)
   265  			pri.funcBody(fn)
   266  
   267  			// Instantiated generic function: add to Decls for typechecking
   268  			// and compilation.
   269  			if fn.OClosure == nil && len(pri.dict.targs) != 0 {
   270  				// cmd/link does not support a type symbol referencing a method symbol
   271  				// across DSO boundary, so force re-compiling methods on a generic type
   272  				// even it was seen from imported package in linkshared mode, see #58966.
   273  				canSkipNonGenericMethod := !(base.Ctxt.Flag_linkshared && ir.IsMethod(fn))
   274  				if duringInlining && canSkipNonGenericMethod {
   275  					inlDecls = append(inlDecls, fn)
   276  				} else {
   277  					target.Funcs = append(target.Funcs, fn)
   278  				}
   279  			}
   280  
   281  			continue
   282  		}
   283  
   284  		break
   285  	}
   286  
   287  	todoDicts = nil
   288  	todoBodies = nil
   289  
   290  	if len(inlDecls) != 0 {
   291  		// If we instantiated any generic functions during inlining, we need
   292  		// to call CanInline on them so they'll be transitively inlined
   293  		// correctly (#56280).
   294  		//
   295  		// We know these functions were already compiled in an imported
   296  		// package though, so we don't need to actually apply InlineCalls or
   297  		// save the function bodies any further than this.
   298  		//
   299  		// We can also lower the -m flag to 0, to suppress duplicate "can
   300  		// inline" diagnostics reported against the imported package. Again,
   301  		// we already reported those diagnostics in the original package, so
   302  		// it's pointless repeating them here.
   303  
   304  		oldLowerM := base.Flag.LowerM
   305  		base.Flag.LowerM = 0
   306  		inline.CanInlineFuncs(inlDecls, nil)
   307  		base.Flag.LowerM = oldLowerM
   308  
   309  		for _, fn := range inlDecls {
   310  			fn.Body = nil // free memory
   311  		}
   312  	}
   313  }
   314  
   315  // writePkgStub type checks the given parsed source files,
   316  // writes an export data package stub representing them,
   317  // and returns the result.
   318  func writePkgStub(m posMap, noders []*noder) string {
   319  	pkg, info, otherInfo := checkFiles(m, noders)
   320  
   321  	pw := newPkgWriter(m, pkg, info, otherInfo)
   322  
   323  	pw.collectDecls(noders)
   324  
   325  	publicRootWriter := pw.newWriter(pkgbits.RelocMeta, pkgbits.SyncPublic)
   326  	privateRootWriter := pw.newWriter(pkgbits.RelocMeta, pkgbits.SyncPrivate)
   327  
   328  	assert(publicRootWriter.Idx == pkgbits.PublicRootIdx)
   329  	assert(privateRootWriter.Idx == pkgbits.PrivateRootIdx)
   330  
   331  	{
   332  		w := publicRootWriter
   333  		w.pkg(pkg)
   334  
   335  		if w.Version().Has(pkgbits.HasInit) {
   336  			w.Bool(false)
   337  		}
   338  
   339  		scope := pkg.Scope()
   340  		names := scope.Names()
   341  		w.Len(len(names))
   342  		for _, name := range names {
   343  			w.obj(scope.Lookup(name), nil)
   344  		}
   345  
   346  		w.Sync(pkgbits.SyncEOF)
   347  		w.Flush()
   348  	}
   349  
   350  	{
   351  		w := privateRootWriter
   352  		w.pkgInit(noders)
   353  		w.Flush()
   354  	}
   355  
   356  	var sb strings.Builder
   357  	pw.DumpTo(&sb)
   358  
   359  	// At this point, we're done with types2. Make sure the package is
   360  	// garbage collected.
   361  	freePackage(pkg)
   362  
   363  	return sb.String()
   364  }
   365  
   366  // freePackage ensures the given package is garbage collected.
   367  func freePackage(pkg *types2.Package) {
   368  	// The GC test below relies on a precise GC that runs finalizers as
   369  	// soon as objects are unreachable. Our implementation provides
   370  	// this, but other/older implementations may not (e.g., Go 1.4 does
   371  	// not because of #22350). To avoid imposing unnecessary
   372  	// restrictions on the GOROOT_BOOTSTRAP toolchain, we skip the test
   373  	// during bootstrapping.
   374  	if base.CompilerBootstrap || base.Debug.GCCheck == 0 {
   375  		*pkg = types2.Package{}
   376  		return
   377  	}
   378  
   379  	// Set a finalizer on pkg so we can detect if/when it's collected.
   380  	done := make(chan struct{})
   381  	runtime.SetFinalizer(pkg, func(*types2.Package) { close(done) })
   382  
   383  	// Important: objects involved in cycles are not finalized, so zero
   384  	// out pkg to break its cycles and allow the finalizer to run.
   385  	*pkg = types2.Package{}
   386  
   387  	// It typically takes just 1 or 2 cycles to release pkg, but it
   388  	// doesn't hurt to try a few more times.
   389  	for i := 0; i < 10; i++ {
   390  		select {
   391  		case <-done:
   392  			return
   393  		default:
   394  			runtime.GC()
   395  		}
   396  	}
   397  
   398  	base.Fatalf("package never finalized")
   399  }
   400  
   401  // readPackage reads package export data from pr to populate
   402  // importpkg.
   403  //
   404  // localStub indicates whether pr is reading the stub export data for
   405  // the local package, as opposed to relocated export data for an
   406  // import.
   407  func readPackage(pr *pkgReader, importpkg *types.Pkg, localStub bool) {
   408  	{
   409  		r := pr.newReader(pkgbits.RelocMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic)
   410  
   411  		pkg := r.pkg()
   412  		// This error can happen if "go tool compile" is called with wrong "-p" flag, see issue #54542.
   413  		if pkg != importpkg {
   414  			base.ErrorfAt(base.AutogeneratedPos, errors.BadImportPath, "mismatched import path, have %q (%p), want %q (%p)", pkg.Path, pkg, importpkg.Path, importpkg)
   415  			base.ErrorExit()
   416  		}
   417  
   418  		if r.Version().Has(pkgbits.HasInit) {
   419  			r.Bool()
   420  		}
   421  
   422  		for i, n := 0, r.Len(); i < n; i++ {
   423  			r.Sync(pkgbits.SyncObject)
   424  			if r.Version().Has(pkgbits.DerivedFuncInstance) {
   425  				assert(!r.Bool())
   426  			}
   427  			idx := r.Reloc(pkgbits.RelocObj)
   428  			assert(r.Len() == 0)
   429  
   430  			path, name, code := r.p.PeekObj(idx)
   431  			if code != pkgbits.ObjStub {
   432  				objReader[types.NewPkg(path, "").Lookup(name)] = pkgReaderIndex{pr, idx, nil, nil, nil}
   433  			}
   434  		}
   435  
   436  		r.Sync(pkgbits.SyncEOF)
   437  	}
   438  
   439  	if !localStub {
   440  		r := pr.newReader(pkgbits.RelocMeta, pkgbits.PrivateRootIdx, pkgbits.SyncPrivate)
   441  
   442  		if r.Bool() {
   443  			sym := importpkg.Lookup(".inittask")
   444  			task := ir.NewNameAt(src.NoXPos, sym, nil)
   445  			task.Class = ir.PEXTERN
   446  			sym.Def = task
   447  		}
   448  
   449  		for i, n := 0, r.Len(); i < n; i++ {
   450  			path := r.String()
   451  			name := r.String()
   452  			idx := r.Reloc(pkgbits.RelocBody)
   453  
   454  			sym := types.NewPkg(path, "").Lookup(name)
   455  			if _, ok := importBodyReader[sym]; !ok {
   456  				importBodyReader[sym] = pkgReaderIndex{pr, idx, nil, nil, nil}
   457  			}
   458  		}
   459  
   460  		r.Sync(pkgbits.SyncEOF)
   461  	}
   462  }
   463  
   464  // writeUnifiedExport writes to `out` the finalized, self-contained
   465  // Unified IR export data file for the current compilation unit.
   466  func writeUnifiedExport(out io.Writer) {
   467  	// Use V2 as the encoded version aliastypeparams GOEXPERIMENT is enabled.
   468  	version := pkgbits.V1
   469  	if buildcfg.Experiment.AliasTypeParams {
   470  		version = pkgbits.V2
   471  	}
   472  	l := linker{
   473  		pw: pkgbits.NewPkgEncoder(version, base.Debug.SyncFrames),
   474  
   475  		pkgs:   make(map[string]index),
   476  		decls:  make(map[*types.Sym]index),
   477  		bodies: make(map[*types.Sym]index),
   478  	}
   479  
   480  	publicRootWriter := l.pw.NewEncoder(pkgbits.RelocMeta, pkgbits.SyncPublic)
   481  	privateRootWriter := l.pw.NewEncoder(pkgbits.RelocMeta, pkgbits.SyncPrivate)
   482  	assert(publicRootWriter.Idx == pkgbits.PublicRootIdx)
   483  	assert(privateRootWriter.Idx == pkgbits.PrivateRootIdx)
   484  
   485  	var selfPkgIdx index
   486  
   487  	{
   488  		pr := localPkgReader
   489  		r := pr.NewDecoder(pkgbits.RelocMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic)
   490  
   491  		r.Sync(pkgbits.SyncPkg)
   492  		selfPkgIdx = l.relocIdx(pr, pkgbits.RelocPkg, r.Reloc(pkgbits.RelocPkg))
   493  
   494  		if r.Version().Has(pkgbits.HasInit) {
   495  			r.Bool()
   496  		}
   497  
   498  		for i, n := 0, r.Len(); i < n; i++ {
   499  			r.Sync(pkgbits.SyncObject)
   500  			if r.Version().Has(pkgbits.DerivedFuncInstance) {
   501  				assert(!r.Bool())
   502  			}
   503  			idx := r.Reloc(pkgbits.RelocObj)
   504  			assert(r.Len() == 0)
   505  
   506  			xpath, xname, xtag := pr.PeekObj(idx)
   507  			assert(xpath == pr.PkgPath())
   508  			assert(xtag != pkgbits.ObjStub)
   509  
   510  			if types.IsExported(xname) {
   511  				l.relocIdx(pr, pkgbits.RelocObj, idx)
   512  			}
   513  		}
   514  
   515  		r.Sync(pkgbits.SyncEOF)
   516  	}
   517  
   518  	{
   519  		var idxs []index
   520  		for _, idx := range l.decls {
   521  			idxs = append(idxs, idx)
   522  		}
   523  		slices.Sort(idxs)
   524  
   525  		w := publicRootWriter
   526  
   527  		w.Sync(pkgbits.SyncPkg)
   528  		w.Reloc(pkgbits.RelocPkg, selfPkgIdx)
   529  
   530  		if w.Version().Has(pkgbits.HasInit) {
   531  			w.Bool(false)
   532  		}
   533  
   534  		w.Len(len(idxs))
   535  		for _, idx := range idxs {
   536  			w.Sync(pkgbits.SyncObject)
   537  			if w.Version().Has(pkgbits.DerivedFuncInstance) {
   538  				w.Bool(false)
   539  			}
   540  			w.Reloc(pkgbits.RelocObj, idx)
   541  			w.Len(0)
   542  		}
   543  
   544  		w.Sync(pkgbits.SyncEOF)
   545  		w.Flush()
   546  	}
   547  
   548  	{
   549  		type symIdx struct {
   550  			sym *types.Sym
   551  			idx index
   552  		}
   553  		var bodies []symIdx
   554  		for sym, idx := range l.bodies {
   555  			bodies = append(bodies, symIdx{sym, idx})
   556  		}
   557  		slices.SortFunc(bodies, func(a, b symIdx) int { return cmp.Compare(a.idx, b.idx) })
   558  
   559  		w := privateRootWriter
   560  
   561  		w.Bool(typecheck.Lookup(".inittask").Def != nil)
   562  
   563  		w.Len(len(bodies))
   564  		for _, body := range bodies {
   565  			w.String(body.sym.Pkg.Path)
   566  			w.String(body.sym.Name)
   567  			w.Reloc(pkgbits.RelocBody, body.idx)
   568  		}
   569  
   570  		w.Sync(pkgbits.SyncEOF)
   571  		w.Flush()
   572  	}
   573  
   574  	base.Ctxt.Fingerprint = l.pw.DumpTo(out)
   575  }
   576
View as plain text