unified.go

     1  // Copyright 2021 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package noder
     6  
     7  import (
     8  	"fmt"
     9  	"internal/pkgbits"
    10  	"internal/types/errors"
    11  	"io"
    12  	"runtime"
    13  	"sort"
    14  	"strings"
    15  
    16  	"cmd/compile/internal/base"
    17  	"cmd/compile/internal/inline"
    18  	"cmd/compile/internal/ir"
    19  	"cmd/compile/internal/pgoir"
    20  	"cmd/compile/internal/typecheck"
    21  	"cmd/compile/internal/types"
    22  	"cmd/compile/internal/types2"
    23  	"cmd/internal/src"
    24  )
    25  
    26  // localPkgReader holds the package reader used for reading the local
    27  // package. It exists so the unified IR linker can refer back to it
    28  // later.
    29  var localPkgReader *pkgReader
    30  
    31  // LookupFunc returns the ir.Func for an arbitrary full symbol name if
    32  // that function exists in the set of available export data.
    33  //
    34  // This allows lookup of arbitrary functions and methods that aren't otherwise
    35  // referenced by the local package and thus haven't been read yet.
    36  //
    37  // TODO(prattmic): Does not handle instantiation of generic types. Currently
    38  // profiles don't contain the original type arguments, so we won't be able to
    39  // create the runtime dictionaries.
    40  //
    41  // TODO(prattmic): Hit rate of this function is usually fairly low, and errors
    42  // are only used when debug logging is enabled. Consider constructing cheaper
    43  // errors by default.
    44  func LookupFunc(fullName string) (*ir.Func, error) {
    45  	pkgPath, symName, err := ir.ParseLinkFuncName(fullName)
    46  	if err != nil {
    47  		return nil, fmt.Errorf("error parsing symbol name %q: %v", fullName, err)
    48  	}
    49  
    50  	pkg, ok := types.PkgMap()[pkgPath]
    51  	if !ok {
    52  		return nil, fmt.Errorf("pkg %s doesn't exist in %v", pkgPath, types.PkgMap())
    53  	}
    54  
    55  	// Symbol naming is ambiguous. We can't necessarily distinguish between
    56  	// a method and a closure. e.g., is foo.Bar.func1 a closure defined in
    57  	// function Bar, or a method on type Bar? Thus we must simply attempt
    58  	// to lookup both.
    59  
    60  	fn, err := lookupFunction(pkg, symName)
    61  	if err == nil {
    62  		return fn, nil
    63  	}
    64  
    65  	fn, mErr := lookupMethod(pkg, symName)
    66  	if mErr == nil {
    67  		return fn, nil
    68  	}
    69  
    70  	return nil, fmt.Errorf("%s is not a function (%v) or method (%v)", fullName, err, mErr)
    71  }
    72  
    73  // PostLookupCleanup performs cleanup operations needed
    74  // after a series of calls to LookupFunc, specifically invoking
    75  // readBodies to post-process any funcs on the "todoBodies" list
    76  // that were added as a result of the lookup operations.
    77  func PostLookupCleanup() {
    78  	readBodies(typecheck.Target, false)
    79  }
    80  
    81  func lookupFunction(pkg *types.Pkg, symName string) (*ir.Func, error) {
    82  	sym := pkg.Lookup(symName)
    83  
    84  	// TODO(prattmic): Enclosed functions (e.g., foo.Bar.func1) are not
    85  	// present in objReader, only as OCLOSURE nodes in the enclosing
    86  	// function.
    87  	pri, ok := objReader[sym]
    88  	if !ok {
    89  		return nil, fmt.Errorf("func sym %v missing objReader", sym)
    90  	}
    91  
    92  	node, err := pri.pr.objIdxMayFail(pri.idx, nil, nil, false)
    93  	if err != nil {
    94  		return nil, fmt.Errorf("func sym %v lookup error: %w", sym, err)
    95  	}
    96  	name := node.(*ir.Name)
    97  	if name.Op() != ir.ONAME || name.Class != ir.PFUNC {
    98  		return nil, fmt.Errorf("func sym %v refers to non-function name: %v", sym, name)
    99  	}
   100  	return name.Func, nil
   101  }
   102  
   103  func lookupMethod(pkg *types.Pkg, symName string) (*ir.Func, error) {
   104  	// N.B. readPackage creates a Sym for every object in the package to
   105  	// initialize objReader and importBodyReader, even if the object isn't
   106  	// read.
   107  	//
   108  	// However, objReader is only initialized for top-level objects, so we
   109  	// must first lookup the type and use that to find the method rather
   110  	// than looking for the method directly.
   111  	typ, meth, err := ir.LookupMethodSelector(pkg, symName)
   112  	if err != nil {
   113  		return nil, fmt.Errorf("error looking up method symbol %q: %v", symName, err)
   114  	}
   115  
   116  	pri, ok := objReader[typ]
   117  	if !ok {
   118  		return nil, fmt.Errorf("type sym %v missing objReader", typ)
   119  	}
   120  
   121  	node, err := pri.pr.objIdxMayFail(pri.idx, nil, nil, false)
   122  	if err != nil {
   123  		return nil, fmt.Errorf("func sym %v lookup error: %w", typ, err)
   124  	}
   125  	name := node.(*ir.Name)
   126  	if name.Op() != ir.OTYPE {
   127  		return nil, fmt.Errorf("type sym %v refers to non-type name: %v", typ, name)
   128  	}
   129  	if name.Alias() {
   130  		return nil, fmt.Errorf("type sym %v refers to alias", typ)
   131  	}
   132  	if name.Type().IsInterface() {
   133  		return nil, fmt.Errorf("type sym %v refers to interface type", typ)
   134  	}
   135  
   136  	for _, m := range name.Type().Methods() {
   137  		if m.Sym == meth {
   138  			fn := m.Nname.(*ir.Name).Func
   139  			return fn, nil
   140  		}
   141  	}
   142  
   143  	return nil, fmt.Errorf("method %s missing from method set of %v", symName, typ)
   144  }
   145  
   146  // unified constructs the local package's Internal Representation (IR)
   147  // from its syntax tree (AST).
   148  //
   149  // The pipeline contains 2 steps:
   150  //
   151  //  1. Generate the export data "stub".
   152  //
   153  //  2. Generate the IR from the export data above.
   154  //
   155  // The package data "stub" at step (1) contains everything from the local package,
   156  // but nothing that has been imported. When we're actually writing out export data
   157  // to the output files (see writeNewExport), we run the "linker", which:
   158  //
   159  //   - Updates compiler extensions data (e.g. inlining cost, escape analysis results).
   160  //
   161  //   - Handles re-exporting any transitive dependencies.
   162  //
   163  //   - Prunes out any unnecessary details (e.g. non-inlineable functions, because any
   164  //     downstream importers only care about inlinable functions).
   165  //
   166  // The source files are typechecked twice: once before writing the export data
   167  // using types2, and again after reading the export data using gc/typecheck.
   168  // The duplication of work will go away once we only use the types2 type checker,
   169  // removing the gc/typecheck step. For now, it is kept because:
   170  //
   171  //   - It reduces the engineering costs in maintaining a fork of typecheck
   172  //     (e.g. no need to backport fixes like CL 327651).
   173  //
   174  //   - It makes it easier to pass toolstash -cmp.
   175  //
   176  //   - Historically, we would always re-run the typechecker after importing a package,
   177  //     even though we know the imported data is valid. It's not ideal, but it's
   178  //     not causing any problems either.
   179  //
   180  //   - gc/typecheck is still in charge of some transformations, such as rewriting
   181  //     multi-valued function calls or transforming ir.OINDEX to ir.OINDEXMAP.
   182  //
   183  // Using the syntax tree with types2, which has a complete representation of generics,
   184  // the unified IR has the full typed AST needed for introspection during step (1).
   185  // In other words, we have all the necessary information to build the generic IR form
   186  // (see writer.captureVars for an example).
   187  func unified(m posMap, noders []*noder) {
   188  	inline.InlineCall = unifiedInlineCall
   189  	typecheck.HaveInlineBody = unifiedHaveInlineBody
   190  	pgoir.LookupFunc = LookupFunc
   191  	pgoir.PostLookupCleanup = PostLookupCleanup
   192  
   193  	data := writePkgStub(m, noders)
   194  
   195  	target := typecheck.Target
   196  
   197  	localPkgReader = newPkgReader(pkgbits.NewPkgDecoder(types.LocalPkg.Path, data))
   198  	readPackage(localPkgReader, types.LocalPkg, true)
   199  
   200  	r := localPkgReader.newReader(pkgbits.RelocMeta, pkgbits.PrivateRootIdx, pkgbits.SyncPrivate)
   201  	r.pkgInit(types.LocalPkg, target)
   202  
   203  	readBodies(target, false)
   204  
   205  	// Check that nothing snuck past typechecking.
   206  	for _, fn := range target.Funcs {
   207  		if fn.Typecheck() == 0 {
   208  			base.FatalfAt(fn.Pos(), "missed typecheck: %v", fn)
   209  		}
   210  
   211  		// For functions, check that at least their first statement (if
   212  		// any) was typechecked too.
   213  		if len(fn.Body) != 0 {
   214  			if stmt := fn.Body[0]; stmt.Typecheck() == 0 {
   215  				base.FatalfAt(stmt.Pos(), "missed typecheck: %v", stmt)
   216  			}
   217  		}
   218  	}
   219  
   220  	// For functions originally came from package runtime,
   221  	// mark as norace to prevent instrumenting, see issue #60439.
   222  	for _, fn := range target.Funcs {
   223  		if !base.Flag.CompilingRuntime && types.RuntimeSymName(fn.Sym()) != "" {
   224  			fn.Pragma |= ir.Norace
   225  		}
   226  	}
   227  
   228  	base.ExitIfErrors() // just in case
   229  }
   230  
   231  // readBodies iteratively expands all pending dictionaries and
   232  // function bodies.
   233  //
   234  // If duringInlining is true, then the inline.InlineDecls is called as
   235  // necessary on instantiations of imported generic functions, so their
   236  // inlining costs can be computed.
   237  func readBodies(target *ir.Package, duringInlining bool) {
   238  	var inlDecls []*ir.Func
   239  
   240  	// Don't use range--bodyIdx can add closures to todoBodies.
   241  	for {
   242  		// The order we expand dictionaries and bodies doesn't matter, so
   243  		// pop from the end to reduce todoBodies reallocations if it grows
   244  		// further.
   245  		//
   246  		// However, we do at least need to flush any pending dictionaries
   247  		// before reading bodies, because bodies might reference the
   248  		// dictionaries.
   249  
   250  		if len(todoDicts) > 0 {
   251  			fn := todoDicts[len(todoDicts)-1]
   252  			todoDicts = todoDicts[:len(todoDicts)-1]
   253  			fn()
   254  			continue
   255  		}
   256  
   257  		if len(todoBodies) > 0 {
   258  			fn := todoBodies[len(todoBodies)-1]
   259  			todoBodies = todoBodies[:len(todoBodies)-1]
   260  
   261  			pri, ok := bodyReader[fn]
   262  			assert(ok)
   263  			pri.funcBody(fn)
   264  
   265  			// Instantiated generic function: add to Decls for typechecking
   266  			// and compilation.
   267  			if fn.OClosure == nil && len(pri.dict.targs) != 0 {
   268  				// cmd/link does not support a type symbol referencing a method symbol
   269  				// across DSO boundary, so force re-compiling methods on a generic type
   270  				// even it was seen from imported package in linkshared mode, see #58966.
   271  				canSkipNonGenericMethod := !(base.Ctxt.Flag_linkshared && ir.IsMethod(fn))
   272  				if duringInlining && canSkipNonGenericMethod {
   273  					inlDecls = append(inlDecls, fn)
   274  				} else {
   275  					target.Funcs = append(target.Funcs, fn)
   276  				}
   277  			}
   278  
   279  			continue
   280  		}
   281  
   282  		break
   283  	}
   284  
   285  	todoDicts = nil
   286  	todoBodies = nil
   287  
   288  	if len(inlDecls) != 0 {
   289  		// If we instantiated any generic functions during inlining, we need
   290  		// to call CanInline on them so they'll be transitively inlined
   291  		// correctly (#56280).
   292  		//
   293  		// We know these functions were already compiled in an imported
   294  		// package though, so we don't need to actually apply InlineCalls or
   295  		// save the function bodies any further than this.
   296  		//
   297  		// We can also lower the -m flag to 0, to suppress duplicate "can
   298  		// inline" diagnostics reported against the imported package. Again,
   299  		// we already reported those diagnostics in the original package, so
   300  		// it's pointless repeating them here.
   301  
   302  		oldLowerM := base.Flag.LowerM
   303  		base.Flag.LowerM = 0
   304  		inline.CanInlineFuncs(inlDecls, nil)
   305  		base.Flag.LowerM = oldLowerM
   306  
   307  		for _, fn := range inlDecls {
   308  			fn.Body = nil // free memory
   309  		}
   310  	}
   311  }
   312  
   313  // writePkgStub type checks the given parsed source files,
   314  // writes an export data package stub representing them,
   315  // and returns the result.
   316  func writePkgStub(m posMap, noders []*noder) string {
   317  	pkg, info, otherInfo := checkFiles(m, noders)
   318  
   319  	pw := newPkgWriter(m, pkg, info, otherInfo)
   320  
   321  	pw.collectDecls(noders)
   322  
   323  	publicRootWriter := pw.newWriter(pkgbits.RelocMeta, pkgbits.SyncPublic)
   324  	privateRootWriter := pw.newWriter(pkgbits.RelocMeta, pkgbits.SyncPrivate)
   325  
   326  	assert(publicRootWriter.Idx == pkgbits.PublicRootIdx)
   327  	assert(privateRootWriter.Idx == pkgbits.PrivateRootIdx)
   328  
   329  	{
   330  		w := publicRootWriter
   331  		w.pkg(pkg)
   332  		w.Bool(false) // TODO(mdempsky): Remove; was "has init"
   333  
   334  		scope := pkg.Scope()
   335  		names := scope.Names()
   336  		w.Len(len(names))
   337  		for _, name := range names {
   338  			w.obj(scope.Lookup(name), nil)
   339  		}
   340  
   341  		w.Sync(pkgbits.SyncEOF)
   342  		w.Flush()
   343  	}
   344  
   345  	{
   346  		w := privateRootWriter
   347  		w.pkgInit(noders)
   348  		w.Flush()
   349  	}
   350  
   351  	var sb strings.Builder
   352  	pw.DumpTo(&sb)
   353  
   354  	// At this point, we're done with types2. Make sure the package is
   355  	// garbage collected.
   356  	freePackage(pkg)
   357  
   358  	return sb.String()
   359  }
   360  
   361  // freePackage ensures the given package is garbage collected.
   362  func freePackage(pkg *types2.Package) {
   363  	// The GC test below relies on a precise GC that runs finalizers as
   364  	// soon as objects are unreachable. Our implementation provides
   365  	// this, but other/older implementations may not (e.g., Go 1.4 does
   366  	// not because of #22350). To avoid imposing unnecessary
   367  	// restrictions on the GOROOT_BOOTSTRAP toolchain, we skip the test
   368  	// during bootstrapping.
   369  	if base.CompilerBootstrap || base.Debug.GCCheck == 0 {
   370  		*pkg = types2.Package{}
   371  		return
   372  	}
   373  
   374  	// Set a finalizer on pkg so we can detect if/when it's collected.
   375  	done := make(chan struct{})
   376  	runtime.SetFinalizer(pkg, func(*types2.Package) { close(done) })
   377  
   378  	// Important: objects involved in cycles are not finalized, so zero
   379  	// out pkg to break its cycles and allow the finalizer to run.
   380  	*pkg = types2.Package{}
   381  
   382  	// It typically takes just 1 or 2 cycles to release pkg, but it
   383  	// doesn't hurt to try a few more times.
   384  	for i := 0; i < 10; i++ {
   385  		select {
   386  		case <-done:
   387  			return
   388  		default:
   389  			runtime.GC()
   390  		}
   391  	}
   392  
   393  	base.Fatalf("package never finalized")
   394  }
   395  
   396  // readPackage reads package export data from pr to populate
   397  // importpkg.
   398  //
   399  // localStub indicates whether pr is reading the stub export data for
   400  // the local package, as opposed to relocated export data for an
   401  // import.
   402  func readPackage(pr *pkgReader, importpkg *types.Pkg, localStub bool) {
   403  	{
   404  		r := pr.newReader(pkgbits.RelocMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic)
   405  
   406  		pkg := r.pkg()
   407  		if pkg != importpkg {
   408  			base.ErrorfAt(base.AutogeneratedPos, errors.BadImportPath, "mismatched import path, have %q (%p), want %q (%p)", pkg.Path, pkg, importpkg.Path, importpkg)
   409  			base.ErrorExit()
   410  		}
   411  
   412  		r.Bool() // TODO(mdempsky): Remove; was "has init"
   413  
   414  		for i, n := 0, r.Len(); i < n; i++ {
   415  			r.Sync(pkgbits.SyncObject)
   416  			assert(!r.Bool())
   417  			idx := r.Reloc(pkgbits.RelocObj)
   418  			assert(r.Len() == 0)
   419  
   420  			path, name, code := r.p.PeekObj(idx)
   421  			if code != pkgbits.ObjStub {
   422  				objReader[types.NewPkg(path, "").Lookup(name)] = pkgReaderIndex{pr, idx, nil, nil, nil}
   423  			}
   424  		}
   425  
   426  		r.Sync(pkgbits.SyncEOF)
   427  	}
   428  
   429  	if !localStub {
   430  		r := pr.newReader(pkgbits.RelocMeta, pkgbits.PrivateRootIdx, pkgbits.SyncPrivate)
   431  
   432  		if r.Bool() {
   433  			sym := importpkg.Lookup(".inittask")
   434  			task := ir.NewNameAt(src.NoXPos, sym, nil)
   435  			task.Class = ir.PEXTERN
   436  			sym.Def = task
   437  		}
   438  
   439  		for i, n := 0, r.Len(); i < n; i++ {
   440  			path := r.String()
   441  			name := r.String()
   442  			idx := r.Reloc(pkgbits.RelocBody)
   443  
   444  			sym := types.NewPkg(path, "").Lookup(name)
   445  			if _, ok := importBodyReader[sym]; !ok {
   446  				importBodyReader[sym] = pkgReaderIndex{pr, idx, nil, nil, nil}
   447  			}
   448  		}
   449  
   450  		r.Sync(pkgbits.SyncEOF)
   451  	}
   452  }
   453  
   454  // writeUnifiedExport writes to `out` the finalized, self-contained
   455  // Unified IR export data file for the current compilation unit.
   456  func writeUnifiedExport(out io.Writer) {
   457  	l := linker{
   458  		pw: pkgbits.NewPkgEncoder(base.Debug.SyncFrames),
   459  
   460  		pkgs:   make(map[string]pkgbits.Index),
   461  		decls:  make(map[*types.Sym]pkgbits.Index),
   462  		bodies: make(map[*types.Sym]pkgbits.Index),
   463  	}
   464  
   465  	publicRootWriter := l.pw.NewEncoder(pkgbits.RelocMeta, pkgbits.SyncPublic)
   466  	privateRootWriter := l.pw.NewEncoder(pkgbits.RelocMeta, pkgbits.SyncPrivate)
   467  	assert(publicRootWriter.Idx == pkgbits.PublicRootIdx)
   468  	assert(privateRootWriter.Idx == pkgbits.PrivateRootIdx)
   469  
   470  	var selfPkgIdx pkgbits.Index
   471  
   472  	{
   473  		pr := localPkgReader
   474  		r := pr.NewDecoder(pkgbits.RelocMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic)
   475  
   476  		r.Sync(pkgbits.SyncPkg)
   477  		selfPkgIdx = l.relocIdx(pr, pkgbits.RelocPkg, r.Reloc(pkgbits.RelocPkg))
   478  
   479  		r.Bool() // TODO(mdempsky): Remove; was "has init"
   480  
   481  		for i, n := 0, r.Len(); i < n; i++ {
   482  			r.Sync(pkgbits.SyncObject)
   483  			assert(!r.Bool())
   484  			idx := r.Reloc(pkgbits.RelocObj)
   485  			assert(r.Len() == 0)
   486  
   487  			xpath, xname, xtag := pr.PeekObj(idx)
   488  			assert(xpath == pr.PkgPath())
   489  			assert(xtag != pkgbits.ObjStub)
   490  
   491  			if types.IsExported(xname) {
   492  				l.relocIdx(pr, pkgbits.RelocObj, idx)
   493  			}
   494  		}
   495  
   496  		r.Sync(pkgbits.SyncEOF)
   497  	}
   498  
   499  	{
   500  		var idxs []pkgbits.Index
   501  		for _, idx := range l.decls {
   502  			idxs = append(idxs, idx)
   503  		}
   504  		sort.Slice(idxs, func(i, j int) bool { return idxs[i] < idxs[j] })
   505  
   506  		w := publicRootWriter
   507  
   508  		w.Sync(pkgbits.SyncPkg)
   509  		w.Reloc(pkgbits.RelocPkg, selfPkgIdx)
   510  		w.Bool(false) // TODO(mdempsky): Remove; was "has init"
   511  
   512  		w.Len(len(idxs))
   513  		for _, idx := range idxs {
   514  			w.Sync(pkgbits.SyncObject)
   515  			w.Bool(false)
   516  			w.Reloc(pkgbits.RelocObj, idx)
   517  			w.Len(0)
   518  		}
   519  
   520  		w.Sync(pkgbits.SyncEOF)
   521  		w.Flush()
   522  	}
   523  
   524  	{
   525  		type symIdx struct {
   526  			sym *types.Sym
   527  			idx pkgbits.Index
   528  		}
   529  		var bodies []symIdx
   530  		for sym, idx := range l.bodies {
   531  			bodies = append(bodies, symIdx{sym, idx})
   532  		}
   533  		sort.Slice(bodies, func(i, j int) bool { return bodies[i].idx < bodies[j].idx })
   534  
   535  		w := privateRootWriter
   536  
   537  		w.Bool(typecheck.Lookup(".inittask").Def != nil)
   538  
   539  		w.Len(len(bodies))
   540  		for _, body := range bodies {
   541  			w.String(body.sym.Pkg.Path)
   542  			w.String(body.sym.Name)
   543  			w.Reloc(pkgbits.RelocBody, body.idx)
   544  		}
   545  
   546  		w.Sync(pkgbits.SyncEOF)
   547  		w.Flush()
   548  	}
   549  
   550  	base.Ctxt.Fingerprint = l.pw.DumpTo(out)
   551  }
   552
View as plain text