exportdata.go

     1  // Copyright 2024 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package exportdata implements common utilities for finding
     6  // and reading gc-generated object files.
     7  package exportdata
     8  
     9  // This file should be kept in sync with src/cmd/compile/internal/gc/obj.go .
    10  
    11  import (
    12  	"bufio"
    13  	"bytes"
    14  	"errors"
    15  	"fmt"
    16  	"go/build"
    17  	"internal/saferio"
    18  	"io"
    19  	"os"
    20  	"os/exec"
    21  	"path/filepath"
    22  	"strings"
    23  	"sync"
    24  )
    25  
    26  // ReadUnified reads the contents of the unified export data from a reader r
    27  // that contains the contents of a GC-created archive file.
    28  //
    29  // On success, the reader will be positioned after the end-of-section marker "\n$$\n".
    30  //
    31  // Supported GC-created archive files have 4 layers of nesting:
    32  //   - An archive file containing a package definition file.
    33  //   - The package definition file contains headers followed by a data section.
    34  //     Headers are lines (≤ 4kb) that do not start with "$$".
    35  //   - The data section starts with "$$B\n" followed by export data followed
    36  //     by an end of section marker "\n$$\n". (The section start "$$\n" is no
    37  //     longer supported.)
    38  //   - The export data starts with a format byte ('u') followed by the <data> in
    39  //     the given format. (See ReadExportDataHeader for older formats.)
    40  //
    41  // Putting this together, the bytes in a GC-created archive files are expected
    42  // to look like the following.
    43  // See cmd/internal/archive for more details on ar file headers.
    44  //
    45  // | <!arch>\n             | ar file signature
    46  // | __.PKGDEF...size...\n | ar header for __.PKGDEF including size.
    47  // | go object <...>\n     | objabi header
    48  // | <optional headers>\n  | other headers such as build id
    49  // | $$B\n                 | binary format marker
    50  // | u<data>\n             | unified export <data>
    51  // | $$\n                  | end-of-section marker
    52  // | [optional padding]    | padding byte (0x0A) if size is odd
    53  // | [ar file header]      | other ar files
    54  // | [ar file data]        |
    55  func ReadUnified(r *bufio.Reader) (data []byte, err error) {
    56  	// We historically guaranteed headers at the default buffer size (4096) work.
    57  	// This ensures we can use ReadSlice throughout.
    58  	const minBufferSize = 4096
    59  	r = bufio.NewReaderSize(r, minBufferSize)
    60  
    61  	size, err := FindPackageDefinition(r)
    62  	if err != nil {
    63  		return
    64  	}
    65  	n := size
    66  
    67  	objapi, headers, err := ReadObjectHeaders(r)
    68  	if err != nil {
    69  		return
    70  	}
    71  	n -= len(objapi)
    72  	for _, h := range headers {
    73  		n -= len(h)
    74  	}
    75  
    76  	hdrlen, err := ReadExportDataHeader(r)
    77  	if err != nil {
    78  		return
    79  	}
    80  	n -= hdrlen
    81  
    82  	// size also includes the end of section marker. Remove that many bytes from the end.
    83  	const marker = "\n$$\n"
    84  	n -= len(marker)
    85  
    86  	if n < 0 {
    87  		err = fmt.Errorf("invalid size (%d) in the archive file: %d bytes remain without section headers (recompile package)", size, n)
    88  		return
    89  	}
    90  
    91  	// Read n bytes from buf.
    92  	data, err = saferio.ReadData(r, uint64(n))
    93  	if err != nil {
    94  		return
    95  	}
    96  
    97  	// Check for marker at the end.
    98  	var suffix [len(marker)]byte
    99  	_, err = io.ReadFull(r, suffix[:])
   100  	if err != nil {
   101  		return
   102  	}
   103  	if s := string(suffix[:]); s != marker {
   104  		err = fmt.Errorf("read %q instead of end-of-section marker (%q)", s, marker)
   105  		return
   106  	}
   107  
   108  	return
   109  }
   110  
   111  // FindPackageDefinition positions the reader r at the beginning of a package
   112  // definition file ("__.PKGDEF") within a GC-created archive by reading
   113  // from it, and returns the size of the package definition file in the archive.
   114  //
   115  // The reader must be positioned at the start of the archive file before calling
   116  // this function, and "__.PKGDEF" is assumed to be the first file in the archive.
   117  //
   118  // See cmd/internal/archive for details on the archive format.
   119  func FindPackageDefinition(r *bufio.Reader) (size int, err error) {
   120  	// Uses ReadSlice to limit risk of malformed inputs.
   121  
   122  	// Read first line to make sure this is an object file.
   123  	line, err := r.ReadSlice('\n')
   124  	if err != nil {
   125  		err = fmt.Errorf("can't find export data (%v)", err)
   126  		return
   127  	}
   128  
   129  	// Is the first line an archive file signature?
   130  	if string(line) != "!<arch>\n" {
   131  		err = fmt.Errorf("not the start of an archive file (%q)", line)
   132  		return
   133  	}
   134  
   135  	// package export block should be first
   136  	size = readArchiveHeader(r, "__.PKGDEF")
   137  	if size <= 0 {
   138  		err = fmt.Errorf("not a package file")
   139  		return
   140  	}
   141  
   142  	return
   143  }
   144  
   145  // ReadObjectHeaders reads object headers from the reader. Object headers are
   146  // lines that do not start with an end-of-section marker "$$". The first header
   147  // is the objabi header. On success, the reader will be positioned at the beginning
   148  // of the end-of-section marker.
   149  //
   150  // It returns an error if any header does not fit in r.Size() bytes.
   151  func ReadObjectHeaders(r *bufio.Reader) (objapi string, headers []string, err error) {
   152  	// line is a temporary buffer for headers.
   153  	// Use bounded reads (ReadSlice, Peek) to limit risk of malformed inputs.
   154  	var line []byte
   155  
   156  	// objapi header should be the first line
   157  	if line, err = r.ReadSlice('\n'); err != nil {
   158  		err = fmt.Errorf("can't find export data (%v)", err)
   159  		return
   160  	}
   161  	objapi = string(line)
   162  
   163  	// objapi header begins with "go object ".
   164  	if !strings.HasPrefix(objapi, "go object ") {
   165  		err = fmt.Errorf("not a go object file: %s", objapi)
   166  		return
   167  	}
   168  
   169  	// process remaining object header lines
   170  	for {
   171  		// check for an end of section marker "$$"
   172  		line, err = r.Peek(2)
   173  		if err != nil {
   174  			return
   175  		}
   176  		if string(line) == "$$" {
   177  			return // stop
   178  		}
   179  
   180  		// read next header
   181  		line, err = r.ReadSlice('\n')
   182  		if err != nil {
   183  			return
   184  		}
   185  		headers = append(headers, string(line))
   186  	}
   187  }
   188  
   189  // ReadExportDataHeader reads the export data header and format from r.
   190  // It returns the number of bytes read, or an error if the format is no longer
   191  // supported or it failed to read.
   192  //
   193  // The only currently supported format is binary export data in the
   194  // unified export format.
   195  func ReadExportDataHeader(r *bufio.Reader) (n int, err error) {
   196  	// Read export data header.
   197  	line, err := r.ReadSlice('\n')
   198  	if err != nil {
   199  		return
   200  	}
   201  
   202  	hdr := string(line)
   203  	switch hdr {
   204  	case "$$\n":
   205  		err = fmt.Errorf("old textual export format no longer supported (recompile package)")
   206  		return
   207  
   208  	case "$$B\n":
   209  		var format byte
   210  		format, err = r.ReadByte()
   211  		if err != nil {
   212  			return
   213  		}
   214  		// The unified export format starts with a 'u'.
   215  		switch format {
   216  		case 'u':
   217  		default:
   218  			// Older no longer supported export formats include:
   219  			// indexed export format which started with an 'i'; and
   220  			// the older binary export format which started with a 'c',
   221  			// 'd', or 'v' (from "version").
   222  			err = fmt.Errorf("binary export format %q is no longer supported (recompile package)", format)
   223  			return
   224  		}
   225  
   226  	default:
   227  		err = fmt.Errorf("unknown export data header: %q", hdr)
   228  		return
   229  	}
   230  
   231  	n = len(hdr) + 1 // + 1 is for 'u'
   232  	return
   233  }
   234  
   235  // FindPkg returns the filename and unique package id for an import
   236  // path based on package information provided by build.Import (using
   237  // the build.Default build.Context). A relative srcDir is interpreted
   238  // relative to the current working directory.
   239  func FindPkg(path, srcDir string) (filename, id string, err error) {
   240  	if path == "" {
   241  		return "", "", errors.New("path is empty")
   242  	}
   243  
   244  	var noext string
   245  	switch {
   246  	default:
   247  		// "x" -> "$GOPATH/pkg/$GOOS_$GOARCH/x.ext", "x"
   248  		// Don't require the source files to be present.
   249  		if abs, err := filepath.Abs(srcDir); err == nil { // see issue 14282
   250  			srcDir = abs
   251  		}
   252  		var bp *build.Package
   253  		bp, err = build.Import(path, srcDir, build.FindOnly|build.AllowBinary)
   254  		if bp.PkgObj == "" {
   255  			if bp.Goroot && bp.Dir != "" {
   256  				filename, err = lookupGorootExport(bp.Dir)
   257  				if err == nil {
   258  					_, err = os.Stat(filename)
   259  				}
   260  				if err == nil {
   261  					return filename, bp.ImportPath, nil
   262  				}
   263  			}
   264  			goto notfound
   265  		} else {
   266  			noext = strings.TrimSuffix(bp.PkgObj, ".a")
   267  		}
   268  		id = bp.ImportPath
   269  
   270  	case build.IsLocalImport(path):
   271  		// "./x" -> "/this/directory/x.ext", "/this/directory/x"
   272  		noext = filepath.Join(srcDir, path)
   273  		id = noext
   274  
   275  	case filepath.IsAbs(path):
   276  		// for completeness only - go/build.Import
   277  		// does not support absolute imports
   278  		// "/x" -> "/x.ext", "/x"
   279  		noext = path
   280  		id = path
   281  	}
   282  
   283  	if false { // for debugging
   284  		if path != id {
   285  			fmt.Printf("%s -> %s\n", path, id)
   286  		}
   287  	}
   288  
   289  	// try extensions
   290  	for _, ext := range pkgExts {
   291  		filename = noext + ext
   292  		f, statErr := os.Stat(filename)
   293  		if statErr == nil && !f.IsDir() {
   294  			return filename, id, nil
   295  		}
   296  		if err == nil {
   297  			err = statErr
   298  		}
   299  	}
   300  
   301  notfound:
   302  	if err == nil {
   303  		return "", path, fmt.Errorf("can't find import: %q", path)
   304  	}
   305  	return "", path, fmt.Errorf("can't find import: %q: %w", path, err)
   306  }
   307  
   308  var pkgExts = [...]string{".a", ".o"} // a file from the build cache will have no extension
   309  
   310  var exportMap sync.Map // package dir → func() (string, error)
   311  
   312  // lookupGorootExport returns the location of the export data
   313  // (normally found in the build cache, but located in GOROOT/pkg
   314  // in prior Go releases) for the package located in pkgDir.
   315  //
   316  // (We use the package's directory instead of its import path
   317  // mainly to simplify handling of the packages in src/vendor
   318  // and cmd/vendor.)
   319  func lookupGorootExport(pkgDir string) (string, error) {
   320  	f, ok := exportMap.Load(pkgDir)
   321  	if !ok {
   322  		var (
   323  			listOnce   sync.Once
   324  			exportPath string
   325  			err        error
   326  		)
   327  		f, _ = exportMap.LoadOrStore(pkgDir, func() (string, error) {
   328  			listOnce.Do(func() {
   329  				cmd := exec.Command(filepath.Join(build.Default.GOROOT, "bin", "go"), "list", "-export", "-f", "{{.Export}}", pkgDir)
   330  				cmd.Dir = build.Default.GOROOT
   331  				cmd.Env = append(os.Environ(), "PWD="+cmd.Dir, "GOROOT="+build.Default.GOROOT)
   332  				var output []byte
   333  				output, err = cmd.Output()
   334  				if err != nil {
   335  					if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
   336  						err = errors.New(string(ee.Stderr))
   337  					}
   338  					return
   339  				}
   340  
   341  				exports := strings.Split(string(bytes.TrimSpace(output)), "\n")
   342  				if len(exports) != 1 {
   343  					err = fmt.Errorf("go list reported %d exports; expected 1", len(exports))
   344  					return
   345  				}
   346  
   347  				exportPath = exports[0]
   348  			})
   349  
   350  			return exportPath, err
   351  		})
   352  	}
   353  
   354  	return f.(func() (string, error))()
   355  }
   356
View as plain text