fetch.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modfetch
     6  
     7  import (
     8  	"archive/zip"
     9  	"bytes"
    10  	"context"
    11  	"crypto/sha256"
    12  	"encoding/base64"
    13  	"errors"
    14  	"fmt"
    15  	"io"
    16  	"io/fs"
    17  	"os"
    18  	"path/filepath"
    19  	"sort"
    20  	"strings"
    21  	"sync"
    22  
    23  	"cmd/go/internal/base"
    24  	"cmd/go/internal/cfg"
    25  	"cmd/go/internal/fsys"
    26  	"cmd/go/internal/gover"
    27  	"cmd/go/internal/lockedfile"
    28  	"cmd/go/internal/str"
    29  	"cmd/go/internal/trace"
    30  	"cmd/internal/par"
    31  	"cmd/internal/robustio"
    32  
    33  	"golang.org/x/mod/module"
    34  	"golang.org/x/mod/sumdb/dirhash"
    35  	modzip "golang.org/x/mod/zip"
    36  )
    37  
    38  var downloadCache par.ErrCache[module.Version, string] // version → directory
    39  
    40  var ErrToolchain = errors.New("internal error: invalid operation on toolchain module")
    41  
    42  // Download downloads the specific module version to the
    43  // local download cache and returns the name of the directory
    44  // corresponding to the root of the module's file tree.
    45  func Download(ctx context.Context, mod module.Version) (dir string, err error) {
    46  	if gover.IsToolchain(mod.Path) {
    47  		return "", ErrToolchain
    48  	}
    49  	if err := checkCacheDir(ctx); err != nil {
    50  		base.Fatal(err)
    51  	}
    52  
    53  	// The par.Cache here avoids duplicate work.
    54  	return downloadCache.Do(mod, func() (string, error) {
    55  		dir, err := download(ctx, mod)
    56  		if err != nil {
    57  			return "", err
    58  		}
    59  		checkMod(ctx, mod)
    60  
    61  		// If go.mod exists (not an old legacy module), check version is not too new.
    62  		if data, err := os.ReadFile(filepath.Join(dir, "go.mod")); err == nil {
    63  			goVersion := gover.GoModLookup(data, "go")
    64  			if gover.Compare(goVersion, gover.Local()) > 0 {
    65  				return "", &gover.TooNewError{What: mod.String(), GoVersion: goVersion}
    66  			}
    67  		} else if !errors.Is(err, fs.ErrNotExist) {
    68  			return "", err
    69  		}
    70  
    71  		return dir, nil
    72  	})
    73  }
    74  
    75  func download(ctx context.Context, mod module.Version) (dir string, err error) {
    76  	ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String())
    77  	defer span.Done()
    78  
    79  	dir, err = DownloadDir(ctx, mod)
    80  	if err == nil {
    81  		// The directory has already been completely extracted (no .partial file exists).
    82  		return dir, nil
    83  	} else if dir == "" || !errors.Is(err, fs.ErrNotExist) {
    84  		return "", err
    85  	}
    86  
    87  	// To avoid cluttering the cache with extraneous files,
    88  	// DownloadZip uses the same lockfile as Download.
    89  	// Invoke DownloadZip before locking the file.
    90  	zipfile, err := DownloadZip(ctx, mod)
    91  	if err != nil {
    92  		return "", err
    93  	}
    94  
    95  	unlock, err := lockVersion(ctx, mod)
    96  	if err != nil {
    97  		return "", err
    98  	}
    99  	defer unlock()
   100  
   101  	ctx, span = trace.StartSpan(ctx, "unzip "+zipfile)
   102  	defer span.Done()
   103  
   104  	// Check whether the directory was populated while we were waiting on the lock.
   105  	_, dirErr := DownloadDir(ctx, mod)
   106  	if dirErr == nil {
   107  		return dir, nil
   108  	}
   109  	_, dirExists := dirErr.(*DownloadDirPartialError)
   110  
   111  	// Clean up any remaining temporary directories created by old versions
   112  	// (before 1.16), as well as partially extracted directories (indicated by
   113  	// DownloadDirPartialError, usually because of a .partial file). This is only
   114  	// safe to do because the lock file ensures that their writers are no longer
   115  	// active.
   116  	parentDir := filepath.Dir(dir)
   117  	tmpPrefix := filepath.Base(dir) + ".tmp-"
   118  	if old, err := filepath.Glob(filepath.Join(str.QuoteGlob(parentDir), str.QuoteGlob(tmpPrefix)+"*")); err == nil {
   119  		for _, path := range old {
   120  			RemoveAll(path) // best effort
   121  		}
   122  	}
   123  	if dirExists {
   124  		if err := RemoveAll(dir); err != nil {
   125  			return "", err
   126  		}
   127  	}
   128  
   129  	partialPath, err := CachePath(ctx, mod, "partial")
   130  	if err != nil {
   131  		return "", err
   132  	}
   133  
   134  	// Extract the module zip directory at its final location.
   135  	//
   136  	// To prevent other processes from reading the directory if we crash,
   137  	// create a .partial file before extracting the directory, and delete
   138  	// the .partial file afterward (all while holding the lock).
   139  	//
   140  	// Before Go 1.16, we extracted to a temporary directory with a random name
   141  	// then renamed it into place with os.Rename. On Windows, this failed with
   142  	// ERROR_ACCESS_DENIED when another process (usually an anti-virus scanner)
   143  	// opened files in the temporary directory.
   144  	//
   145  	// Go 1.14.2 and higher respect .partial files. Older versions may use
   146  	// partially extracted directories. 'go mod verify' can detect this,
   147  	// and 'go clean -modcache' can fix it.
   148  	if err := os.MkdirAll(parentDir, 0777); err != nil {
   149  		return "", err
   150  	}
   151  	if err := os.WriteFile(partialPath, nil, 0666); err != nil {
   152  		return "", err
   153  	}
   154  	if err := modzip.Unzip(dir, mod, zipfile); err != nil {
   155  		fmt.Fprintf(os.Stderr, "-> %s\n", err)
   156  		if rmErr := RemoveAll(dir); rmErr == nil {
   157  			os.Remove(partialPath)
   158  		}
   159  		return "", err
   160  	}
   161  	if err := os.Remove(partialPath); err != nil {
   162  		return "", err
   163  	}
   164  
   165  	if !cfg.ModCacheRW {
   166  		makeDirsReadOnly(dir)
   167  	}
   168  	return dir, nil
   169  }
   170  
   171  var downloadZipCache par.ErrCache[module.Version, string]
   172  
   173  // DownloadZip downloads the specific module version to the
   174  // local zip cache and returns the name of the zip file.
   175  func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) {
   176  	// The par.Cache here avoids duplicate work.
   177  	return downloadZipCache.Do(mod, func() (string, error) {
   178  		zipfile, err := CachePath(ctx, mod, "zip")
   179  		if err != nil {
   180  			return "", err
   181  		}
   182  		ziphashfile := zipfile + "hash"
   183  
   184  		// Return without locking if the zip and ziphash files exist.
   185  		if _, err := os.Stat(zipfile); err == nil {
   186  			if _, err := os.Stat(ziphashfile); err == nil {
   187  				return zipfile, nil
   188  			}
   189  		}
   190  
   191  		// The zip or ziphash file does not exist. Acquire the lock and create them.
   192  		if cfg.CmdName != "mod download" {
   193  			vers := mod.Version
   194  			if mod.Path == "golang.org/toolchain" {
   195  				// Shorten v0.0.1-go1.13.1.darwin-amd64 to go1.13.1.darwin-amd64
   196  				_, vers, _ = strings.Cut(vers, "-")
   197  				if i := strings.LastIndex(vers, "."); i >= 0 {
   198  					goos, goarch, _ := strings.Cut(vers[i+1:], "-")
   199  					vers = vers[:i] + " (" + goos + "/" + goarch + ")"
   200  				}
   201  				fmt.Fprintf(os.Stderr, "go: downloading %s\n", vers)
   202  			} else {
   203  				fmt.Fprintf(os.Stderr, "go: downloading %s %s\n", mod.Path, vers)
   204  			}
   205  		}
   206  		unlock, err := lockVersion(ctx, mod)
   207  		if err != nil {
   208  			return "", err
   209  		}
   210  		defer unlock()
   211  
   212  		if err := downloadZip(ctx, mod, zipfile); err != nil {
   213  			return "", err
   214  		}
   215  		return zipfile, nil
   216  	})
   217  }
   218  
   219  func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) {
   220  	ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile)
   221  	defer span.Done()
   222  
   223  	// Double-check that the zipfile was not created while we were waiting for
   224  	// the lock in DownloadZip.
   225  	ziphashfile := zipfile + "hash"
   226  	var zipExists, ziphashExists bool
   227  	if _, err := os.Stat(zipfile); err == nil {
   228  		zipExists = true
   229  	}
   230  	if _, err := os.Stat(ziphashfile); err == nil {
   231  		ziphashExists = true
   232  	}
   233  	if zipExists && ziphashExists {
   234  		return nil
   235  	}
   236  
   237  	// Create parent directories.
   238  	if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
   239  		return err
   240  	}
   241  
   242  	// Clean up any remaining tempfiles from previous runs.
   243  	// This is only safe to do because the lock file ensures that their
   244  	// writers are no longer active.
   245  	tmpPattern := filepath.Base(zipfile) + "*.tmp"
   246  	if old, err := filepath.Glob(filepath.Join(str.QuoteGlob(filepath.Dir(zipfile)), tmpPattern)); err == nil {
   247  		for _, path := range old {
   248  			os.Remove(path) // best effort
   249  		}
   250  	}
   251  
   252  	// If the zip file exists, the ziphash file must have been deleted
   253  	// or lost after a file system crash. Re-hash the zip without downloading.
   254  	if zipExists {
   255  		return hashZip(mod, zipfile, ziphashfile)
   256  	}
   257  
   258  	// From here to the os.Rename call below is functionally almost equivalent to
   259  	// renameio.WriteToFile, with one key difference: we want to validate the
   260  	// contents of the file (by hashing it) before we commit it. Because the file
   261  	// is zip-compressed, we need an actual file — or at least an io.ReaderAt — to
   262  	// validate it: we can't just tee the stream as we write it.
   263  	f, err := tempFile(ctx, filepath.Dir(zipfile), filepath.Base(zipfile), 0666)
   264  	if err != nil {
   265  		return err
   266  	}
   267  	defer func() {
   268  		if err != nil {
   269  			f.Close()
   270  			os.Remove(f.Name())
   271  		}
   272  	}()
   273  
   274  	var unrecoverableErr error
   275  	err = TryProxies(func(proxy string) error {
   276  		if unrecoverableErr != nil {
   277  			return unrecoverableErr
   278  		}
   279  		repo := Lookup(ctx, proxy, mod.Path)
   280  		err := repo.Zip(ctx, f, mod.Version)
   281  		if err != nil {
   282  			// Zip may have partially written to f before failing.
   283  			// (Perhaps the server crashed while sending the file?)
   284  			// Since we allow fallback on error in some cases, we need to fix up the
   285  			// file to be empty again for the next attempt.
   286  			if _, err := f.Seek(0, io.SeekStart); err != nil {
   287  				unrecoverableErr = err
   288  				return err
   289  			}
   290  			if err := f.Truncate(0); err != nil {
   291  				unrecoverableErr = err
   292  				return err
   293  			}
   294  		}
   295  		return err
   296  	})
   297  	if err != nil {
   298  		return err
   299  	}
   300  
   301  	// Double-check that the paths within the zip file are well-formed.
   302  	//
   303  	// TODO(bcmills): There is a similar check within the Unzip function. Can we eliminate one?
   304  	fi, err := f.Stat()
   305  	if err != nil {
   306  		return err
   307  	}
   308  	z, err := zip.NewReader(f, fi.Size())
   309  	if err != nil {
   310  		return err
   311  	}
   312  	prefix := mod.Path + "@" + mod.Version + "/"
   313  	for _, f := range z.File {
   314  		if !strings.HasPrefix(f.Name, prefix) {
   315  			return fmt.Errorf("zip for %s has unexpected file %s", prefix[:len(prefix)-1], f.Name)
   316  		}
   317  	}
   318  
   319  	if err := f.Close(); err != nil {
   320  		return err
   321  	}
   322  
   323  	// Hash the zip file and check the sum before renaming to the final location.
   324  	if err := hashZip(mod, f.Name(), ziphashfile); err != nil {
   325  		return err
   326  	}
   327  	if err := os.Rename(f.Name(), zipfile); err != nil {
   328  		return err
   329  	}
   330  
   331  	// TODO(bcmills): Should we make the .zip and .ziphash files read-only to discourage tampering?
   332  
   333  	return nil
   334  }
   335  
   336  // hashZip reads the zip file opened in f, then writes the hash to ziphashfile,
   337  // overwriting that file if it exists.
   338  //
   339  // If the hash does not match go.sum (or the sumdb if enabled), hashZip returns
   340  // an error and does not write ziphashfile.
   341  func hashZip(mod module.Version, zipfile, ziphashfile string) (err error) {
   342  	hash, err := dirhash.HashZip(zipfile, dirhash.DefaultHash)
   343  	if err != nil {
   344  		return err
   345  	}
   346  	if err := checkModSum(mod, hash); err != nil {
   347  		return err
   348  	}
   349  	hf, err := lockedfile.Create(ziphashfile)
   350  	if err != nil {
   351  		return err
   352  	}
   353  	defer func() {
   354  		if closeErr := hf.Close(); err == nil && closeErr != nil {
   355  			err = closeErr
   356  		}
   357  	}()
   358  	if err := hf.Truncate(int64(len(hash))); err != nil {
   359  		return err
   360  	}
   361  	if _, err := hf.WriteAt([]byte(hash), 0); err != nil {
   362  		return err
   363  	}
   364  	return nil
   365  }
   366  
   367  // makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir
   368  // and its transitive contents.
   369  func makeDirsReadOnly(dir string) {
   370  	type pathMode struct {
   371  		path string
   372  		mode fs.FileMode
   373  	}
   374  	var dirs []pathMode // in lexical order
   375  	filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
   376  		if err == nil && d.IsDir() {
   377  			info, err := d.Info()
   378  			if err == nil && info.Mode()&0222 != 0 {
   379  				dirs = append(dirs, pathMode{path, info.Mode()})
   380  			}
   381  		}
   382  		return nil
   383  	})
   384  
   385  	// Run over list backward to chmod children before parents.
   386  	for i := len(dirs) - 1; i >= 0; i-- {
   387  		os.Chmod(dirs[i].path, dirs[i].mode&^0222)
   388  	}
   389  }
   390  
   391  // RemoveAll removes a directory written by Download or Unzip, first applying
   392  // any permission changes needed to do so.
   393  func RemoveAll(dir string) error {
   394  	// Module cache has 0555 directories; make them writable in order to remove content.
   395  	filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error {
   396  		if err != nil {
   397  			return nil // ignore errors walking in file system
   398  		}
   399  		if info.IsDir() {
   400  			os.Chmod(path, 0777)
   401  		}
   402  		return nil
   403  	})
   404  	return robustio.RemoveAll(dir)
   405  }
   406  
   407  var GoSumFile string             // path to go.sum; set by package modload
   408  var WorkspaceGoSumFiles []string // path to module go.sums in workspace; set by package modload
   409  
   410  type modSum struct {
   411  	mod module.Version
   412  	sum string
   413  }
   414  
   415  var goSum struct {
   416  	mu        sync.Mutex
   417  	m         map[module.Version][]string            // content of go.sum file
   418  	w         map[string]map[module.Version][]string // sum file in workspace -> content of that sum file
   419  	status    map[modSum]modSumStatus                // state of sums in m
   420  	overwrite bool                                   // if true, overwrite go.sum without incorporating its contents
   421  	enabled   bool                                   // whether to use go.sum at all
   422  }
   423  
   424  type modSumStatus struct {
   425  	used, dirty bool
   426  }
   427  
   428  // Reset resets globals in the modfetch package, so previous loads don't affect
   429  // contents of go.sum files.
   430  func Reset() {
   431  	GoSumFile = ""
   432  	WorkspaceGoSumFiles = nil
   433  
   434  	// Uses of lookupCache and downloadCache both can call checkModSum,
   435  	// which in turn sets the used bit on goSum.status for modules.
   436  	// Reset them so used can be computed properly.
   437  	lookupCache = par.Cache[lookupCacheKey, Repo]{}
   438  	downloadCache = par.ErrCache[module.Version, string]{}
   439  
   440  	// Clear all fields on goSum. It will be initialized later
   441  	goSum.mu.Lock()
   442  	goSum.m = nil
   443  	goSum.w = nil
   444  	goSum.status = nil
   445  	goSum.overwrite = false
   446  	goSum.enabled = false
   447  	goSum.mu.Unlock()
   448  }
   449  
   450  // initGoSum initializes the go.sum data.
   451  // The boolean it returns reports whether the
   452  // use of go.sum is now enabled.
   453  // The goSum lock must be held.
   454  func initGoSum() (bool, error) {
   455  	if GoSumFile == "" {
   456  		return false, nil
   457  	}
   458  	if goSum.m != nil {
   459  		return true, nil
   460  	}
   461  
   462  	goSum.m = make(map[module.Version][]string)
   463  	goSum.status = make(map[modSum]modSumStatus)
   464  	goSum.w = make(map[string]map[module.Version][]string)
   465  
   466  	for _, f := range WorkspaceGoSumFiles {
   467  		goSum.w[f] = make(map[module.Version][]string)
   468  		_, err := readGoSumFile(goSum.w[f], f)
   469  		if err != nil {
   470  			return false, err
   471  		}
   472  	}
   473  
   474  	enabled, err := readGoSumFile(goSum.m, GoSumFile)
   475  	goSum.enabled = enabled
   476  	return enabled, err
   477  }
   478  
   479  func readGoSumFile(dst map[module.Version][]string, file string) (bool, error) {
   480  	var (
   481  		data []byte
   482  		err  error
   483  	)
   484  	if actualSumFile, ok := fsys.OverlayPath(file); ok {
   485  		// Don't lock go.sum if it's part of the overlay.
   486  		// On Plan 9, locking requires chmod, and we don't want to modify any file
   487  		// in the overlay. See #44700.
   488  		data, err = os.ReadFile(actualSumFile)
   489  	} else {
   490  		data, err = lockedfile.Read(file)
   491  	}
   492  	if err != nil && !os.IsNotExist(err) {
   493  		return false, err
   494  	}
   495  	readGoSum(dst, file, data)
   496  
   497  	return true, nil
   498  }
   499  
   500  // emptyGoModHash is the hash of a 1-file tree containing a 0-length go.mod.
   501  // A bug caused us to write these into go.sum files for non-modules.
   502  // We detect and remove them.
   503  const emptyGoModHash = "h1:G7mAYYxgmS0lVkHyy2hEOLQCFB0DlQFTMLWggykrydY="
   504  
   505  // readGoSum parses data, which is the content of file,
   506  // and adds it to goSum.m. The goSum lock must be held.
   507  func readGoSum(dst map[module.Version][]string, file string, data []byte) {
   508  	lineno := 0
   509  	for len(data) > 0 {
   510  		var line []byte
   511  		lineno++
   512  		i := bytes.IndexByte(data, '\n')
   513  		if i < 0 {
   514  			line, data = data, nil
   515  		} else {
   516  			line, data = data[:i], data[i+1:]
   517  		}
   518  		f := strings.Fields(string(line))
   519  		if len(f) == 0 {
   520  			// blank line; skip it
   521  			continue
   522  		}
   523  		if len(f) != 3 {
   524  			if cfg.CmdName == "mod tidy" {
   525  				// ignore malformed line so that go mod tidy can fix go.sum
   526  				continue
   527  			} else {
   528  				base.Fatalf("malformed go.sum:\n%s:%d: wrong number of fields %v\n", file, lineno, len(f))
   529  			}
   530  		}
   531  		if f[2] == emptyGoModHash {
   532  			// Old bug; drop it.
   533  			continue
   534  		}
   535  		mod := module.Version{Path: f[0], Version: f[1]}
   536  		dst[mod] = append(dst[mod], f[2])
   537  	}
   538  }
   539  
   540  // HaveSum returns true if the go.sum file contains an entry for mod.
   541  // The entry's hash must be generated with a known hash algorithm.
   542  // mod.Version may have a "/go.mod" suffix to distinguish sums for
   543  // .mod and .zip files.
   544  func HaveSum(mod module.Version) bool {
   545  	goSum.mu.Lock()
   546  	defer goSum.mu.Unlock()
   547  	inited, err := initGoSum()
   548  	if err != nil || !inited {
   549  		return false
   550  	}
   551  	for _, goSums := range goSum.w {
   552  		for _, h := range goSums[mod] {
   553  			if !strings.HasPrefix(h, "h1:") {
   554  				continue
   555  			}
   556  			if !goSum.status[modSum{mod, h}].dirty {
   557  				return true
   558  			}
   559  		}
   560  	}
   561  	for _, h := range goSum.m[mod] {
   562  		if !strings.HasPrefix(h, "h1:") {
   563  			continue
   564  		}
   565  		if !goSum.status[modSum{mod, h}].dirty {
   566  			return true
   567  		}
   568  	}
   569  	return false
   570  }
   571  
   572  // RecordedSum returns the sum if the go.sum file contains an entry for mod.
   573  // The boolean reports true if an entry was found or
   574  // false if no entry found or two conflicting sums are found.
   575  // The entry's hash must be generated with a known hash algorithm.
   576  // mod.Version may have a "/go.mod" suffix to distinguish sums for
   577  // .mod and .zip files.
   578  func RecordedSum(mod module.Version) (sum string, ok bool) {
   579  	goSum.mu.Lock()
   580  	defer goSum.mu.Unlock()
   581  	inited, err := initGoSum()
   582  	foundSum := ""
   583  	if err != nil || !inited {
   584  		return "", false
   585  	}
   586  	for _, goSums := range goSum.w {
   587  		for _, h := range goSums[mod] {
   588  			if !strings.HasPrefix(h, "h1:") {
   589  				continue
   590  			}
   591  			if !goSum.status[modSum{mod, h}].dirty {
   592  				if foundSum != "" && foundSum != h { // conflicting sums exist
   593  					return "", false
   594  				}
   595  				foundSum = h
   596  			}
   597  		}
   598  	}
   599  	for _, h := range goSum.m[mod] {
   600  		if !strings.HasPrefix(h, "h1:") {
   601  			continue
   602  		}
   603  		if !goSum.status[modSum{mod, h}].dirty {
   604  			if foundSum != "" && foundSum != h { // conflicting sums exist
   605  				return "", false
   606  			}
   607  			foundSum = h
   608  		}
   609  	}
   610  	return foundSum, true
   611  }
   612  
   613  // checkMod checks the given module's checksum and Go version.
   614  func checkMod(ctx context.Context, mod module.Version) {
   615  	// Do the file I/O before acquiring the go.sum lock.
   616  	ziphash, err := CachePath(ctx, mod, "ziphash")
   617  	if err != nil {
   618  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   619  	}
   620  	data, err := lockedfile.Read(ziphash)
   621  	if err != nil {
   622  		base.Fatalf("verifying %v", module.VersionError(mod, err))
   623  	}
   624  	data = bytes.TrimSpace(data)
   625  	if !isValidSum(data) {
   626  		// Recreate ziphash file from zip file and use that to check the mod sum.
   627  		zip, err := CachePath(ctx, mod, "zip")
   628  		if err != nil {
   629  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   630  		}
   631  		err = hashZip(mod, zip, ziphash)
   632  		if err != nil {
   633  			base.Fatalf("verifying %v", module.VersionError(mod, err))
   634  		}
   635  		return
   636  	}
   637  	h := string(data)
   638  	if !strings.HasPrefix(h, "h1:") {
   639  		base.Fatalf("verifying %v", module.VersionError(mod, fmt.Errorf("unexpected ziphash: %q", h)))
   640  	}
   641  
   642  	if err := checkModSum(mod, h); err != nil {
   643  		base.Fatalf("%s", err)
   644  	}
   645  }
   646  
   647  // goModSum returns the checksum for the go.mod contents.
   648  func goModSum(data []byte) (string, error) {
   649  	return dirhash.Hash1([]string{"go.mod"}, func(string) (io.ReadCloser, error) {
   650  		return io.NopCloser(bytes.NewReader(data)), nil
   651  	})
   652  }
   653  
   654  // checkGoMod checks the given module's go.mod checksum;
   655  // data is the go.mod content.
   656  func checkGoMod(path, version string, data []byte) error {
   657  	h, err := goModSum(data)
   658  	if err != nil {
   659  		return &module.ModuleError{Path: path, Version: version, Err: fmt.Errorf("verifying go.mod: %v", err)}
   660  	}
   661  
   662  	return checkModSum(module.Version{Path: path, Version: version + "/go.mod"}, h)
   663  }
   664  
   665  // checkModSum checks that the recorded checksum for mod is h.
   666  //
   667  // mod.Version may have the additional suffix "/go.mod" to request the checksum
   668  // for the module's go.mod file only.
   669  func checkModSum(mod module.Version, h string) error {
   670  	// We lock goSum when manipulating it,
   671  	// but we arrange to release the lock when calling checkSumDB,
   672  	// so that parallel calls to checkModHash can execute parallel calls
   673  	// to checkSumDB.
   674  
   675  	// Check whether mod+h is listed in go.sum already. If so, we're done.
   676  	goSum.mu.Lock()
   677  	inited, err := initGoSum()
   678  	if err != nil {
   679  		goSum.mu.Unlock()
   680  		return err
   681  	}
   682  	done := inited && haveModSumLocked(mod, h)
   683  	if inited {
   684  		st := goSum.status[modSum{mod, h}]
   685  		st.used = true
   686  		goSum.status[modSum{mod, h}] = st
   687  	}
   688  	goSum.mu.Unlock()
   689  
   690  	if done {
   691  		return nil
   692  	}
   693  
   694  	// Not listed, so we want to add them.
   695  	// Consult checksum database if appropriate.
   696  	if useSumDB(mod) {
   697  		// Calls base.Fatalf if mismatch detected.
   698  		if err := checkSumDB(mod, h); err != nil {
   699  			return err
   700  		}
   701  	}
   702  
   703  	// Add mod+h to go.sum, if it hasn't appeared already.
   704  	if inited {
   705  		goSum.mu.Lock()
   706  		addModSumLocked(mod, h)
   707  		st := goSum.status[modSum{mod, h}]
   708  		st.dirty = true
   709  		goSum.status[modSum{mod, h}] = st
   710  		goSum.mu.Unlock()
   711  	}
   712  	return nil
   713  }
   714  
   715  // haveModSumLocked reports whether the pair mod,h is already listed in go.sum.
   716  // If it finds a conflicting pair instead, it calls base.Fatalf.
   717  // goSum.mu must be locked.
   718  func haveModSumLocked(mod module.Version, h string) bool {
   719  	sumFileName := "go.sum"
   720  	if strings.HasSuffix(GoSumFile, "go.work.sum") {
   721  		sumFileName = "go.work.sum"
   722  	}
   723  	for _, vh := range goSum.m[mod] {
   724  		if h == vh {
   725  			return true
   726  		}
   727  		if strings.HasPrefix(vh, "h1:") {
   728  			base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s:     %v"+goSumMismatch, mod.Path, mod.Version, h, sumFileName, vh)
   729  		}
   730  	}
   731  	// Also check workspace sums.
   732  	foundMatch := false
   733  	// Check sums from all files in case there are conflicts between
   734  	// the files.
   735  	for goSumFile, goSums := range goSum.w {
   736  		for _, vh := range goSums[mod] {
   737  			if h == vh {
   738  				foundMatch = true
   739  			} else if strings.HasPrefix(vh, "h1:") {
   740  				base.Fatalf("verifying %s@%s: checksum mismatch\n\tdownloaded: %v\n\t%s:     %v"+goSumMismatch, mod.Path, mod.Version, h, goSumFile, vh)
   741  			}
   742  		}
   743  	}
   744  	return foundMatch
   745  }
   746  
   747  // addModSumLocked adds the pair mod,h to go.sum.
   748  // goSum.mu must be locked.
   749  func addModSumLocked(mod module.Version, h string) {
   750  	if haveModSumLocked(mod, h) {
   751  		return
   752  	}
   753  	if len(goSum.m[mod]) > 0 {
   754  		fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v"+hashVersionMismatch, mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h)
   755  	}
   756  	goSum.m[mod] = append(goSum.m[mod], h)
   757  }
   758  
   759  // checkSumDB checks the mod, h pair against the Go checksum database.
   760  // It calls base.Fatalf if the hash is to be rejected.
   761  func checkSumDB(mod module.Version, h string) error {
   762  	modWithoutSuffix := mod
   763  	noun := "module"
   764  	if before, found := strings.CutSuffix(mod.Version, "/go.mod"); found {
   765  		noun = "go.mod"
   766  		modWithoutSuffix.Version = before
   767  	}
   768  
   769  	db, lines, err := lookupSumDB(mod)
   770  	if err != nil {
   771  		return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: %v", noun, err))
   772  	}
   773  
   774  	have := mod.Path + " " + mod.Version + " " + h
   775  	prefix := mod.Path + " " + mod.Version + " h1:"
   776  	for _, line := range lines {
   777  		if line == have {
   778  			return nil
   779  		}
   780  		if strings.HasPrefix(line, prefix) {
   781  			return module.VersionError(modWithoutSuffix, fmt.Errorf("verifying %s: checksum mismatch\n\tdownloaded: %v\n\t%s: %v"+sumdbMismatch, noun, h, db, line[len(prefix)-len("h1:"):]))
   782  		}
   783  	}
   784  	return nil
   785  }
   786  
   787  // Sum returns the checksum for the downloaded copy of the given module,
   788  // if present in the download cache.
   789  func Sum(ctx context.Context, mod module.Version) string {
   790  	if cfg.GOMODCACHE == "" {
   791  		// Do not use current directory.
   792  		return ""
   793  	}
   794  
   795  	ziphash, err := CachePath(ctx, mod, "ziphash")
   796  	if err != nil {
   797  		return ""
   798  	}
   799  	data, err := lockedfile.Read(ziphash)
   800  	if err != nil {
   801  		return ""
   802  	}
   803  	data = bytes.TrimSpace(data)
   804  	if !isValidSum(data) {
   805  		return ""
   806  	}
   807  	return string(data)
   808  }
   809  
   810  // isValidSum returns true if data is the valid contents of a zip hash file.
   811  // Certain critical files are written to disk by first truncating
   812  // then writing the actual bytes, so that if the write fails
   813  // the corrupt file should contain at least one of the null
   814  // bytes written by the truncate operation.
   815  func isValidSum(data []byte) bool {
   816  	if bytes.IndexByte(data, '\000') >= 0 {
   817  		return false
   818  	}
   819  
   820  	if len(data) != len("h1:")+base64.StdEncoding.EncodedLen(sha256.Size) {
   821  		return false
   822  	}
   823  
   824  	return true
   825  }
   826  
   827  var ErrGoSumDirty = errors.New("updates to go.sum needed, disabled by -mod=readonly")
   828  
   829  // WriteGoSum writes the go.sum file if it needs to be updated.
   830  //
   831  // keep is used to check whether a newly added sum should be saved in go.sum.
   832  // It should have entries for both module content sums and go.mod sums
   833  // (version ends with "/go.mod"). Existing sums will be preserved unless they
   834  // have been marked for deletion with TrimGoSum.
   835  func WriteGoSum(ctx context.Context, keep map[module.Version]bool, readonly bool) error {
   836  	goSum.mu.Lock()
   837  	defer goSum.mu.Unlock()
   838  
   839  	// If we haven't read the go.sum file yet, don't bother writing it.
   840  	if !goSum.enabled {
   841  		return nil
   842  	}
   843  
   844  	// Check whether we need to add sums for which keep[m] is true or remove
   845  	// unused sums marked with TrimGoSum. If there are no changes to make,
   846  	// just return without opening go.sum.
   847  	dirty := false
   848  Outer:
   849  	for m, hs := range goSum.m {
   850  		for _, h := range hs {
   851  			st := goSum.status[modSum{m, h}]
   852  			if st.dirty && (!st.used || keep[m]) {
   853  				dirty = true
   854  				break Outer
   855  			}
   856  		}
   857  	}
   858  	if !dirty {
   859  		return nil
   860  	}
   861  	if readonly {
   862  		return ErrGoSumDirty
   863  	}
   864  	if _, ok := fsys.OverlayPath(GoSumFile); ok {
   865  		base.Fatalf("go: updates to go.sum needed, but go.sum is part of the overlay specified with -overlay")
   866  	}
   867  
   868  	// Make a best-effort attempt to acquire the side lock, only to exclude
   869  	// previous versions of the 'go' command from making simultaneous edits.
   870  	if unlock, err := SideLock(ctx); err == nil {
   871  		defer unlock()
   872  	}
   873  
   874  	err := lockedfile.Transform(GoSumFile, func(data []byte) ([]byte, error) {
   875  		tidyGoSum := tidyGoSum(data, keep)
   876  		return tidyGoSum, nil
   877  	})
   878  
   879  	if err != nil {
   880  		return fmt.Errorf("updating go.sum: %w", err)
   881  	}
   882  
   883  	goSum.status = make(map[modSum]modSumStatus)
   884  	goSum.overwrite = false
   885  	return nil
   886  }
   887  
   888  // TidyGoSum returns a tidy version of the go.sum file.
   889  // A missing go.sum file is treated as if empty.
   890  func TidyGoSum(keep map[module.Version]bool) (before, after []byte) {
   891  	goSum.mu.Lock()
   892  	defer goSum.mu.Unlock()
   893  	before, err := lockedfile.Read(GoSumFile)
   894  	if err != nil && !errors.Is(err, fs.ErrNotExist) {
   895  		base.Fatalf("reading go.sum: %v", err)
   896  	}
   897  	after = tidyGoSum(before, keep)
   898  	return before, after
   899  }
   900  
   901  // tidyGoSum returns a tidy version of the go.sum file.
   902  // The goSum lock must be held.
   903  func tidyGoSum(data []byte, keep map[module.Version]bool) []byte {
   904  	if !goSum.overwrite {
   905  		// Incorporate any sums added by other processes in the meantime.
   906  		// Add only the sums that we actually checked: the user may have edited or
   907  		// truncated the file to remove erroneous hashes, and we shouldn't restore
   908  		// them without good reason.
   909  		goSum.m = make(map[module.Version][]string, len(goSum.m))
   910  		readGoSum(goSum.m, GoSumFile, data)
   911  		for ms, st := range goSum.status {
   912  			if st.used && !sumInWorkspaceModulesLocked(ms.mod) {
   913  				addModSumLocked(ms.mod, ms.sum)
   914  			}
   915  		}
   916  	}
   917  
   918  	mods := make([]module.Version, 0, len(goSum.m))
   919  	for m := range goSum.m {
   920  		mods = append(mods, m)
   921  	}
   922  	module.Sort(mods)
   923  
   924  	var buf bytes.Buffer
   925  	for _, m := range mods {
   926  		list := goSum.m[m]
   927  		sort.Strings(list)
   928  		str.Uniq(&list)
   929  		for _, h := range list {
   930  			st := goSum.status[modSum{m, h}]
   931  			if (!st.dirty || (st.used && keep[m])) && !sumInWorkspaceModulesLocked(m) {
   932  				fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h)
   933  			}
   934  		}
   935  	}
   936  	return buf.Bytes()
   937  }
   938  
   939  func sumInWorkspaceModulesLocked(m module.Version) bool {
   940  	for _, goSums := range goSum.w {
   941  		if _, ok := goSums[m]; ok {
   942  			return true
   943  		}
   944  	}
   945  	return false
   946  }
   947  
   948  // TrimGoSum trims go.sum to contain only the modules needed for reproducible
   949  // builds.
   950  //
   951  // keep is used to check whether a sum should be retained in go.mod. It should
   952  // have entries for both module content sums and go.mod sums (version ends
   953  // with "/go.mod").
   954  func TrimGoSum(keep map[module.Version]bool) {
   955  	goSum.mu.Lock()
   956  	defer goSum.mu.Unlock()
   957  	inited, err := initGoSum()
   958  	if err != nil {
   959  		base.Fatalf("%s", err)
   960  	}
   961  	if !inited {
   962  		return
   963  	}
   964  
   965  	for m, hs := range goSum.m {
   966  		if !keep[m] {
   967  			for _, h := range hs {
   968  				goSum.status[modSum{m, h}] = modSumStatus{used: false, dirty: true}
   969  			}
   970  			goSum.overwrite = true
   971  		}
   972  	}
   973  }
   974  
   975  const goSumMismatch = `
   976  
   977  SECURITY ERROR
   978  This download does NOT match an earlier download recorded in go.sum.
   979  The bits may have been replaced on the origin server, or an attacker may
   980  have intercepted the download attempt.
   981  
   982  For more information, see 'go help module-auth'.
   983  `
   984  
   985  const sumdbMismatch = `
   986  
   987  SECURITY ERROR
   988  This download does NOT match the one reported by the checksum server.
   989  The bits may have been replaced on the origin server, or an attacker may
   990  have intercepted the download attempt.
   991  
   992  For more information, see 'go help module-auth'.
   993  `
   994  
   995  const hashVersionMismatch = `
   996  
   997  SECURITY WARNING
   998  This download is listed in go.sum, but using an unknown hash algorithm.
   999  The download cannot be verified.
  1000  
  1001  For more information, see 'go help module-auth'.
  1002  
  1003  `
  1004  
  1005  var HelpModuleAuth = &base.Command{
  1006  	UsageLine: "module-auth",
  1007  	Short:     "module authentication using go.sum",
  1008  	Long: `
  1009  When the go command downloads a module zip file or go.mod file into the
  1010  module cache, it computes a cryptographic hash and compares it with a known
  1011  value to verify the file hasn't changed since it was first downloaded. Known
  1012  hashes are stored in a file in the module root directory named go.sum. Hashes
  1013  may also be downloaded from the checksum database depending on the values of
  1014  GOSUMDB, GOPRIVATE, and GONOSUMDB.
  1015  
  1016  For details, see https://golang.org/ref/mod#authenticating.
  1017  `,
  1018  }
  1019  
  1020  var HelpPrivate = &base.Command{
  1021  	UsageLine: "private",
  1022  	Short:     "configuration for downloading non-public code",
  1023  	Long: `
  1024  The go command defaults to downloading modules from the public Go module
  1025  mirror at proxy.golang.org. It also defaults to validating downloaded modules,
  1026  regardless of source, against the public Go checksum database at sum.golang.org.
  1027  These defaults work well for publicly available source code.
  1028  
  1029  The GOPRIVATE environment variable controls which modules the go command
  1030  considers to be private (not available publicly) and should therefore not use
  1031  the proxy or checksum database. The variable is a comma-separated list of
  1032  glob patterns (in the syntax of Go's path.Match) of module path prefixes.
  1033  For example,
  1034  
  1035  	GOPRIVATE=*.corp.example.com,rsc.io/private
  1036  
  1037  causes the go command to treat as private any module with a path prefix
  1038  matching either pattern, including git.corp.example.com/xyzzy, rsc.io/private,
  1039  and rsc.io/private/quux.
  1040  
  1041  For fine-grained control over module download and validation, the GONOPROXY
  1042  and GONOSUMDB environment variables accept the same kind of glob list
  1043  and override GOPRIVATE for the specific decision of whether to use the proxy
  1044  and checksum database, respectively.
  1045  
  1046  For example, if a company ran a module proxy serving private modules,
  1047  users would configure go using:
  1048  
  1049  	GOPRIVATE=*.corp.example.com
  1050  	GOPROXY=proxy.example.com
  1051  	GONOPROXY=none
  1052  
  1053  The GOPRIVATE variable is also used to define the "public" and "private"
  1054  patterns for the GOVCS variable; see 'go help vcs'. For that usage,
  1055  GOPRIVATE applies even in GOPATH mode. In that case, it matches import paths
  1056  instead of module paths.
  1057  
  1058  The 'go env -w' command (see 'go help env') can be used to set these variables
  1059  for future go command invocations.
  1060  
  1061  For more details, see https://golang.org/ref/mod#private-modules.
  1062  `,
  1063  }
  1064
View as plain text