vendor.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modcmd
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"errors"
    11  	"fmt"
    12  	"go/build"
    13  	"io"
    14  	"io/fs"
    15  	"os"
    16  	"path"
    17  	"path/filepath"
    18  	"sort"
    19  	"strings"
    20  
    21  	"cmd/go/internal/base"
    22  	"cmd/go/internal/cfg"
    23  	"cmd/go/internal/fsys"
    24  	"cmd/go/internal/gover"
    25  	"cmd/go/internal/imports"
    26  	"cmd/go/internal/load"
    27  	"cmd/go/internal/modload"
    28  	"cmd/go/internal/str"
    29  
    30  	"golang.org/x/mod/module"
    31  )
    32  
    33  var cmdVendor = &base.Command{
    34  	UsageLine: "go mod vendor [-e] [-v] [-o outdir]",
    35  	Short:     "make vendored copy of dependencies",
    36  	Long: `
    37  Vendor resets the main module's vendor directory to include all packages
    38  needed to build and test all the main module's packages.
    39  It does not include test code for vendored packages.
    40  
    41  The -v flag causes vendor to print the names of vendored
    42  modules and packages to standard error.
    43  
    44  The -e flag causes vendor to attempt to proceed despite errors
    45  encountered while loading packages.
    46  
    47  The -o flag causes vendor to create the vendor directory at the given
    48  path instead of "vendor". The go command can only use a vendor directory
    49  named "vendor" within the module root directory, so this flag is
    50  primarily useful for other tools.
    51  
    52  See https://golang.org/ref/mod#go-mod-vendor for more about 'go mod vendor'.
    53  	`,
    54  	Run: runVendor,
    55  }
    56  
    57  var vendorE bool   // if true, report errors but proceed anyway
    58  var vendorO string // if set, overrides the default output directory
    59  
    60  func init() {
    61  	cmdVendor.Flag.BoolVar(&cfg.BuildV, "v", false, "")
    62  	cmdVendor.Flag.BoolVar(&vendorE, "e", false, "")
    63  	cmdVendor.Flag.StringVar(&vendorO, "o", "", "")
    64  	base.AddChdirFlag(&cmdVendor.Flag)
    65  	base.AddModCommonFlags(&cmdVendor.Flag)
    66  }
    67  
    68  func runVendor(ctx context.Context, cmd *base.Command, args []string) {
    69  	modload.InitWorkfile()
    70  	if modload.WorkFilePath() != "" {
    71  		base.Fatalf("go: 'go mod vendor' cannot be run in workspace mode. Run 'go work vendor' to vendor the workspace or set 'GOWORK=off' to exit workspace mode.")
    72  	}
    73  	RunVendor(ctx, vendorE, vendorO, args)
    74  }
    75  
    76  func RunVendor(ctx context.Context, vendorE bool, vendorO string, args []string) {
    77  	if len(args) != 0 {
    78  		base.Fatalf("go: 'go mod vendor' accepts no arguments")
    79  	}
    80  	modload.ForceUseModules = true
    81  	modload.RootMode = modload.NeedRoot
    82  
    83  	loadOpts := modload.PackageOpts{
    84  		Tags:                     imports.AnyTags(),
    85  		VendorModulesInGOROOTSrc: true,
    86  		ResolveMissingImports:    true,
    87  		UseVendorAll:             true,
    88  		AllowErrors:              vendorE,
    89  		SilenceMissingStdImports: true,
    90  	}
    91  	_, pkgs := modload.LoadPackages(ctx, loadOpts, "all")
    92  
    93  	var vdir string
    94  	switch {
    95  	case filepath.IsAbs(vendorO):
    96  		vdir = vendorO
    97  	case vendorO != "":
    98  		vdir = filepath.Join(base.Cwd(), vendorO)
    99  	default:
   100  		vdir = filepath.Join(modload.VendorDir())
   101  	}
   102  	if err := os.RemoveAll(vdir); err != nil {
   103  		base.Fatal(err)
   104  	}
   105  
   106  	modpkgs := make(map[module.Version][]string)
   107  	for _, pkg := range pkgs {
   108  		m := modload.PackageModule(pkg)
   109  		if m.Path == "" || modload.MainModules.Contains(m.Path) {
   110  			continue
   111  		}
   112  		modpkgs[m] = append(modpkgs[m], pkg)
   113  	}
   114  	checkPathCollisions(modpkgs)
   115  
   116  	includeAllReplacements := false
   117  	includeGoVersions := false
   118  	isExplicit := map[module.Version]bool{}
   119  	gv := modload.MainModules.GoVersion()
   120  	if gover.Compare(gv, "1.14") >= 0 && (modload.FindGoWork(base.Cwd()) != "" || modload.ModFile().Go != nil) {
   121  		// If the Go version is at least 1.14, annotate all explicit 'require' and
   122  		// 'replace' targets found in the go.mod file so that we can perform a
   123  		// stronger consistency check when -mod=vendor is set.
   124  		for _, m := range modload.MainModules.Versions() {
   125  			if modFile := modload.MainModules.ModFile(m); modFile != nil {
   126  				for _, r := range modFile.Require {
   127  					isExplicit[r.Mod] = true
   128  				}
   129  			}
   130  
   131  		}
   132  		includeAllReplacements = true
   133  	}
   134  	if gover.Compare(gv, "1.17") >= 0 {
   135  		// If the Go version is at least 1.17, annotate all modules with their
   136  		// 'go' version directives.
   137  		includeGoVersions = true
   138  	}
   139  
   140  	var vendorMods []module.Version
   141  	for m := range isExplicit {
   142  		vendorMods = append(vendorMods, m)
   143  	}
   144  	for m := range modpkgs {
   145  		if !isExplicit[m] {
   146  			vendorMods = append(vendorMods, m)
   147  		}
   148  	}
   149  	gover.ModSort(vendorMods)
   150  
   151  	var (
   152  		buf bytes.Buffer
   153  		w   io.Writer = &buf
   154  	)
   155  	if cfg.BuildV {
   156  		w = io.MultiWriter(&buf, os.Stderr)
   157  	}
   158  
   159  	if modload.MainModules.WorkFile() != nil {
   160  		fmt.Fprintf(w, "## workspace\n")
   161  	}
   162  
   163  	replacementWritten := make(map[module.Version]bool)
   164  	for _, m := range vendorMods {
   165  		replacement := modload.Replacement(m)
   166  		line := moduleLine(m, replacement)
   167  		replacementWritten[m] = true
   168  		io.WriteString(w, line)
   169  
   170  		goVersion := ""
   171  		if includeGoVersions {
   172  			goVersion = modload.ModuleInfo(ctx, m.Path).GoVersion
   173  		}
   174  		switch {
   175  		case isExplicit[m] && goVersion != "":
   176  			fmt.Fprintf(w, "## explicit; go %s\n", goVersion)
   177  		case isExplicit[m]:
   178  			io.WriteString(w, "## explicit\n")
   179  		case goVersion != "":
   180  			fmt.Fprintf(w, "## go %s\n", goVersion)
   181  		}
   182  
   183  		pkgs := modpkgs[m]
   184  		sort.Strings(pkgs)
   185  		for _, pkg := range pkgs {
   186  			fmt.Fprintf(w, "%s\n", pkg)
   187  			vendorPkg(vdir, pkg)
   188  		}
   189  	}
   190  
   191  	if includeAllReplacements {
   192  		// Record unused and wildcard replacements at the end of the modules.txt file:
   193  		// without access to the complete build list, the consumer of the vendor
   194  		// directory can't otherwise determine that those replacements had no effect.
   195  		for _, m := range modload.MainModules.Versions() {
   196  			if workFile := modload.MainModules.WorkFile(); workFile != nil {
   197  				for _, r := range workFile.Replace {
   198  					if replacementWritten[r.Old] {
   199  						// We already recorded this replacement.
   200  						continue
   201  					}
   202  					replacementWritten[r.Old] = true
   203  
   204  					line := moduleLine(r.Old, r.New)
   205  					buf.WriteString(line)
   206  					if cfg.BuildV {
   207  						os.Stderr.WriteString(line)
   208  					}
   209  				}
   210  			}
   211  			if modFile := modload.MainModules.ModFile(m); modFile != nil {
   212  				for _, r := range modFile.Replace {
   213  					if replacementWritten[r.Old] {
   214  						// We already recorded this replacement.
   215  						continue
   216  					}
   217  					replacementWritten[r.Old] = true
   218  					rNew := modload.Replacement(r.Old)
   219  					if rNew == (module.Version{}) {
   220  						// There is no replacement. Don't try to write it.
   221  						continue
   222  					}
   223  
   224  					line := moduleLine(r.Old, rNew)
   225  					buf.WriteString(line)
   226  					if cfg.BuildV {
   227  						os.Stderr.WriteString(line)
   228  					}
   229  				}
   230  			}
   231  		}
   232  	}
   233  
   234  	if buf.Len() == 0 {
   235  		fmt.Fprintf(os.Stderr, "go: no dependencies to vendor\n")
   236  		return
   237  	}
   238  
   239  	if err := os.MkdirAll(vdir, 0777); err != nil {
   240  		base.Fatal(err)
   241  	}
   242  
   243  	if err := os.WriteFile(filepath.Join(vdir, "modules.txt"), buf.Bytes(), 0666); err != nil {
   244  		base.Fatal(err)
   245  	}
   246  }
   247  
   248  func moduleLine(m, r module.Version) string {
   249  	b := new(strings.Builder)
   250  	b.WriteString("# ")
   251  	b.WriteString(m.Path)
   252  	if m.Version != "" {
   253  		b.WriteString(" ")
   254  		b.WriteString(m.Version)
   255  	}
   256  	if r.Path != "" {
   257  		if str.HasFilePathPrefix(filepath.Clean(r.Path), "vendor") {
   258  			base.Fatalf("go: replacement path %s inside vendor directory", r.Path)
   259  		}
   260  		b.WriteString(" => ")
   261  		b.WriteString(r.Path)
   262  		if r.Version != "" {
   263  			b.WriteString(" ")
   264  			b.WriteString(r.Version)
   265  		}
   266  	}
   267  	b.WriteString("\n")
   268  	return b.String()
   269  }
   270  
   271  func vendorPkg(vdir, pkg string) {
   272  	src, realPath, _ := modload.Lookup("", false, pkg)
   273  	if src == "" {
   274  		base.Errorf("internal error: no pkg for %s\n", pkg)
   275  		return
   276  	}
   277  	if realPath != pkg {
   278  		// TODO(#26904): Revisit whether this behavior still makes sense.
   279  		// This should actually be impossible today, because the import map is the
   280  		// identity function for packages outside of the standard library.
   281  		//
   282  		// Part of the purpose of the vendor directory is to allow the packages in
   283  		// the module to continue to build in GOPATH mode, and GOPATH-mode users
   284  		// won't know about replacement aliasing. How important is it to maintain
   285  		// compatibility?
   286  		fmt.Fprintf(os.Stderr, "warning: %s imported as both %s and %s; making two copies.\n", realPath, realPath, pkg)
   287  	}
   288  
   289  	copiedFiles := make(map[string]bool)
   290  	dst := filepath.Join(vdir, pkg)
   291  	copyDir(dst, src, matchPotentialSourceFile, copiedFiles)
   292  	if m := modload.PackageModule(realPath); m.Path != "" {
   293  		copyMetadata(m.Path, realPath, dst, src, copiedFiles)
   294  	}
   295  
   296  	ctx := build.Default
   297  	ctx.UseAllFiles = true
   298  	bp, err := ctx.ImportDir(src, build.IgnoreVendor)
   299  	// Because UseAllFiles is set on the build.Context, it's possible ta get
   300  	// a MultiplePackageError on an otherwise valid package: the package could
   301  	// have different names for GOOS=windows and GOOS=mac for example. On the
   302  	// other hand if there's a NoGoError, the package might have source files
   303  	// specifying "//go:build ignore" those packages should be skipped because
   304  	// embeds from ignored files can't be used.
   305  	// TODO(#42504): Find a better way to avoid errors from ImportDir. We'll
   306  	// need to figure this out when we switch to PackagesAndErrors as per the
   307  	// TODO above.
   308  	var multiplePackageError *build.MultiplePackageError
   309  	var noGoError *build.NoGoError
   310  	if err != nil {
   311  		if errors.As(err, &noGoError) {
   312  			return // No source files in this package are built. Skip embeds in ignored files.
   313  		} else if !errors.As(err, &multiplePackageError) { // multiplePackageErrors are OK, but others are not.
   314  			base.Fatalf("internal error: failed to find embedded files of %s: %v\n", pkg, err)
   315  		}
   316  	}
   317  	var embedPatterns []string
   318  	if gover.Compare(modload.MainModules.GoVersion(), "1.22") >= 0 {
   319  		embedPatterns = bp.EmbedPatterns
   320  	} else {
   321  		// Maintain the behavior of https://github.com/golang/go/issues/63473
   322  		// so that we continue to agree with older versions of the go command
   323  		// about the contents of vendor directories in existing modules
   324  		embedPatterns = str.StringList(bp.EmbedPatterns, bp.TestEmbedPatterns, bp.XTestEmbedPatterns)
   325  	}
   326  	embeds, err := load.ResolveEmbed(bp.Dir, embedPatterns)
   327  	if err != nil {
   328  		format := "go: resolving embeds in %s: %v\n"
   329  		if vendorE {
   330  			fmt.Fprintf(os.Stderr, format, pkg, err)
   331  		} else {
   332  			base.Errorf(format, pkg, err)
   333  		}
   334  		return
   335  	}
   336  	for _, embed := range embeds {
   337  		embedDst := filepath.Join(dst, embed)
   338  		if copiedFiles[embedDst] {
   339  			continue
   340  		}
   341  
   342  		// Copy the file as is done by copyDir below.
   343  		err := func() error {
   344  			r, err := os.Open(filepath.Join(src, embed))
   345  			if err != nil {
   346  				return err
   347  			}
   348  			if err := os.MkdirAll(filepath.Dir(embedDst), 0777); err != nil {
   349  				return err
   350  			}
   351  			w, err := os.Create(embedDst)
   352  			if err != nil {
   353  				return err
   354  			}
   355  			if _, err := io.Copy(w, r); err != nil {
   356  				return err
   357  			}
   358  			r.Close()
   359  			return w.Close()
   360  		}()
   361  		if err != nil {
   362  			if vendorE {
   363  				fmt.Fprintf(os.Stderr, "go: %v\n", err)
   364  			} else {
   365  				base.Error(err)
   366  			}
   367  		}
   368  	}
   369  }
   370  
   371  type metakey struct {
   372  	modPath string
   373  	dst     string
   374  }
   375  
   376  var copiedMetadata = make(map[metakey]bool)
   377  
   378  // copyMetadata copies metadata files from parents of src to parents of dst,
   379  // stopping after processing the src parent for modPath.
   380  func copyMetadata(modPath, pkg, dst, src string, copiedFiles map[string]bool) {
   381  	for parent := 0; ; parent++ {
   382  		if copiedMetadata[metakey{modPath, dst}] {
   383  			break
   384  		}
   385  		copiedMetadata[metakey{modPath, dst}] = true
   386  		if parent > 0 {
   387  			copyDir(dst, src, matchMetadata, copiedFiles)
   388  		}
   389  		if modPath == pkg {
   390  			break
   391  		}
   392  		pkg = path.Dir(pkg)
   393  		dst = filepath.Dir(dst)
   394  		src = filepath.Dir(src)
   395  	}
   396  }
   397  
   398  // metaPrefixes is the list of metadata file prefixes.
   399  // Vendoring copies metadata files from parents of copied directories.
   400  // Note that this list could be arbitrarily extended, and it is longer
   401  // in other tools (such as godep or dep). By using this limited set of
   402  // prefixes and also insisting on capitalized file names, we are trying
   403  // to nudge people toward more agreement on the naming
   404  // and also trying to avoid false positives.
   405  var metaPrefixes = []string{
   406  	"AUTHORS",
   407  	"CONTRIBUTORS",
   408  	"COPYLEFT",
   409  	"COPYING",
   410  	"COPYRIGHT",
   411  	"LEGAL",
   412  	"LICENSE",
   413  	"NOTICE",
   414  	"PATENTS",
   415  }
   416  
   417  // matchMetadata reports whether info is a metadata file.
   418  func matchMetadata(dir string, info fs.DirEntry) bool {
   419  	name := info.Name()
   420  	for _, p := range metaPrefixes {
   421  		if strings.HasPrefix(name, p) {
   422  			return true
   423  		}
   424  	}
   425  	return false
   426  }
   427  
   428  // matchPotentialSourceFile reports whether info may be relevant to a build operation.
   429  func matchPotentialSourceFile(dir string, info fs.DirEntry) bool {
   430  	if strings.HasSuffix(info.Name(), "_test.go") {
   431  		return false
   432  	}
   433  	if info.Name() == "go.mod" || info.Name() == "go.sum" {
   434  		if gv := modload.MainModules.GoVersion(); gover.Compare(gv, "1.17") >= 0 {
   435  			// As of Go 1.17, we strip go.mod and go.sum files from dependency modules.
   436  			// Otherwise, 'go' commands invoked within the vendor subtree may misidentify
   437  			// an arbitrary directory within the vendor tree as a module root.
   438  			// (See https://golang.org/issue/42970.)
   439  			return false
   440  		}
   441  	}
   442  	if strings.HasSuffix(info.Name(), ".go") {
   443  		f, err := fsys.Open(filepath.Join(dir, info.Name()))
   444  		if err != nil {
   445  			base.Fatal(err)
   446  		}
   447  		defer f.Close()
   448  
   449  		content, err := imports.ReadImports(f, false, nil)
   450  		if err == nil && !imports.ShouldBuild(content, imports.AnyTags()) {
   451  			// The file is explicitly tagged "ignore", so it can't affect the build.
   452  			// Leave it out.
   453  			return false
   454  		}
   455  		return true
   456  	}
   457  
   458  	// We don't know anything about this file, so optimistically assume that it is
   459  	// needed.
   460  	return true
   461  }
   462  
   463  // copyDir copies all regular files satisfying match(info) from src to dst.
   464  func copyDir(dst, src string, match func(dir string, info fs.DirEntry) bool, copiedFiles map[string]bool) {
   465  	files, err := os.ReadDir(src)
   466  	if err != nil {
   467  		base.Fatal(err)
   468  	}
   469  	if err := os.MkdirAll(dst, 0777); err != nil {
   470  		base.Fatal(err)
   471  	}
   472  	for _, file := range files {
   473  		if file.IsDir() || !file.Type().IsRegular() || !match(src, file) {
   474  			continue
   475  		}
   476  		copiedFiles[file.Name()] = true
   477  		r, err := os.Open(filepath.Join(src, file.Name()))
   478  		if err != nil {
   479  			base.Fatal(err)
   480  		}
   481  		dstPath := filepath.Join(dst, file.Name())
   482  		copiedFiles[dstPath] = true
   483  		w, err := os.Create(dstPath)
   484  		if err != nil {
   485  			base.Fatal(err)
   486  		}
   487  		if _, err := io.Copy(w, r); err != nil {
   488  			base.Fatal(err)
   489  		}
   490  		r.Close()
   491  		if err := w.Close(); err != nil {
   492  			base.Fatal(err)
   493  		}
   494  	}
   495  }
   496  
   497  // checkPathCollisions will fail if case-insensitive collisions are present.
   498  // The reason why we do this check in go mod vendor is to keep consistency
   499  // with go build. If modifying, consider changing load() in
   500  // src/cmd/go/internal/load/pkg.go
   501  func checkPathCollisions(modpkgs map[module.Version][]string) {
   502  	var foldPath = make(map[string]string, len(modpkgs))
   503  	for m := range modpkgs {
   504  		fold := str.ToFold(m.Path)
   505  		if other := foldPath[fold]; other == "" {
   506  			foldPath[fold] = m.Path
   507  		} else if other != m.Path {
   508  			base.Fatalf("go.mod: case-insensitive import collision: %q and %q", m.Path, other)
   509  		}
   510  	}
   511  }
   512
View as plain text