repo.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modfetch
     6  
     7  import (
     8  	"context"
     9  	"fmt"
    10  	"io"
    11  	"io/fs"
    12  	"os"
    13  	"strconv"
    14  	"time"
    15  
    16  	"cmd/go/internal/cfg"
    17  	"cmd/go/internal/modfetch/codehost"
    18  	"cmd/go/internal/vcs"
    19  	web "cmd/go/internal/web"
    20  	"cmd/internal/par"
    21  
    22  	"golang.org/x/mod/module"
    23  )
    24  
    25  const traceRepo = false // trace all repo actions, for debugging
    26  
    27  // A Repo represents a repository storing all versions of a single module.
    28  // It must be safe for simultaneous use by multiple goroutines.
    29  type Repo interface {
    30  	// ModulePath returns the module path.
    31  	ModulePath() string
    32  
    33  	// CheckReuse checks whether the validation criteria in the origin
    34  	// are still satisfied on the server corresponding to this module.
    35  	// If so, the caller can reuse any cached Versions or RevInfo containing
    36  	// this origin rather than redownloading those from the server.
    37  	CheckReuse(ctx context.Context, old *codehost.Origin) error
    38  
    39  	// Versions lists all known versions with the given prefix.
    40  	// Pseudo-versions are not included.
    41  	//
    42  	// Versions should be returned sorted in semver order
    43  	// (implementations can use semver.Sort).
    44  	//
    45  	// Versions returns a non-nil error only if there was a problem
    46  	// fetching the list of versions: it may return an empty list
    47  	// along with a nil error if the list of matching versions
    48  	// is known to be empty.
    49  	//
    50  	// If the underlying repository does not exist,
    51  	// Versions returns an error matching errors.Is(_, os.NotExist).
    52  	Versions(ctx context.Context, prefix string) (*Versions, error)
    53  
    54  	// Stat returns information about the revision rev.
    55  	// A revision can be any identifier known to the underlying service:
    56  	// commit hash, branch, tag, and so on.
    57  	Stat(ctx context.Context, rev string) (*RevInfo, error)
    58  
    59  	// Latest returns the latest revision on the default branch,
    60  	// whatever that means in the underlying source code repository.
    61  	// It is only used when there are no tagged versions.
    62  	Latest(ctx context.Context) (*RevInfo, error)
    63  
    64  	// GoMod returns the go.mod file for the given version.
    65  	GoMod(ctx context.Context, version string) (data []byte, err error)
    66  
    67  	// Zip writes a zip file for the given version to dst.
    68  	Zip(ctx context.Context, dst io.Writer, version string) error
    69  }
    70  
    71  // A Versions describes the available versions in a module repository.
    72  type Versions struct {
    73  	Origin *codehost.Origin `json:",omitempty"` // origin information for reuse
    74  
    75  	List []string // semver versions
    76  }
    77  
    78  // A RevInfo describes a single revision in a module repository.
    79  type RevInfo struct {
    80  	Version string    // suggested version string for this revision
    81  	Time    time.Time // commit time
    82  
    83  	// These fields are used for Stat of arbitrary rev,
    84  	// but they are not recorded when talking about module versions.
    85  	Name  string `json:"-"` // complete ID in underlying repository
    86  	Short string `json:"-"` // shortened ID, for use in pseudo-version
    87  
    88  	Origin *codehost.Origin `json:",omitempty"` // provenance for reuse
    89  }
    90  
    91  // Re: module paths, import paths, repository roots, and lookups
    92  //
    93  // A module is a collection of Go packages stored in a file tree
    94  // with a go.mod file at the root of the tree.
    95  // The go.mod defines the module path, which is the import path
    96  // corresponding to the root of the file tree.
    97  // The import path of a directory within that file tree is the module path
    98  // joined with the name of the subdirectory relative to the root.
    99  //
   100  // For example, the module with path rsc.io/qr corresponds to the
   101  // file tree in the repository https://github.com/rsc/qr.
   102  // That file tree has a go.mod that says "module rsc.io/qr".
   103  // The package in the root directory has import path "rsc.io/qr".
   104  // The package in the gf256 subdirectory has import path "rsc.io/qr/gf256".
   105  // In this example, "rsc.io/qr" is both a module path and an import path.
   106  // But "rsc.io/qr/gf256" is only an import path, not a module path:
   107  // it names an importable package, but not a module.
   108  //
   109  // As a special case to incorporate code written before modules were
   110  // introduced, if a path p resolves using the pre-module "go get" lookup
   111  // to the root of a source code repository without a go.mod file,
   112  // that repository is treated as if it had a go.mod in its root directory
   113  // declaring module path p. (The go.mod is further considered to
   114  // contain requirements corresponding to any legacy version
   115  // tracking format such as Gopkg.lock, vendor/vendor.conf, and so on.)
   116  //
   117  // The presentation so far ignores the fact that a source code repository
   118  // has many different versions of a file tree, and those versions may
   119  // differ in whether a particular go.mod exists and what it contains.
   120  // In fact there is a well-defined mapping only from a module path, version
   121  // pair - often written path@version - to a particular file tree.
   122  // For example rsc.io/qr@v0.1.0 depends on the "implicit go.mod at root of
   123  // repository" rule, while rsc.io/qr@v0.2.0 has an explicit go.mod.
   124  // Because the "go get" import paths rsc.io/qr and github.com/rsc/qr
   125  // both redirect to the Git repository https://github.com/rsc/qr,
   126  // github.com/rsc/qr@v0.1.0 is the same file tree as rsc.io/qr@v0.1.0
   127  // but a different module (a different name). In contrast, since v0.2.0
   128  // of that repository has an explicit go.mod that declares path rsc.io/qr,
   129  // github.com/rsc/qr@v0.2.0 is an invalid module path, version pair.
   130  // Before modules, import comments would have had the same effect.
   131  //
   132  // The set of import paths associated with a given module path is
   133  // clearly not fixed: at the least, new directories with new import paths
   134  // can always be added. But another potential operation is to split a
   135  // subtree out of a module into its own module. If done carefully,
   136  // this operation can be done while preserving compatibility for clients.
   137  // For example, suppose that we want to split rsc.io/qr/gf256 into its
   138  // own module, so that there would be two modules rsc.io/qr and rsc.io/qr/gf256.
   139  // Then we can simultaneously issue rsc.io/qr v0.3.0 (dropping the gf256 subdirectory)
   140  // and rsc.io/qr/gf256 v0.1.0, including in their respective go.mod
   141  // cyclic requirements pointing at each other: rsc.io/qr v0.3.0 requires
   142  // rsc.io/qr/gf256 v0.1.0 and vice versa. Then a build can be
   143  // using an older rsc.io/qr module that includes the gf256 package, but if
   144  // it adds a requirement on either the newer rsc.io/qr or the newer
   145  // rsc.io/qr/gf256 module, it will automatically add the requirement
   146  // on the complementary half, ensuring both that rsc.io/qr/gf256 is
   147  // available for importing by the build and also that it is only defined
   148  // by a single module. The gf256 package could move back into the
   149  // original by another simultaneous release of rsc.io/qr v0.4.0 including
   150  // the gf256 subdirectory and an rsc.io/qr/gf256 v0.2.0 with no code
   151  // in its root directory, along with a new requirement cycle.
   152  // The ability to shift module boundaries in this way is expected to be
   153  // important in large-scale program refactorings, similar to the ones
   154  // described in https://talks.golang.org/2016/refactor.article.
   155  //
   156  // The possibility of shifting module boundaries reemphasizes
   157  // that you must know both the module path and its version
   158  // to determine the set of packages provided directly by that module.
   159  //
   160  // On top of all this, it is possible for a single code repository
   161  // to contain multiple modules, either in branches or subdirectories,
   162  // as a limited kind of monorepo. For example rsc.io/qr/v2,
   163  // the v2.x.x continuation of rsc.io/qr, is expected to be found
   164  // in v2-tagged commits in https://github.com/rsc/qr, either
   165  // in the root or in a v2 subdirectory, disambiguated by go.mod.
   166  // Again the precise file tree corresponding to a module
   167  // depends on which version we are considering.
   168  //
   169  // It is also possible for the underlying repository to change over time,
   170  // without changing the module path. If I copy the github repo over
   171  // to https://bitbucket.org/rsc/qr and update https://rsc.io/qr?go-get=1,
   172  // then clients of all versions should start fetching from bitbucket
   173  // instead of github. That is, in contrast to the exact file tree,
   174  // the location of the source code repository associated with a module path
   175  // does not depend on the module version. (This is by design, as the whole
   176  // point of these redirects is to allow package authors to establish a stable
   177  // name that can be updated as code moves from one service to another.)
   178  //
   179  // All of this is important background for the lookup APIs defined in this
   180  // file.
   181  //
   182  // The Lookup function takes a module path and returns a Repo representing
   183  // that module path. Lookup can do only a little with the path alone.
   184  // It can check that the path is well-formed (see semver.CheckPath)
   185  // and it can check that the path can be resolved to a target repository.
   186  // To avoid version control access except when absolutely necessary,
   187  // Lookup does not attempt to connect to the repository itself.
   188  
   189  var lookupCache par.Cache[lookupCacheKey, Repo]
   190  
   191  type lookupCacheKey struct {
   192  	proxy, path string
   193  }
   194  
   195  // Lookup returns the module with the given module path,
   196  // fetched through the given proxy.
   197  //
   198  // The distinguished proxy "direct" indicates that the path should be fetched
   199  // from its origin, and "noproxy" indicates that the patch should be fetched
   200  // directly only if GONOPROXY matches the given path.
   201  //
   202  // For the distinguished proxy "off", Lookup always returns a Repo that returns
   203  // a non-nil error for every method call.
   204  //
   205  // A successful return does not guarantee that the module
   206  // has any defined versions.
   207  func Lookup(ctx context.Context, proxy, path string) Repo {
   208  	if traceRepo {
   209  		defer logCall("Lookup(%q, %q)", proxy, path)()
   210  	}
   211  
   212  	return lookupCache.Do(lookupCacheKey{proxy, path}, func() Repo {
   213  		return newCachingRepo(ctx, path, func(ctx context.Context) (Repo, error) {
   214  			r, err := lookup(ctx, proxy, path)
   215  			if err == nil && traceRepo {
   216  				r = newLoggingRepo(r)
   217  			}
   218  			return r, err
   219  		})
   220  	})
   221  }
   222  
   223  var lookupLocalCache par.Cache[string, Repo] // path, Repo
   224  
   225  // LookupLocal will only use local VCS information to fetch the Repo.
   226  func LookupLocal(ctx context.Context, path string) Repo {
   227  	if traceRepo {
   228  		defer logCall("LookupLocal(%q)", path)()
   229  	}
   230  
   231  	return lookupLocalCache.Do(path, func() Repo {
   232  		return newCachingRepo(ctx, path, func(ctx context.Context) (Repo, error) {
   233  			repoDir, vcsCmd, err := vcs.FromDir(path, "", true)
   234  			if err != nil {
   235  				return nil, err
   236  			}
   237  			code, err := lookupCodeRepo(ctx, &vcs.RepoRoot{Repo: repoDir, Root: repoDir, VCS: vcsCmd}, true)
   238  			if err != nil {
   239  				return nil, err
   240  			}
   241  			r, err := newCodeRepo(code, repoDir, path)
   242  			if err == nil && traceRepo {
   243  				r = newLoggingRepo(r)
   244  			}
   245  			return r, err
   246  		})
   247  	})
   248  }
   249  
   250  // lookup returns the module with the given module path.
   251  func lookup(ctx context.Context, proxy, path string) (r Repo, err error) {
   252  	if cfg.BuildMod == "vendor" {
   253  		return nil, errLookupDisabled
   254  	}
   255  
   256  	switch path {
   257  	case "go", "toolchain":
   258  		return &toolchainRepo{path, Lookup(ctx, proxy, "golang.org/toolchain")}, nil
   259  	}
   260  
   261  	if module.MatchPrefixPatterns(cfg.GONOPROXY, path) {
   262  		switch proxy {
   263  		case "noproxy", "direct":
   264  			return lookupDirect(ctx, path)
   265  		default:
   266  			return nil, errNoproxy
   267  		}
   268  	}
   269  
   270  	switch proxy {
   271  	case "off":
   272  		return errRepo{path, errProxyOff}, nil
   273  	case "direct":
   274  		return lookupDirect(ctx, path)
   275  	case "noproxy":
   276  		return nil, errUseProxy
   277  	default:
   278  		return newProxyRepo(proxy, path)
   279  	}
   280  }
   281  
   282  type lookupDisabledError struct{}
   283  
   284  func (lookupDisabledError) Error() string {
   285  	if cfg.BuildModReason == "" {
   286  		return fmt.Sprintf("module lookup disabled by -mod=%s", cfg.BuildMod)
   287  	}
   288  	return fmt.Sprintf("module lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason)
   289  }
   290  
   291  var errLookupDisabled error = lookupDisabledError{}
   292  
   293  var (
   294  	errProxyOff       = notExistErrorf("module lookup disabled by GOPROXY=off")
   295  	errNoproxy  error = notExistErrorf("disabled by GOPRIVATE/GONOPROXY")
   296  	errUseProxy error = notExistErrorf("path does not match GOPRIVATE/GONOPROXY")
   297  )
   298  
   299  func lookupDirect(ctx context.Context, path string) (Repo, error) {
   300  	security := web.SecureOnly
   301  
   302  	if module.MatchPrefixPatterns(cfg.GOINSECURE, path) {
   303  		security = web.Insecure
   304  	}
   305  	rr, err := vcs.RepoRootForImportPath(path, vcs.PreferMod, security)
   306  	if err != nil {
   307  		// We don't know where to find code for a module with this path.
   308  		return nil, notExistError{err: err}
   309  	}
   310  
   311  	if rr.VCS.Name == "mod" {
   312  		// Fetch module from proxy with base URL rr.Repo.
   313  		return newProxyRepo(rr.Repo, path)
   314  	}
   315  
   316  	code, err := lookupCodeRepo(ctx, rr, false)
   317  	if err != nil {
   318  		return nil, err
   319  	}
   320  	return newCodeRepo(code, rr.Root, path)
   321  }
   322  
   323  func lookupCodeRepo(ctx context.Context, rr *vcs.RepoRoot, local bool) (codehost.Repo, error) {
   324  	code, err := codehost.NewRepo(ctx, rr.VCS.Cmd, rr.Repo, local)
   325  	if err != nil {
   326  		if _, ok := err.(*codehost.VCSError); ok {
   327  			return nil, err
   328  		}
   329  		return nil, fmt.Errorf("lookup %s: %v", rr.Root, err)
   330  	}
   331  	return code, nil
   332  }
   333  
   334  // A loggingRepo is a wrapper around an underlying Repo
   335  // that prints a log message at the start and end of each call.
   336  // It can be inserted when debugging.
   337  type loggingRepo struct {
   338  	r Repo
   339  }
   340  
   341  func newLoggingRepo(r Repo) *loggingRepo {
   342  	return &loggingRepo{r}
   343  }
   344  
   345  // logCall prints a log message using format and args and then
   346  // also returns a function that will print the same message again,
   347  // along with the elapsed time.
   348  // Typical usage is:
   349  //
   350  //	defer logCall("hello %s", arg)()
   351  //
   352  // Note the final ().
   353  func logCall(format string, args ...any) func() {
   354  	start := time.Now()
   355  	fmt.Fprintf(os.Stderr, "+++ %s\n", fmt.Sprintf(format, args...))
   356  	return func() {
   357  		fmt.Fprintf(os.Stderr, "%.3fs %s\n", time.Since(start).Seconds(), fmt.Sprintf(format, args...))
   358  	}
   359  }
   360  
   361  func (l *loggingRepo) ModulePath() string {
   362  	return l.r.ModulePath()
   363  }
   364  
   365  func (l *loggingRepo) CheckReuse(ctx context.Context, old *codehost.Origin) (err error) {
   366  	defer func() {
   367  		logCall("CheckReuse[%s]: %v", l.r.ModulePath(), err)
   368  	}()
   369  	return l.r.CheckReuse(ctx, old)
   370  }
   371  
   372  func (l *loggingRepo) Versions(ctx context.Context, prefix string) (*Versions, error) {
   373  	defer logCall("Repo[%s]: Versions(%q)", l.r.ModulePath(), prefix)()
   374  	return l.r.Versions(ctx, prefix)
   375  }
   376  
   377  func (l *loggingRepo) Stat(ctx context.Context, rev string) (*RevInfo, error) {
   378  	defer logCall("Repo[%s]: Stat(%q)", l.r.ModulePath(), rev)()
   379  	return l.r.Stat(ctx, rev)
   380  }
   381  
   382  func (l *loggingRepo) Latest(ctx context.Context) (*RevInfo, error) {
   383  	defer logCall("Repo[%s]: Latest()", l.r.ModulePath())()
   384  	return l.r.Latest(ctx)
   385  }
   386  
   387  func (l *loggingRepo) GoMod(ctx context.Context, version string) ([]byte, error) {
   388  	defer logCall("Repo[%s]: GoMod(%q)", l.r.ModulePath(), version)()
   389  	return l.r.GoMod(ctx, version)
   390  }
   391  
   392  func (l *loggingRepo) Zip(ctx context.Context, dst io.Writer, version string) error {
   393  	dstName := "_"
   394  	if dst, ok := dst.(interface{ Name() string }); ok {
   395  		dstName = strconv.Quote(dst.Name())
   396  	}
   397  	defer logCall("Repo[%s]: Zip(%s, %q)", l.r.ModulePath(), dstName, version)()
   398  	return l.r.Zip(ctx, dst, version)
   399  }
   400  
   401  // errRepo is a Repo that returns the same error for all operations.
   402  //
   403  // It is useful in conjunction with caching, since cache hits will not attempt
   404  // the prohibited operations.
   405  type errRepo struct {
   406  	modulePath string
   407  	err        error
   408  }
   409  
   410  func (r errRepo) ModulePath() string { return r.modulePath }
   411  
   412  func (r errRepo) CheckReuse(ctx context.Context, old *codehost.Origin) error     { return r.err }
   413  func (r errRepo) Versions(ctx context.Context, prefix string) (*Versions, error) { return nil, r.err }
   414  func (r errRepo) Stat(ctx context.Context, rev string) (*RevInfo, error)         { return nil, r.err }
   415  func (r errRepo) Latest(ctx context.Context) (*RevInfo, error)                   { return nil, r.err }
   416  func (r errRepo) GoMod(ctx context.Context, version string) ([]byte, error)      { return nil, r.err }
   417  func (r errRepo) Zip(ctx context.Context, dst io.Writer, version string) error   { return r.err }
   418  
   419  // A notExistError is like fs.ErrNotExist, but with a custom message
   420  type notExistError struct {
   421  	err error
   422  }
   423  
   424  func notExistErrorf(format string, args ...any) error {
   425  	return notExistError{fmt.Errorf(format, args...)}
   426  }
   427  
   428  func (e notExistError) Error() string {
   429  	return e.err.Error()
   430  }
   431  
   432  func (notExistError) Is(target error) bool {
   433  	return target == fs.ErrNotExist
   434  }
   435  
   436  func (e notExistError) Unwrap() error {
   437  	return e.err
   438  }
   439
View as plain text