Source file src/cmd/go/internal/search/search.go

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package search
     6  
     7  import (
     8  	"cmd/go/internal/base"
     9  	"cmd/go/internal/cfg"
    10  	"cmd/go/internal/fsys"
    11  	"cmd/go/internal/str"
    12  	"cmd/internal/pkgpattern"
    13  	"fmt"
    14  	"go/build"
    15  	"io/fs"
    16  	"os"
    17  	"path"
    18  	"path/filepath"
    19  	"strings"
    20  
    21  	"golang.org/x/mod/modfile"
    22  )
    23  
    24  // A Match represents the result of matching a single package pattern.
    25  type Match struct {
    26  	pattern string   // the pattern itself
    27  	Dirs    []string // if the pattern is local, directories that potentially contain matching packages
    28  	Pkgs    []string // matching packages (import paths)
    29  	Errs    []error  // errors matching the patterns to packages, NOT errors loading those packages
    30  
    31  	// Errs may be non-empty even if len(Pkgs) > 0, indicating that some matching
    32  	// packages could be located but results may be incomplete.
    33  	// If len(Pkgs) == 0 && len(Errs) == 0, the pattern is well-formed but did not
    34  	// match any packages.
    35  }
    36  
    37  // NewMatch returns a Match describing the given pattern,
    38  // without resolving its packages or errors.
    39  func NewMatch(pattern string) *Match {
    40  	return &Match{pattern: pattern}
    41  }
    42  
    43  // Pattern returns the pattern to be matched.
    44  func (m *Match) Pattern() string { return m.pattern }
    45  
    46  // AddError appends a MatchError wrapping err to m.Errs.
    47  func (m *Match) AddError(err error) {
    48  	m.Errs = append(m.Errs, &MatchError{Match: m, Err: err})
    49  }
    50  
    51  // IsLiteral reports whether the pattern is free of wildcards and meta-patterns.
    52  //
    53  // A literal pattern must match at most one package.
    54  func (m *Match) IsLiteral() bool {
    55  	return !strings.Contains(m.pattern, "...") && !m.IsMeta()
    56  }
    57  
    58  // IsLocal reports whether the pattern must be resolved from a specific root or
    59  // directory, such as a filesystem path or a single module.
    60  func (m *Match) IsLocal() bool {
    61  	return build.IsLocalImport(m.pattern) || filepath.IsAbs(m.pattern)
    62  }
    63  
    64  // IsMeta reports whether the pattern is a “meta-package” keyword that represents
    65  // multiple packages, such as "std", "cmd", "tool", "work", or "all".
    66  func (m *Match) IsMeta() bool {
    67  	return IsMetaPackage(m.pattern)
    68  }
    69  
    70  // IsMetaPackage checks if name is a reserved package name that expands to multiple packages.
    71  func IsMetaPackage(name string) bool {
    72  	return name == "std" || name == "cmd" || name == "tool" || name == "work" || name == "all"
    73  }
    74  
    75  // A MatchError indicates an error that occurred while attempting to match a
    76  // pattern.
    77  type MatchError struct {
    78  	Match *Match
    79  	Err   error
    80  }
    81  
    82  func (e *MatchError) Error() string {
    83  	if e.Match.IsLiteral() {
    84  		return fmt.Sprintf("%s: %v", e.Match.Pattern(), e.Err)
    85  	}
    86  	return fmt.Sprintf("pattern %s: %v", e.Match.Pattern(), e.Err)
    87  }
    88  
    89  func (e *MatchError) Unwrap() error {
    90  	return e.Err
    91  }
    92  
    93  // MatchPackages sets m.Pkgs to a non-nil slice containing all the packages that
    94  // can be found under the $GOPATH directories and $GOROOT that match the
    95  // pattern. The pattern must be either "all" (all packages), "std" (standard
    96  // packages), "cmd" (standard commands), or a path including "...".
    97  //
    98  // If any errors may have caused the set of packages to be incomplete,
    99  // MatchPackages appends those errors to m.Errs.
   100  func (m *Match) MatchPackages() {
   101  	m.Pkgs = []string{}
   102  	if m.IsLocal() {
   103  		m.AddError(fmt.Errorf("internal error: MatchPackages: %s is not a valid package pattern", m.pattern))
   104  		return
   105  	}
   106  
   107  	if m.IsLiteral() {
   108  		m.Pkgs = []string{m.pattern}
   109  		return
   110  	}
   111  
   112  	match := func(string) bool { return true }
   113  	treeCanMatch := func(string) bool { return true }
   114  	if !m.IsMeta() {
   115  		match = pkgpattern.MatchPattern(m.pattern)
   116  		treeCanMatch = pkgpattern.TreeCanMatchPattern(m.pattern)
   117  	}
   118  
   119  	have := map[string]bool{
   120  		"builtin": true, // ignore pseudo-package that exists only for documentation
   121  	}
   122  	if !cfg.BuildContext.CgoEnabled {
   123  		have["runtime/cgo"] = true // ignore during walk
   124  	}
   125  
   126  	for _, src := range cfg.BuildContext.SrcDirs() {
   127  		if (m.pattern == "std" || m.pattern == "cmd") && src != cfg.GOROOTsrc {
   128  			continue
   129  		}
   130  
   131  		// If the root itself is a symlink to a directory,
   132  		// we want to follow it (see https://go.dev/issue/50807).
   133  		// Add a trailing separator to force that to happen.
   134  		src = str.WithFilePathSeparator(filepath.Clean(src))
   135  		root := src
   136  		if m.pattern == "cmd" {
   137  			root += "cmd" + string(filepath.Separator)
   138  		}
   139  
   140  		err := fsys.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
   141  			if err != nil {
   142  				return err // Likely a permission error, which could interfere with matching.
   143  			}
   144  			if path == src {
   145  				return nil // GOROOT/src and GOPATH/src cannot contain packages.
   146  			}
   147  
   148  			want := true
   149  			// Avoid .foo, _foo, and testdata directory trees.
   150  			_, elem := filepath.Split(path)
   151  			if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" {
   152  				want = false
   153  			}
   154  
   155  			name := filepath.ToSlash(path[len(src):])
   156  			if m.pattern == "std" && (!IsStandardImportPath(name) || name == "cmd") {
   157  				// The name "std" is only the standard library.
   158  				// If the name is cmd, it's the root of the command tree.
   159  				want = false
   160  			}
   161  			if !treeCanMatch(name) {
   162  				want = false
   163  			}
   164  
   165  			if !d.IsDir() {
   166  				if d.Type()&fs.ModeSymlink != 0 && want && strings.Contains(m.pattern, "...") {
   167  					if target, err := fsys.Stat(path); err == nil && target.IsDir() {
   168  						fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path)
   169  					}
   170  				}
   171  				return nil
   172  			}
   173  			if !want {
   174  				return filepath.SkipDir
   175  			}
   176  
   177  			if have[name] {
   178  				return nil
   179  			}
   180  			have[name] = true
   181  			if !match(name) {
   182  				return nil
   183  			}
   184  			pkg, err := cfg.BuildContext.ImportDir(path, 0)
   185  			if err != nil {
   186  				if _, noGo := err.(*build.NoGoError); noGo {
   187  					// The package does not actually exist, so record neither the package
   188  					// nor the error.
   189  					return nil
   190  				}
   191  				// There was an error importing path, but not matching it,
   192  				// which is all that Match promises to do.
   193  				// Ignore the import error.
   194  			}
   195  
   196  			// If we are expanding "cmd", skip main
   197  			// packages under cmd/vendor. At least as of
   198  			// March, 2017, there is one there for the
   199  			// vendored pprof tool.
   200  			if m.pattern == "cmd" && pkg != nil && strings.HasPrefix(pkg.ImportPath, "cmd/vendor") && pkg.Name == "main" {
   201  				return nil
   202  			}
   203  
   204  			m.Pkgs = append(m.Pkgs, name)
   205  			return nil
   206  		})
   207  		if err != nil {
   208  			m.AddError(err)
   209  		}
   210  	}
   211  }
   212  
   213  // IgnorePatterns is normalized with normalizePath.
   214  type IgnorePatterns struct {
   215  	relativePatterns []string
   216  	anyPatterns      []string
   217  }
   218  
   219  // ShouldIgnore returns true if the given directory should be ignored
   220  // based on the ignore patterns.
   221  //
   222  // An ignore pattern "x" will cause any file or directory named "x"
   223  // (and its entire subtree) to be ignored, regardless of its location
   224  // within the module.
   225  //
   226  // An ignore pattern "./x" will only cause the specific file or directory
   227  // named "x" at the root of the module to be ignored.
   228  // Wildcards in ignore patterns are not supported.
   229  func (ignorePatterns *IgnorePatterns) ShouldIgnore(dir string) bool {
   230  	if dir == "" {
   231  		return false
   232  	}
   233  	dir = normalizePath(dir)
   234  	for _, pattern := range ignorePatterns.relativePatterns {
   235  		if strings.HasPrefix(dir, pattern) {
   236  			return true
   237  		}
   238  	}
   239  	for _, pattern := range ignorePatterns.anyPatterns {
   240  		if strings.Contains(dir, pattern) {
   241  			return true
   242  		}
   243  	}
   244  	return false
   245  }
   246  
   247  func NewIgnorePatterns(patterns []string) *IgnorePatterns {
   248  	var relativePatterns, anyPatterns []string
   249  	for _, pattern := range patterns {
   250  		ignorePatternPath, isRelative := strings.CutPrefix(pattern, "./")
   251  		ignorePatternPath = normalizePath(ignorePatternPath)
   252  		if isRelative {
   253  			relativePatterns = append(relativePatterns, ignorePatternPath)
   254  		} else {
   255  			anyPatterns = append(anyPatterns, ignorePatternPath)
   256  		}
   257  	}
   258  	return &IgnorePatterns{
   259  		relativePatterns: relativePatterns,
   260  		anyPatterns:      anyPatterns,
   261  	}
   262  }
   263  
   264  // normalizePath adds slashes to the front and end of the given path.
   265  func normalizePath(path string) string {
   266  	path = filepath.ToSlash(path)
   267  	if !strings.HasPrefix(path, "/") {
   268  		path = "/" + path
   269  	}
   270  	if !strings.HasSuffix(path, "/") {
   271  		path += "/"
   272  	}
   273  	return path
   274  }
   275  
   276  // MatchDirs sets m.Dirs to a non-nil slice containing all directories that
   277  // potentially match a local pattern. The pattern must begin with an absolute
   278  // path, or "./", or "../". On Windows, the pattern may use slash or backslash
   279  // separators or a mix of both.
   280  //
   281  // If any errors may have caused the set of directories to be incomplete,
   282  // MatchDirs appends those errors to m.Errs.
   283  func (m *Match) MatchDirs(modRoots []string) {
   284  	m.Dirs = []string{}
   285  	if !m.IsLocal() {
   286  		m.AddError(fmt.Errorf("internal error: MatchDirs: %s is not a valid filesystem pattern", m.pattern))
   287  		return
   288  	}
   289  
   290  	if m.IsLiteral() {
   291  		m.Dirs = []string{m.pattern}
   292  		return
   293  	}
   294  
   295  	// Clean the path and create a matching predicate.
   296  	// filepath.Clean removes "./" prefixes (and ".\" on Windows). We need to
   297  	// preserve these, since they are meaningful in MatchPattern and in
   298  	// returned import paths.
   299  	cleanPattern := filepath.Clean(m.pattern)
   300  	isLocal := strings.HasPrefix(m.pattern, "./") || (os.PathSeparator == '\\' && strings.HasPrefix(m.pattern, `.\`))
   301  	prefix := ""
   302  	if cleanPattern != "." && isLocal {
   303  		prefix = "./"
   304  		cleanPattern = "." + string(os.PathSeparator) + cleanPattern
   305  	}
   306  	slashPattern := filepath.ToSlash(cleanPattern)
   307  	match := pkgpattern.MatchPattern(slashPattern)
   308  
   309  	// Find directory to begin the scan.
   310  	// Could be smarter but this one optimization
   311  	// is enough for now, since ... is usually at the
   312  	// end of a path.
   313  	i := strings.Index(cleanPattern, "...")
   314  	dir, _ := filepath.Split(cleanPattern[:i])
   315  
   316  	// pattern begins with ./ or ../.
   317  	// path.Clean will discard the ./ but not the ../.
   318  	// We need to preserve the ./ for pattern matching
   319  	// and in the returned import paths.
   320  
   321  	var modRoot string
   322  	if len(modRoots) > 0 {
   323  		abs, err := filepath.Abs(dir)
   324  		if err != nil {
   325  			m.AddError(err)
   326  			return
   327  		}
   328  		var found bool
   329  		for _, mr := range modRoots {
   330  			if mr != "" && str.HasFilePathPrefix(abs, mr) {
   331  				found = true
   332  				modRoot = mr
   333  			}
   334  		}
   335  		if !found {
   336  			plural := ""
   337  			if len(modRoots) > 1 {
   338  				plural = "s"
   339  			}
   340  			m.AddError(fmt.Errorf("directory %s is outside module root%s (%s)", abs, plural, strings.Join(modRoots, ", ")))
   341  		}
   342  	}
   343  
   344  	ignorePatterns := parseIgnorePatterns(modRoot)
   345  	// If dir is actually a symlink to a directory,
   346  	// we want to follow it (see https://go.dev/issue/50807).
   347  	// Add a trailing separator to force that to happen.
   348  	dir = str.WithFilePathSeparator(dir)
   349  	err := fsys.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
   350  		if err != nil {
   351  			return err // Likely a permission error, which could interfere with matching.
   352  		}
   353  		if !d.IsDir() {
   354  			return nil
   355  		}
   356  		top := false
   357  		if path == dir {
   358  			// Walk starts at dir and recurses. For the recursive case,
   359  			// the path is the result of filepath.Join, which calls filepath.Clean.
   360  			// The initial case is not Cleaned, though, so we do this explicitly.
   361  			//
   362  			// This converts a path like "./io/" to "io". Without this step, running
   363  			// "cd $GOROOT/src; go list ./io/..." would incorrectly skip the io
   364  			// package, because prepending the prefix "./" to the unclean path would
   365  			// result in "././io", and match("././io") returns false.
   366  			top = true
   367  			path = filepath.Clean(path)
   368  		}
   369  
   370  		// Avoid .foo, _foo, and testdata directory trees, but do not avoid "." or "..".
   371  		_, elem := filepath.Split(path)
   372  		dot := strings.HasPrefix(elem, ".") && elem != "." && elem != ".."
   373  		if dot || strings.HasPrefix(elem, "_") || elem == "testdata" {
   374  			return filepath.SkipDir
   375  		}
   376  		absPath, err := filepath.Abs(path)
   377  		if err != nil {
   378  			return err
   379  		}
   380  
   381  		if ignorePatterns != nil && ignorePatterns.ShouldIgnore(InDir(absPath, modRoot)) {
   382  			if cfg.BuildX {
   383  				fmt.Fprintf(os.Stderr, "# ignoring directory %s\n", absPath)
   384  			}
   385  			return filepath.SkipDir
   386  		}
   387  
   388  		if !top && cfg.ModulesEnabled {
   389  			// Ignore other modules found in subdirectories.
   390  			if info, err := fsys.Stat(filepath.Join(path, "go.mod")); err == nil && !info.IsDir() {
   391  				return filepath.SkipDir
   392  			}
   393  		}
   394  
   395  		name := prefix + filepath.ToSlash(path)
   396  		if !match(name) {
   397  			return nil
   398  		}
   399  
   400  		// We keep the directory if we can import it, or if we can't import it
   401  		// due to invalid Go source files. This means that directories containing
   402  		// parse errors will be built (and fail) instead of being silently skipped
   403  		// as not matching the pattern. Go 1.5 and earlier skipped, but that
   404  		// behavior means people miss serious mistakes.
   405  		// See golang.org/issue/11407.
   406  		if p, err := cfg.BuildContext.ImportDir(path, 0); err != nil && (p == nil || len(p.InvalidGoFiles) == 0) {
   407  			if _, noGo := err.(*build.NoGoError); noGo {
   408  				// The package does not actually exist, so record neither the package
   409  				// nor the error.
   410  				return nil
   411  			}
   412  			// There was an error importing path, but not matching it,
   413  			// which is all that Match promises to do.
   414  			// Ignore the import error.
   415  		}
   416  		m.Dirs = append(m.Dirs, name)
   417  		return nil
   418  	})
   419  	if err != nil {
   420  		m.AddError(err)
   421  	}
   422  }
   423  
   424  // WarnUnmatched warns about patterns that didn't match any packages.
   425  func WarnUnmatched(matches []*Match) {
   426  	for _, m := range matches {
   427  		if len(m.Pkgs) == 0 && len(m.Errs) == 0 {
   428  			fmt.Fprintf(os.Stderr, "go: warning: %q matched no packages\n", m.pattern)
   429  		}
   430  	}
   431  }
   432  
   433  // ImportPaths returns the matching paths to use for the given command line.
   434  // It calls ImportPathsQuiet and then WarnUnmatched.
   435  func ImportPaths(patterns []string) []*Match {
   436  	matches := ImportPathsQuiet(patterns)
   437  	WarnUnmatched(matches)
   438  	return matches
   439  }
   440  
   441  // ImportPathsQuiet is like ImportPaths but does not warn about patterns with no matches.
   442  func ImportPathsQuiet(patterns []string) []*Match {
   443  	patterns = CleanPatterns(patterns)
   444  	out := make([]*Match, 0, len(patterns))
   445  	for _, a := range patterns {
   446  		m := NewMatch(a)
   447  		if m.IsLocal() {
   448  			m.MatchDirs(nil)
   449  
   450  			// Change the file import path to a regular import path if the package
   451  			// is in GOPATH or GOROOT. We don't report errors here; LoadImport
   452  			// (or something similar) will report them later.
   453  			m.Pkgs = make([]string, len(m.Dirs))
   454  			for i, dir := range m.Dirs {
   455  				absDir := dir
   456  				if !filepath.IsAbs(dir) {
   457  					absDir = filepath.Join(base.Cwd(), dir)
   458  				}
   459  				if bp, _ := cfg.BuildContext.ImportDir(absDir, build.FindOnly); bp.ImportPath != "" && bp.ImportPath != "." {
   460  					m.Pkgs[i] = bp.ImportPath
   461  				} else {
   462  					m.Pkgs[i] = dir
   463  				}
   464  			}
   465  		} else {
   466  			m.MatchPackages()
   467  		}
   468  
   469  		out = append(out, m)
   470  	}
   471  	return out
   472  }
   473  
   474  // CleanPatterns returns the patterns to use for the given command line. It
   475  // canonicalizes the patterns but does not evaluate any matches. For patterns
   476  // that are not local or absolute paths, it preserves text after '@' to avoid
   477  // modifying version queries.
   478  func CleanPatterns(patterns []string) []string {
   479  	if len(patterns) == 0 {
   480  		return []string{"."}
   481  	}
   482  	out := make([]string, 0, len(patterns))
   483  	for _, a := range patterns {
   484  		var p, v string
   485  		if build.IsLocalImport(a) || filepath.IsAbs(a) {
   486  			p = a
   487  		} else if i := strings.IndexByte(a, '@'); i < 0 {
   488  			p = a
   489  		} else {
   490  			p = a[:i]
   491  			v = a[i:]
   492  		}
   493  
   494  		// Arguments may be either file paths or import paths.
   495  		// As a courtesy to Windows developers, rewrite \ to /
   496  		// in arguments that look like import paths.
   497  		// Don't replace slashes in absolute paths.
   498  		if filepath.IsAbs(p) {
   499  			p = filepath.Clean(p)
   500  		} else {
   501  			p = strings.ReplaceAll(p, `\`, `/`)
   502  
   503  			// Put argument in canonical form, but preserve leading ./.
   504  			if strings.HasPrefix(p, "./") {
   505  				p = "./" + path.Clean(p)
   506  				if p == "./." {
   507  					p = "."
   508  				}
   509  			} else {
   510  				p = path.Clean(p)
   511  			}
   512  		}
   513  
   514  		out = append(out, p+v)
   515  	}
   516  	return out
   517  }
   518  
   519  // IsStandardImportPath reports whether $GOROOT/src/path should be considered
   520  // part of the standard distribution. For historical reasons we allow people to add
   521  // their own code to $GOROOT instead of using $GOPATH, but we assume that
   522  // code will start with a domain name (dot in the first element).
   523  //
   524  // Note that this function is meant to evaluate whether a directory found in GOROOT
   525  // should be treated as part of the standard library. It should not be used to decide
   526  // that a directory found in GOPATH should be rejected: directories in GOPATH
   527  // need not have dots in the first element, and they just take their chances
   528  // with future collisions in the standard library.
   529  func IsStandardImportPath(path string) bool {
   530  	i := strings.Index(path, "/")
   531  	if i < 0 {
   532  		i = len(path)
   533  	}
   534  	elem := path[:i]
   535  	return !strings.Contains(elem, ".")
   536  }
   537  
   538  // IsRelativePath reports whether pattern should be interpreted as a directory
   539  // path relative to the current directory, as opposed to a pattern matching
   540  // import paths.
   541  func IsRelativePath(pattern string) bool {
   542  	return strings.HasPrefix(pattern, "./") || strings.HasPrefix(pattern, "../") || pattern == "." || pattern == ".."
   543  }
   544  
   545  // InDir checks whether path is in the file tree rooted at dir.
   546  // If so, InDir returns an equivalent path relative to dir.
   547  // If not, InDir returns an empty string.
   548  // InDir makes some effort to succeed even in the presence of symbolic links.
   549  func InDir(path, dir string) string {
   550  	// inDirLex reports whether path is lexically in dir,
   551  	// without considering symbolic or hard links.
   552  	inDirLex := func(path, dir string) (string, bool) {
   553  		if dir == "" {
   554  			return path, true
   555  		}
   556  		rel := str.TrimFilePathPrefix(path, dir)
   557  		if rel == path {
   558  			return "", false
   559  		}
   560  		if rel == "" {
   561  			return ".", true
   562  		}
   563  		return rel, true
   564  	}
   565  
   566  	if rel, ok := inDirLex(path, dir); ok {
   567  		return rel
   568  	}
   569  	xpath, err := filepath.EvalSymlinks(path)
   570  	if err != nil || xpath == path {
   571  		xpath = ""
   572  	} else {
   573  		if rel, ok := inDirLex(xpath, dir); ok {
   574  			return rel
   575  		}
   576  	}
   577  
   578  	xdir, err := filepath.EvalSymlinks(dir)
   579  	if err == nil && xdir != dir {
   580  		if rel, ok := inDirLex(path, xdir); ok {
   581  			return rel
   582  		}
   583  		if xpath != "" {
   584  			if rel, ok := inDirLex(xpath, xdir); ok {
   585  				return rel
   586  			}
   587  		}
   588  	}
   589  	return ""
   590  }
   591  
   592  // parseIgnorePatterns reads the go.mod file at the given module root
   593  // and extracts the ignore patterns defined within it.
   594  // If modRoot is empty, it returns nil.
   595  func parseIgnorePatterns(modRoot string) *IgnorePatterns {
   596  	if modRoot == "" {
   597  		return nil
   598  	}
   599  	data, err := os.ReadFile(filepath.Join(modRoot, "go.mod"))
   600  	if err != nil {
   601  		return nil
   602  	}
   603  	modFile, err := modfile.Parse("go.mod", data, nil)
   604  	if err != nil {
   605  		return nil
   606  	}
   607  	var patterns []string
   608  	for _, i := range modFile.Ignore {
   609  		patterns = append(patterns, i.Path)
   610  	}
   611  	return NewIgnorePatterns(patterns)
   612  }
   613  

View as plain text