Source file src/cmd/api/main_test.go

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // This package computes the exported API of a set of Go packages.
     6  // It is only a test, not a command, nor a usefully importable package.
     7  
     8  package main
     9  
    10  import (
    11  	"bufio"
    12  	"bytes"
    13  	"encoding/json"
    14  	"fmt"
    15  	"go/ast"
    16  	"go/build"
    17  	"go/parser"
    18  	"go/token"
    19  	"go/types"
    20  	"internal/buildcfg"
    21  	"internal/testenv"
    22  	"io"
    23  	"log"
    24  	"os"
    25  	"os/exec"
    26  	"path/filepath"
    27  	"regexp"
    28  	"runtime"
    29  	"slices"
    30  	"strconv"
    31  	"strings"
    32  	"sync"
    33  	"testing"
    34  )
    35  
    36  const verbose = false
    37  
    38  func goCmd() string {
    39  	var exeSuffix string
    40  	if runtime.GOOS == "windows" {
    41  		exeSuffix = ".exe"
    42  	}
    43  	path := filepath.Join(testenv.GOROOT(nil), "bin", "go"+exeSuffix)
    44  	if _, err := os.Stat(path); err == nil {
    45  		return path
    46  	}
    47  	return "go"
    48  }
    49  
    50  // contexts are the default contexts which are scanned.
    51  var contexts = []*build.Context{
    52  	{GOOS: "linux", GOARCH: "386", CgoEnabled: true},
    53  	{GOOS: "linux", GOARCH: "386"},
    54  	{GOOS: "linux", GOARCH: "amd64", CgoEnabled: true},
    55  	{GOOS: "linux", GOARCH: "amd64"},
    56  	{GOOS: "linux", GOARCH: "arm", CgoEnabled: true},
    57  	{GOOS: "linux", GOARCH: "arm"},
    58  	{GOOS: "darwin", GOARCH: "amd64", CgoEnabled: true},
    59  	{GOOS: "darwin", GOARCH: "amd64"},
    60  	{GOOS: "darwin", GOARCH: "arm64", CgoEnabled: true},
    61  	{GOOS: "darwin", GOARCH: "arm64"},
    62  	{GOOS: "windows", GOARCH: "amd64"},
    63  	{GOOS: "windows", GOARCH: "386"},
    64  	{GOOS: "freebsd", GOARCH: "386", CgoEnabled: true},
    65  	{GOOS: "freebsd", GOARCH: "386"},
    66  	{GOOS: "freebsd", GOARCH: "amd64", CgoEnabled: true},
    67  	{GOOS: "freebsd", GOARCH: "amd64"},
    68  	{GOOS: "freebsd", GOARCH: "arm", CgoEnabled: true},
    69  	{GOOS: "freebsd", GOARCH: "arm"},
    70  	{GOOS: "freebsd", GOARCH: "arm64", CgoEnabled: true},
    71  	{GOOS: "freebsd", GOARCH: "arm64"},
    72  	{GOOS: "freebsd", GOARCH: "riscv64", CgoEnabled: true},
    73  	{GOOS: "freebsd", GOARCH: "riscv64"},
    74  	{GOOS: "netbsd", GOARCH: "386", CgoEnabled: true},
    75  	{GOOS: "netbsd", GOARCH: "386"},
    76  	{GOOS: "netbsd", GOARCH: "amd64", CgoEnabled: true},
    77  	{GOOS: "netbsd", GOARCH: "amd64"},
    78  	{GOOS: "netbsd", GOARCH: "arm", CgoEnabled: true},
    79  	{GOOS: "netbsd", GOARCH: "arm"},
    80  	{GOOS: "netbsd", GOARCH: "arm64", CgoEnabled: true},
    81  	{GOOS: "netbsd", GOARCH: "arm64"},
    82  	{GOOS: "openbsd", GOARCH: "386", CgoEnabled: true},
    83  	{GOOS: "openbsd", GOARCH: "386"},
    84  	{GOOS: "openbsd", GOARCH: "amd64", CgoEnabled: true},
    85  	{GOOS: "openbsd", GOARCH: "amd64"},
    86  }
    87  
    88  func contextName(c *build.Context) string {
    89  	s := c.GOOS + "-" + c.GOARCH
    90  	if c.CgoEnabled {
    91  		s += "-cgo"
    92  	}
    93  	if c.Dir != "" {
    94  		s += fmt.Sprintf(" [%s]", c.Dir)
    95  	}
    96  	return s
    97  }
    98  
    99  var internalPkg = regexp.MustCompile(`(^|/)internal($|/)`)
   100  
   101  var exitCode = 0
   102  
   103  func Check(t *testing.T) {
   104  	checkFiles, err := filepath.Glob(filepath.Join(testenv.GOROOT(t), "api/go1*.txt"))
   105  	if err != nil {
   106  		t.Fatal(err)
   107  	}
   108  
   109  	var nextFiles []string
   110  	if v := runtime.Version(); strings.Contains(v, "devel") || strings.Contains(v, "beta") {
   111  		next, err := filepath.Glob(filepath.Join(testenv.GOROOT(t), "api/next/*.txt"))
   112  		if err != nil {
   113  			t.Fatal(err)
   114  		}
   115  		nextFiles = next
   116  	}
   117  
   118  	for _, c := range contexts {
   119  		c.Compiler = build.Default.Compiler
   120  
   121  		// Include baseline goexperiment.* tool tags.
   122  		baseline, err := buildcfg.ParseGOEXPERIMENT(c.GOOS, c.GOARCH, "")
   123  		if err != nil {
   124  			t.Fatal(err)
   125  		}
   126  		for _, exp := range baseline.Enabled() {
   127  			c.ToolTags = append(c.ToolTags, "goexperiment."+exp)
   128  		}
   129  	}
   130  
   131  	walkers := make([]*Walker, len(contexts))
   132  	var wg sync.WaitGroup
   133  	for i, context := range contexts {
   134  		wg.Add(1)
   135  		go func() {
   136  			defer wg.Done()
   137  			walkers[i] = NewWalker(context, filepath.Join(testenv.GOROOT(t), "src"))
   138  		}()
   139  	}
   140  	wg.Wait()
   141  
   142  	var featureCtx = make(map[string]map[string]bool) // feature -> context name -> true
   143  	for _, w := range walkers {
   144  		for _, name := range w.stdPackages {
   145  			pkg, err := w.import_(name)
   146  			if _, nogo := err.(*build.NoGoError); nogo {
   147  				continue
   148  			}
   149  			if err != nil {
   150  				log.Fatalf("Import(%q): %v", name, err)
   151  			}
   152  			w.export(pkg)
   153  		}
   154  
   155  		ctxName := contextName(w.context)
   156  		for _, f := range w.Features() {
   157  			if featureCtx[f] == nil {
   158  				featureCtx[f] = make(map[string]bool)
   159  			}
   160  			featureCtx[f][ctxName] = true
   161  		}
   162  	}
   163  
   164  	var features []string
   165  	for f, cmap := range featureCtx {
   166  		if len(cmap) == len(contexts) {
   167  			features = append(features, f)
   168  			continue
   169  		}
   170  		comma := strings.Index(f, ",")
   171  		for cname := range cmap {
   172  			f2 := fmt.Sprintf("%s (%s)%s", f[:comma], cname, f[comma:])
   173  			features = append(features, f2)
   174  		}
   175  	}
   176  
   177  	bw := bufio.NewWriter(os.Stdout)
   178  	defer bw.Flush()
   179  
   180  	var required []string
   181  	for _, file := range checkFiles {
   182  		required = append(required, fileFeatures(file, needApproval(file))...)
   183  	}
   184  	for _, file := range nextFiles {
   185  		required = append(required, fileFeatures(file, true)...)
   186  	}
   187  	exception := fileFeatures(filepath.Join(testenv.GOROOT(t), "api/except.txt"), false)
   188  
   189  	if exitCode == 1 {
   190  		t.Errorf("API database problems found")
   191  	}
   192  	if !compareAPI(bw, features, required, exception) {
   193  		t.Errorf("API differences found")
   194  	}
   195  }
   196  
   197  // export emits the exported package features.
   198  func (w *Walker) export(pkg *apiPackage) {
   199  	if verbose {
   200  		log.Println(pkg)
   201  	}
   202  	pop := w.pushScope("pkg " + pkg.Path())
   203  	w.current = pkg
   204  	w.collectDeprecated()
   205  	scope := pkg.Scope()
   206  	for _, name := range scope.Names() {
   207  		if token.IsExported(name) {
   208  			w.emitObj(scope.Lookup(name))
   209  		}
   210  	}
   211  	pop()
   212  }
   213  
   214  func set(items []string) map[string]bool {
   215  	s := make(map[string]bool)
   216  	for _, v := range items {
   217  		s[v] = true
   218  	}
   219  	return s
   220  }
   221  
   222  var spaceParensRx = regexp.MustCompile(` \(\S+?\)`)
   223  
   224  func featureWithoutContext(f string) string {
   225  	if !strings.Contains(f, "(") {
   226  		return f
   227  	}
   228  	return spaceParensRx.ReplaceAllString(f, "")
   229  }
   230  
   231  // portRemoved reports whether the given port-specific API feature is
   232  // okay to no longer exist because its port was removed.
   233  func portRemoved(feature string) bool {
   234  	return strings.Contains(feature, "(darwin-386)") ||
   235  		strings.Contains(feature, "(darwin-386-cgo)")
   236  }
   237  
   238  func compareAPI(w io.Writer, features, required, exception []string) (ok bool) {
   239  	ok = true
   240  
   241  	featureSet := set(features)
   242  	exceptionSet := set(exception)
   243  
   244  	slices.Sort(features)
   245  	slices.Sort(required)
   246  
   247  	take := func(sl *[]string) string {
   248  		s := (*sl)[0]
   249  		*sl = (*sl)[1:]
   250  		return s
   251  	}
   252  
   253  	for len(features) > 0 || len(required) > 0 {
   254  		switch {
   255  		case len(features) == 0 || (len(required) > 0 && required[0] < features[0]):
   256  			feature := take(&required)
   257  			if exceptionSet[feature] {
   258  				// An "unfortunate" case: the feature was once
   259  				// included in the API (e.g. go1.txt), but was
   260  				// subsequently removed. These are already
   261  				// acknowledged by being in the file
   262  				// "api/except.txt". No need to print them out
   263  				// here.
   264  			} else if portRemoved(feature) {
   265  				// okay.
   266  			} else if featureSet[featureWithoutContext(feature)] {
   267  				// okay.
   268  			} else {
   269  				fmt.Fprintf(w, "-%s\n", feature)
   270  				ok = false // broke compatibility
   271  			}
   272  		case len(required) == 0 || (len(features) > 0 && required[0] > features[0]):
   273  			newFeature := take(&features)
   274  			fmt.Fprintf(w, "+%s\n", newFeature)
   275  			ok = false // feature not in api/next/*
   276  		default:
   277  			take(&required)
   278  			take(&features)
   279  		}
   280  	}
   281  
   282  	return ok
   283  }
   284  
   285  // aliasReplacer applies type aliases to earlier API files,
   286  // to avoid misleading negative results.
   287  // This makes all the references to os.FileInfo in go1.txt
   288  // be read as if they said fs.FileInfo, since os.FileInfo is now an alias.
   289  // If there are many of these, we could do a more general solution,
   290  // but for now the replacer is fine.
   291  var aliasReplacer = strings.NewReplacer(
   292  	"os.FileInfo", "fs.FileInfo",
   293  	"os.FileMode", "fs.FileMode",
   294  	"os.PathError", "fs.PathError",
   295  )
   296  
   297  func fileFeatures(filename string, needApproval bool) []string {
   298  	bs, err := os.ReadFile(filename)
   299  	if err != nil {
   300  		log.Fatal(err)
   301  	}
   302  	s := string(bs)
   303  
   304  	// Diagnose common mistakes people make,
   305  	// since there is no apifmt to format these files.
   306  	// The missing final newline is important for the
   307  	// final release step of cat next/*.txt >go1.X.txt.
   308  	// If the files don't end in full lines, the concatenation goes awry.
   309  	if strings.Contains(s, "\r") {
   310  		log.Printf("%s: contains CRLFs", filename)
   311  		exitCode = 1
   312  	}
   313  	if filepath.Base(filename) == "go1.4.txt" {
   314  		// No use for blank lines in api files, except go1.4.txt
   315  		// used them in a reasonable way and we should let it be.
   316  	} else if strings.HasPrefix(s, "\n") || strings.Contains(s, "\n\n") {
   317  		log.Printf("%s: contains a blank line", filename)
   318  		exitCode = 1
   319  	}
   320  	if s == "" {
   321  		log.Printf("%s: empty file", filename)
   322  		exitCode = 1
   323  	} else if s[len(s)-1] != '\n' {
   324  		log.Printf("%s: missing final newline", filename)
   325  		exitCode = 1
   326  	}
   327  	s = aliasReplacer.Replace(s)
   328  	lines := strings.Split(s, "\n")
   329  	var nonblank []string
   330  	for i, line := range lines {
   331  		line = strings.TrimSpace(line)
   332  		if line == "" || strings.HasPrefix(line, "#") {
   333  			continue
   334  		}
   335  		if needApproval {
   336  			feature, approval, ok := strings.Cut(line, "#")
   337  			if !ok {
   338  				log.Printf("%s:%d: missing proposal approval\n", filename, i+1)
   339  				exitCode = 1
   340  			} else {
   341  				_, err := strconv.Atoi(approval)
   342  				if err != nil {
   343  					log.Printf("%s:%d: malformed proposal approval #%s\n", filename, i+1, approval)
   344  					exitCode = 1
   345  				}
   346  			}
   347  			line = strings.TrimSpace(feature)
   348  		} else {
   349  			if strings.Contains(line, " #") {
   350  				log.Printf("%s:%d: unexpected approval\n", filename, i+1)
   351  				exitCode = 1
   352  			}
   353  		}
   354  		nonblank = append(nonblank, line)
   355  	}
   356  	return nonblank
   357  }
   358  
   359  var fset = token.NewFileSet()
   360  
   361  type Walker struct {
   362  	context     *build.Context
   363  	root        string
   364  	scope       []string
   365  	current     *apiPackage
   366  	deprecated  map[token.Pos]bool
   367  	features    map[string]bool              // set
   368  	imported    map[string]*apiPackage       // packages already imported
   369  	stdPackages []string                     // names, omitting "unsafe", internal, and vendored packages
   370  	importMap   map[string]map[string]string // importer dir -> import path -> canonical path
   371  	importDir   map[string]string            // canonical import path -> dir
   372  
   373  }
   374  
   375  func NewWalker(context *build.Context, root string) *Walker {
   376  	w := &Walker{
   377  		context:  context,
   378  		root:     root,
   379  		features: map[string]bool{},
   380  		imported: map[string]*apiPackage{"unsafe": &apiPackage{Package: types.Unsafe}},
   381  	}
   382  	w.loadImports()
   383  	return w
   384  }
   385  
   386  func (w *Walker) Features() (fs []string) {
   387  	for f := range w.features {
   388  		fs = append(fs, f)
   389  	}
   390  	slices.Sort(fs)
   391  	return
   392  }
   393  
   394  var parsedFileCache = make(map[string]*ast.File)
   395  
   396  func (w *Walker) parseFile(dir, file string) (*ast.File, error) {
   397  	filename := filepath.Join(dir, file)
   398  	if f := parsedFileCache[filename]; f != nil {
   399  		return f, nil
   400  	}
   401  
   402  	f, err := parser.ParseFile(fset, filename, nil, parser.ParseComments|parser.SkipObjectResolution)
   403  	if err != nil {
   404  		return nil, err
   405  	}
   406  	parsedFileCache[filename] = f
   407  
   408  	return f, nil
   409  }
   410  
   411  // Disable before debugging non-obvious errors from the type-checker.
   412  const usePkgCache = true
   413  
   414  var (
   415  	pkgCache = map[string]*apiPackage{} // map tagKey to package
   416  	pkgTags  = map[string][]string{}    // map import dir to list of relevant tags
   417  )
   418  
   419  // tagKey returns the tag-based key to use in the pkgCache.
   420  // It is a comma-separated string; the first part is dir, the rest tags.
   421  // The satisfied tags are derived from context but only those that
   422  // matter (the ones listed in the tags argument plus GOOS and GOARCH) are used.
   423  // The tags list, which came from go/build's Package.AllTags,
   424  // is known to be sorted.
   425  func tagKey(dir string, context *build.Context, tags []string) string {
   426  	ctags := map[string]bool{
   427  		context.GOOS:   true,
   428  		context.GOARCH: true,
   429  	}
   430  	if context.CgoEnabled {
   431  		ctags["cgo"] = true
   432  	}
   433  	for _, tag := range context.BuildTags {
   434  		ctags[tag] = true
   435  	}
   436  	// TODO: ReleaseTags (need to load default)
   437  	key := dir
   438  
   439  	// explicit on GOOS and GOARCH as global cache will use "all" cached packages for
   440  	// an indirect imported package. See https://github.com/golang/go/issues/21181
   441  	// for more detail.
   442  	tags = append(tags, context.GOOS, context.GOARCH)
   443  	slices.Sort(tags)
   444  
   445  	for _, tag := range tags {
   446  		if ctags[tag] {
   447  			key += "," + tag
   448  			ctags[tag] = false
   449  		}
   450  	}
   451  	return key
   452  }
   453  
   454  type listImports struct {
   455  	stdPackages []string                     // names, omitting "unsafe", internal, and vendored packages
   456  	importDir   map[string]string            // canonical import path → directory
   457  	importMap   map[string]map[string]string // import path → canonical import path
   458  }
   459  
   460  var listCache sync.Map // map[string]listImports, keyed by contextName
   461  
   462  // listSem is a semaphore restricting concurrent invocations of 'go list'. 'go
   463  // list' has its own internal concurrency, so we use a hard-coded constant (to
   464  // allow the I/O-intensive phases of 'go list' to overlap) instead of scaling
   465  // all the way up to GOMAXPROCS.
   466  var listSem = make(chan semToken, 2)
   467  
   468  type semToken struct{}
   469  
   470  // loadImports populates w with information about the packages in the standard
   471  // library and the packages they themselves import in w's build context.
   472  //
   473  // The source import path and expanded import path are identical except for vendored packages.
   474  // For example, on return:
   475  //
   476  //	w.importMap["math"] = "math"
   477  //	w.importDir["math"] = "<goroot>/src/math"
   478  //
   479  //	w.importMap["golang.org/x/net/route"] = "vendor/golang.org/x/net/route"
   480  //	w.importDir["vendor/golang.org/x/net/route"] = "<goroot>/src/vendor/golang.org/x/net/route"
   481  //
   482  // Since the set of packages that exist depends on context, the result of
   483  // loadImports also depends on context. However, to improve test running time
   484  // the configuration for each environment is cached across runs.
   485  func (w *Walker) loadImports() {
   486  	if w.context == nil {
   487  		return // test-only Walker; does not use the import map
   488  	}
   489  
   490  	name := contextName(w.context)
   491  
   492  	imports, ok := listCache.Load(name)
   493  	if !ok {
   494  		listSem <- semToken{}
   495  		defer func() { <-listSem }()
   496  
   497  		cmd := exec.Command(goCmd(), "list", "-e", "-deps", "-json", "std")
   498  		cmd.Env = listEnv(w.context)
   499  		if w.context.Dir != "" {
   500  			cmd.Dir = w.context.Dir
   501  		}
   502  		cmd.Stderr = os.Stderr
   503  		out, err := cmd.Output()
   504  		if err != nil {
   505  			log.Fatalf("loading imports: %v\n%s", err, out)
   506  		}
   507  
   508  		var stdPackages []string
   509  		importMap := make(map[string]map[string]string)
   510  		importDir := make(map[string]string)
   511  		dec := json.NewDecoder(bytes.NewReader(out))
   512  		for {
   513  			var pkg struct {
   514  				ImportPath, Dir string
   515  				ImportMap       map[string]string
   516  				Standard        bool
   517  			}
   518  			err := dec.Decode(&pkg)
   519  			if err == io.EOF {
   520  				break
   521  			}
   522  			if err != nil {
   523  				log.Fatalf("go list: invalid output: %v", err)
   524  			}
   525  
   526  			// - Package "unsafe" contains special signatures requiring
   527  			//   extra care when printing them - ignore since it is not
   528  			//   going to change w/o a language change.
   529  			// - Internal and vendored packages do not contribute to our
   530  			//   API surface. (If we are running within the "std" module,
   531  			//   vendored dependencies appear as themselves instead of
   532  			//   their "vendor/" standard-library copies.)
   533  			// - 'go list std' does not include commands, which cannot be
   534  			//   imported anyway.
   535  			if ip := pkg.ImportPath; pkg.Standard && ip != "unsafe" && !strings.HasPrefix(ip, "vendor/") && !internalPkg.MatchString(ip) {
   536  				stdPackages = append(stdPackages, ip)
   537  			}
   538  			importDir[pkg.ImportPath] = pkg.Dir
   539  			if len(pkg.ImportMap) > 0 {
   540  				importMap[pkg.Dir] = make(map[string]string, len(pkg.ImportMap))
   541  			}
   542  			for k, v := range pkg.ImportMap {
   543  				importMap[pkg.Dir][k] = v
   544  			}
   545  		}
   546  
   547  		slices.Sort(stdPackages)
   548  		imports = listImports{
   549  			stdPackages: stdPackages,
   550  			importMap:   importMap,
   551  			importDir:   importDir,
   552  		}
   553  		imports, _ = listCache.LoadOrStore(name, imports)
   554  	}
   555  
   556  	li := imports.(listImports)
   557  	w.stdPackages = li.stdPackages
   558  	w.importDir = li.importDir
   559  	w.importMap = li.importMap
   560  }
   561  
   562  // listEnv returns the process environment to use when invoking 'go list' for
   563  // the given context.
   564  func listEnv(c *build.Context) []string {
   565  	if c == nil {
   566  		return os.Environ()
   567  	}
   568  
   569  	environ := append(os.Environ(),
   570  		"GOOS="+c.GOOS,
   571  		"GOARCH="+c.GOARCH)
   572  	if c.CgoEnabled {
   573  		environ = append(environ, "CGO_ENABLED=1")
   574  	} else {
   575  		environ = append(environ, "CGO_ENABLED=0")
   576  	}
   577  	return environ
   578  }
   579  
   580  type apiPackage struct {
   581  	*types.Package
   582  	Files []*ast.File
   583  }
   584  
   585  // Importing is a sentinel taking the place in Walker.imported
   586  // for a package that is in the process of being imported.
   587  var importing apiPackage
   588  
   589  // Import implements types.Importer.
   590  func (w *Walker) Import(name string) (*types.Package, error) {
   591  	return w.ImportFrom(name, "", 0)
   592  }
   593  
   594  // ImportFrom implements types.ImporterFrom.
   595  func (w *Walker) ImportFrom(fromPath, fromDir string, mode types.ImportMode) (*types.Package, error) {
   596  	pkg, err := w.importFrom(fromPath, fromDir, mode)
   597  	if err != nil {
   598  		return nil, err
   599  	}
   600  	return pkg.Package, nil
   601  }
   602  
   603  func (w *Walker) import_(name string) (*apiPackage, error) {
   604  	return w.importFrom(name, "", 0)
   605  }
   606  
   607  func (w *Walker) importFrom(fromPath, fromDir string, mode types.ImportMode) (*apiPackage, error) {
   608  	name := fromPath
   609  	if canonical, ok := w.importMap[fromDir][fromPath]; ok {
   610  		name = canonical
   611  	}
   612  
   613  	pkg := w.imported[name]
   614  	if pkg != nil {
   615  		if pkg == &importing {
   616  			log.Fatalf("cycle importing package %q", name)
   617  		}
   618  		return pkg, nil
   619  	}
   620  	w.imported[name] = &importing
   621  
   622  	// Determine package files.
   623  	dir := w.importDir[name]
   624  	if dir == "" {
   625  		dir = filepath.Join(w.root, filepath.FromSlash(name))
   626  	}
   627  	if fi, err := os.Stat(dir); err != nil || !fi.IsDir() {
   628  		log.Panicf("no source in tree for import %q (from import %s in %s): %v", name, fromPath, fromDir, err)
   629  	}
   630  
   631  	context := w.context
   632  	if context == nil {
   633  		context = &build.Default
   634  	}
   635  
   636  	// Look in cache.
   637  	// If we've already done an import with the same set
   638  	// of relevant tags, reuse the result.
   639  	var key string
   640  	if usePkgCache {
   641  		if tags, ok := pkgTags[dir]; ok {
   642  			key = tagKey(dir, context, tags)
   643  			if pkg := pkgCache[key]; pkg != nil {
   644  				w.imported[name] = pkg
   645  				return pkg, nil
   646  			}
   647  		}
   648  	}
   649  
   650  	info, err := context.ImportDir(dir, 0)
   651  	if err != nil {
   652  		if _, nogo := err.(*build.NoGoError); nogo {
   653  			return nil, err
   654  		}
   655  		log.Fatalf("pkg %q, dir %q: ScanDir: %v", name, dir, err)
   656  	}
   657  
   658  	// Save tags list first time we see a directory.
   659  	if usePkgCache {
   660  		if _, ok := pkgTags[dir]; !ok {
   661  			pkgTags[dir] = info.AllTags
   662  			key = tagKey(dir, context, info.AllTags)
   663  		}
   664  	}
   665  
   666  	filenames := append(append([]string{}, info.GoFiles...), info.CgoFiles...)
   667  
   668  	// Parse package files.
   669  	var files []*ast.File
   670  	for _, file := range filenames {
   671  		f, err := w.parseFile(dir, file)
   672  		if err != nil {
   673  			log.Fatalf("error parsing package %s: %s", name, err)
   674  		}
   675  		files = append(files, f)
   676  	}
   677  
   678  	// Type-check package files.
   679  	var sizes types.Sizes
   680  	if w.context != nil {
   681  		sizes = types.SizesFor(w.context.Compiler, w.context.GOARCH)
   682  	}
   683  	conf := types.Config{
   684  		IgnoreFuncBodies: true,
   685  		FakeImportC:      true,
   686  		Importer:         w,
   687  		Sizes:            sizes,
   688  	}
   689  	tpkg, err := conf.Check(name, fset, files, nil)
   690  	if err != nil {
   691  		ctxt := "<no context>"
   692  		if w.context != nil {
   693  			ctxt = fmt.Sprintf("%s-%s", w.context.GOOS, w.context.GOARCH)
   694  		}
   695  		log.Fatalf("error typechecking package %s: %s (%s)", name, err, ctxt)
   696  	}
   697  	pkg = &apiPackage{tpkg, files}
   698  
   699  	if usePkgCache {
   700  		pkgCache[key] = pkg
   701  	}
   702  
   703  	w.imported[name] = pkg
   704  	return pkg, nil
   705  }
   706  
   707  // pushScope enters a new scope (walking a package, type, node, etc)
   708  // and returns a function that will leave the scope (with sanity checking
   709  // for mismatched pushes & pops)
   710  func (w *Walker) pushScope(name string) (popFunc func()) {
   711  	w.scope = append(w.scope, name)
   712  	return func() {
   713  		if len(w.scope) == 0 {
   714  			log.Fatalf("attempt to leave scope %q with empty scope list", name)
   715  		}
   716  		if w.scope[len(w.scope)-1] != name {
   717  			log.Fatalf("attempt to leave scope %q, but scope is currently %#v", name, w.scope)
   718  		}
   719  		w.scope = w.scope[:len(w.scope)-1]
   720  	}
   721  }
   722  
   723  func sortedMethodNames(typ *types.Interface) []string {
   724  	n := typ.NumMethods()
   725  	list := make([]string, n)
   726  	for i := range list {
   727  		list[i] = typ.Method(i).Name()
   728  	}
   729  	slices.Sort(list)
   730  	return list
   731  }
   732  
   733  // sortedEmbeddeds returns constraint types embedded in an
   734  // interface. It does not include embedded interface types or methods.
   735  func (w *Walker) sortedEmbeddeds(typ *types.Interface) []string {
   736  	n := typ.NumEmbeddeds()
   737  	list := make([]string, 0, n)
   738  	for i := 0; i < n; i++ {
   739  		emb := typ.EmbeddedType(i)
   740  		switch emb := emb.(type) {
   741  		case *types.Interface:
   742  			list = append(list, w.sortedEmbeddeds(emb)...)
   743  		case *types.Union:
   744  			var buf bytes.Buffer
   745  			nu := emb.Len()
   746  			for i := 0; i < nu; i++ {
   747  				if i > 0 {
   748  					buf.WriteString(" | ")
   749  				}
   750  				term := emb.Term(i)
   751  				if term.Tilde() {
   752  					buf.WriteByte('~')
   753  				}
   754  				w.writeType(&buf, term.Type())
   755  			}
   756  			list = append(list, buf.String())
   757  		}
   758  	}
   759  	slices.Sort(list)
   760  	return list
   761  }
   762  
   763  func (w *Walker) writeType(buf *bytes.Buffer, typ types.Type) {
   764  	switch typ := typ.(type) {
   765  	case *types.Basic:
   766  		s := typ.Name()
   767  		switch typ.Kind() {
   768  		case types.UnsafePointer:
   769  			s = "unsafe.Pointer"
   770  		case types.UntypedBool:
   771  			s = "ideal-bool"
   772  		case types.UntypedInt:
   773  			s = "ideal-int"
   774  		case types.UntypedRune:
   775  			// "ideal-char" for compatibility with old tool
   776  			// TODO(gri) change to "ideal-rune"
   777  			s = "ideal-char"
   778  		case types.UntypedFloat:
   779  			s = "ideal-float"
   780  		case types.UntypedComplex:
   781  			s = "ideal-complex"
   782  		case types.UntypedString:
   783  			s = "ideal-string"
   784  		case types.UntypedNil:
   785  			panic("should never see untyped nil type")
   786  		default:
   787  			switch s {
   788  			case "byte":
   789  				s = "uint8"
   790  			case "rune":
   791  				s = "int32"
   792  			}
   793  		}
   794  		buf.WriteString(s)
   795  
   796  	case *types.Array:
   797  		fmt.Fprintf(buf, "[%d]", typ.Len())
   798  		w.writeType(buf, typ.Elem())
   799  
   800  	case *types.Slice:
   801  		buf.WriteString("[]")
   802  		w.writeType(buf, typ.Elem())
   803  
   804  	case *types.Struct:
   805  		buf.WriteString("struct")
   806  
   807  	case *types.Pointer:
   808  		buf.WriteByte('*')
   809  		w.writeType(buf, typ.Elem())
   810  
   811  	case *types.Tuple:
   812  		panic("should never see a tuple type")
   813  
   814  	case *types.Signature:
   815  		buf.WriteString("func")
   816  		w.writeSignature(buf, typ)
   817  
   818  	case *types.Interface:
   819  		buf.WriteString("interface{")
   820  		if typ.NumMethods() > 0 || typ.NumEmbeddeds() > 0 {
   821  			buf.WriteByte(' ')
   822  		}
   823  		if typ.NumMethods() > 0 {
   824  			buf.WriteString(strings.Join(sortedMethodNames(typ), ", "))
   825  		}
   826  		if typ.NumEmbeddeds() > 0 {
   827  			buf.WriteString(strings.Join(w.sortedEmbeddeds(typ), ", "))
   828  		}
   829  		if typ.NumMethods() > 0 || typ.NumEmbeddeds() > 0 {
   830  			buf.WriteByte(' ')
   831  		}
   832  		buf.WriteString("}")
   833  
   834  	case *types.Map:
   835  		buf.WriteString("map[")
   836  		w.writeType(buf, typ.Key())
   837  		buf.WriteByte(']')
   838  		w.writeType(buf, typ.Elem())
   839  
   840  	case *types.Chan:
   841  		var s string
   842  		switch typ.Dir() {
   843  		case types.SendOnly:
   844  			s = "chan<- "
   845  		case types.RecvOnly:
   846  			s = "<-chan "
   847  		case types.SendRecv:
   848  			s = "chan "
   849  		default:
   850  			panic("unreachable")
   851  		}
   852  		buf.WriteString(s)
   853  		w.writeType(buf, typ.Elem())
   854  
   855  	case *types.Alias:
   856  		w.writeType(buf, types.Unalias(typ))
   857  
   858  	case *types.Named:
   859  		obj := typ.Obj()
   860  		pkg := obj.Pkg()
   861  		if pkg != nil && pkg != w.current.Package {
   862  			buf.WriteString(pkg.Name())
   863  			buf.WriteByte('.')
   864  		}
   865  		buf.WriteString(typ.Obj().Name())
   866  		if targs := typ.TypeArgs(); targs.Len() > 0 {
   867  			buf.WriteByte('[')
   868  			for i := 0; i < targs.Len(); i++ {
   869  				if i > 0 {
   870  					buf.WriteString(", ")
   871  				}
   872  				w.writeType(buf, targs.At(i))
   873  			}
   874  			buf.WriteByte(']')
   875  		}
   876  
   877  	case *types.TypeParam:
   878  		// Type parameter names may change, so use a placeholder instead.
   879  		fmt.Fprintf(buf, "$%d", typ.Index())
   880  
   881  	default:
   882  		panic(fmt.Sprintf("unknown type %T", typ))
   883  	}
   884  }
   885  
   886  func (w *Walker) writeSignature(buf *bytes.Buffer, sig *types.Signature) {
   887  	if tparams := sig.TypeParams(); tparams != nil {
   888  		w.writeTypeParams(buf, tparams, true)
   889  	}
   890  	w.writeParams(buf, sig.Params(), sig.Variadic())
   891  	switch res := sig.Results(); res.Len() {
   892  	case 0:
   893  		// nothing to do
   894  	case 1:
   895  		buf.WriteByte(' ')
   896  		w.writeType(buf, res.At(0).Type())
   897  	default:
   898  		buf.WriteByte(' ')
   899  		w.writeParams(buf, res, false)
   900  	}
   901  }
   902  
   903  func (w *Walker) writeTypeParams(buf *bytes.Buffer, tparams *types.TypeParamList, withConstraints bool) {
   904  	buf.WriteByte('[')
   905  	c := tparams.Len()
   906  	for i := 0; i < c; i++ {
   907  		if i > 0 {
   908  			buf.WriteString(", ")
   909  		}
   910  		tp := tparams.At(i)
   911  		w.writeType(buf, tp)
   912  		if withConstraints {
   913  			buf.WriteByte(' ')
   914  			w.writeType(buf, tp.Constraint())
   915  		}
   916  	}
   917  	buf.WriteByte(']')
   918  }
   919  
   920  func (w *Walker) writeParams(buf *bytes.Buffer, t *types.Tuple, variadic bool) {
   921  	buf.WriteByte('(')
   922  	for i, n := 0, t.Len(); i < n; i++ {
   923  		if i > 0 {
   924  			buf.WriteString(", ")
   925  		}
   926  		typ := t.At(i).Type()
   927  		if variadic && i+1 == n {
   928  			buf.WriteString("...")
   929  			typ = typ.(*types.Slice).Elem()
   930  		}
   931  		w.writeType(buf, typ)
   932  	}
   933  	buf.WriteByte(')')
   934  }
   935  
   936  func (w *Walker) typeString(typ types.Type) string {
   937  	var buf bytes.Buffer
   938  	w.writeType(&buf, typ)
   939  	return buf.String()
   940  }
   941  
   942  func (w *Walker) signatureString(sig *types.Signature) string {
   943  	var buf bytes.Buffer
   944  	w.writeSignature(&buf, sig)
   945  	return buf.String()
   946  }
   947  
   948  func (w *Walker) emitObj(obj types.Object) {
   949  	switch obj := obj.(type) {
   950  	case *types.Const:
   951  		if w.isDeprecated(obj) {
   952  			w.emitf("const %s //deprecated", obj.Name())
   953  		}
   954  		w.emitf("const %s %s", obj.Name(), w.typeString(obj.Type()))
   955  		x := obj.Val()
   956  		short := x.String()
   957  		exact := x.ExactString()
   958  		if short == exact {
   959  			w.emitf("const %s = %s", obj.Name(), short)
   960  		} else {
   961  			w.emitf("const %s = %s  // %s", obj.Name(), short, exact)
   962  		}
   963  	case *types.Var:
   964  		if w.isDeprecated(obj) {
   965  			w.emitf("var %s //deprecated", obj.Name())
   966  		}
   967  		w.emitf("var %s %s", obj.Name(), w.typeString(obj.Type()))
   968  	case *types.TypeName:
   969  		w.emitType(obj)
   970  	case *types.Func:
   971  		w.emitFunc(obj)
   972  	default:
   973  		panic("unknown object: " + obj.String())
   974  	}
   975  }
   976  
   977  func (w *Walker) emitType(obj *types.TypeName) {
   978  	name := obj.Name()
   979  	if w.isDeprecated(obj) {
   980  		w.emitf("type %s //deprecated", name)
   981  	}
   982  	typ := obj.Type()
   983  	if obj.IsAlias() {
   984  		w.emitf("type %s = %s", name, w.typeString(typ))
   985  		return
   986  	}
   987  	if tparams := obj.Type().(*types.Named).TypeParams(); tparams != nil {
   988  		var buf bytes.Buffer
   989  		buf.WriteString(name)
   990  		w.writeTypeParams(&buf, tparams, true)
   991  		name = buf.String()
   992  	}
   993  	switch typ := typ.Underlying().(type) {
   994  	case *types.Struct:
   995  		w.emitStructType(name, typ)
   996  	case *types.Interface:
   997  		w.emitIfaceType(name, typ)
   998  		return // methods are handled by emitIfaceType
   999  	default:
  1000  		w.emitf("type %s %s", name, w.typeString(typ.Underlying()))
  1001  	}
  1002  
  1003  	// emit methods with value receiver
  1004  	var methodNames map[string]bool
  1005  	vset := types.NewMethodSet(typ)
  1006  	for i, n := 0, vset.Len(); i < n; i++ {
  1007  		m := vset.At(i)
  1008  		if m.Obj().Exported() {
  1009  			w.emitMethod(m)
  1010  			if methodNames == nil {
  1011  				methodNames = make(map[string]bool)
  1012  			}
  1013  			methodNames[m.Obj().Name()] = true
  1014  		}
  1015  	}
  1016  
  1017  	// emit methods with pointer receiver; exclude
  1018  	// methods that we have emitted already
  1019  	// (the method set of *T includes the methods of T)
  1020  	pset := types.NewMethodSet(types.NewPointer(typ))
  1021  	for i, n := 0, pset.Len(); i < n; i++ {
  1022  		m := pset.At(i)
  1023  		if m.Obj().Exported() && !methodNames[m.Obj().Name()] {
  1024  			w.emitMethod(m)
  1025  		}
  1026  	}
  1027  }
  1028  
  1029  func (w *Walker) emitStructType(name string, typ *types.Struct) {
  1030  	typeStruct := fmt.Sprintf("type %s struct", name)
  1031  	w.emitf("%s", typeStruct)
  1032  	defer w.pushScope(typeStruct)()
  1033  
  1034  	for i := 0; i < typ.NumFields(); i++ {
  1035  		f := typ.Field(i)
  1036  		if !f.Exported() {
  1037  			continue
  1038  		}
  1039  		typ := f.Type()
  1040  		if f.Anonymous() {
  1041  			if w.isDeprecated(f) {
  1042  				w.emitf("embedded %s //deprecated", w.typeString(typ))
  1043  			}
  1044  			w.emitf("embedded %s", w.typeString(typ))
  1045  			continue
  1046  		}
  1047  		if w.isDeprecated(f) {
  1048  			w.emitf("%s //deprecated", f.Name())
  1049  		}
  1050  		w.emitf("%s %s", f.Name(), w.typeString(typ))
  1051  	}
  1052  }
  1053  
  1054  func (w *Walker) emitIfaceType(name string, typ *types.Interface) {
  1055  	pop := w.pushScope("type " + name + " interface")
  1056  
  1057  	var methodNames []string
  1058  	complete := true
  1059  	mset := types.NewMethodSet(typ)
  1060  	for i, n := 0, mset.Len(); i < n; i++ {
  1061  		m := mset.At(i).Obj().(*types.Func)
  1062  		if !m.Exported() {
  1063  			complete = false
  1064  			continue
  1065  		}
  1066  		methodNames = append(methodNames, m.Name())
  1067  		if w.isDeprecated(m) {
  1068  			w.emitf("%s //deprecated", m.Name())
  1069  		}
  1070  		w.emitf("%s%s", m.Name(), w.signatureString(m.Signature()))
  1071  	}
  1072  
  1073  	if !complete {
  1074  		// The method set has unexported methods, so all the
  1075  		// implementations are provided by the same package,
  1076  		// so the method set can be extended. Instead of recording
  1077  		// the full set of names (below), record only that there were
  1078  		// unexported methods. (If the interface shrinks, we will notice
  1079  		// because a method signature emitted during the last loop
  1080  		// will disappear.)
  1081  		w.emitf("unexported methods")
  1082  	}
  1083  
  1084  	pop()
  1085  
  1086  	if !complete {
  1087  		return
  1088  	}
  1089  
  1090  	if len(methodNames) == 0 {
  1091  		w.emitf("type %s interface {}", name)
  1092  		return
  1093  	}
  1094  
  1095  	slices.Sort(methodNames)
  1096  	w.emitf("type %s interface { %s }", name, strings.Join(methodNames, ", "))
  1097  }
  1098  
  1099  func (w *Walker) emitFunc(f *types.Func) {
  1100  	sig := f.Signature()
  1101  	if sig.Recv() != nil {
  1102  		panic("method considered a regular function: " + f.String())
  1103  	}
  1104  	if w.isDeprecated(f) {
  1105  		w.emitf("func %s //deprecated", f.Name())
  1106  	}
  1107  	w.emitf("func %s%s", f.Name(), w.signatureString(sig))
  1108  }
  1109  
  1110  func (w *Walker) emitMethod(m *types.Selection) {
  1111  	sig := m.Type().(*types.Signature)
  1112  	recv := sig.Recv().Type()
  1113  	// report exported methods with unexported receiver base type
  1114  	if true {
  1115  		base := recv
  1116  		if p, _ := recv.(*types.Pointer); p != nil {
  1117  			base = p.Elem()
  1118  		}
  1119  		if obj := base.(*types.Named).Obj(); !obj.Exported() {
  1120  			log.Fatalf("exported method with unexported receiver base type: %s", m)
  1121  		}
  1122  	}
  1123  	tps := ""
  1124  	if rtp := sig.RecvTypeParams(); rtp != nil {
  1125  		var buf bytes.Buffer
  1126  		w.writeTypeParams(&buf, rtp, false)
  1127  		tps = buf.String()
  1128  	}
  1129  	if w.isDeprecated(m.Obj()) {
  1130  		w.emitf("method (%s%s) %s //deprecated", w.typeString(recv), tps, m.Obj().Name())
  1131  	}
  1132  	w.emitf("method (%s%s) %s%s", w.typeString(recv), tps, m.Obj().Name(), w.signatureString(sig))
  1133  }
  1134  
  1135  func (w *Walker) emitf(format string, args ...any) {
  1136  	f := strings.Join(w.scope, ", ") + ", " + fmt.Sprintf(format, args...)
  1137  	if strings.Contains(f, "\n") {
  1138  		panic("feature contains newlines: " + f)
  1139  	}
  1140  
  1141  	if _, dup := w.features[f]; dup {
  1142  		panic("duplicate feature inserted: " + f)
  1143  	}
  1144  	w.features[f] = true
  1145  
  1146  	if verbose {
  1147  		log.Printf("feature: %s", f)
  1148  	}
  1149  }
  1150  
  1151  func needApproval(filename string) bool {
  1152  	name := filepath.Base(filename)
  1153  	if name == "go1.txt" {
  1154  		return false
  1155  	}
  1156  	minor := strings.TrimSuffix(strings.TrimPrefix(name, "go1."), ".txt")
  1157  	n, err := strconv.Atoi(minor)
  1158  	if err != nil {
  1159  		log.Fatalf("unexpected api file: %v", name)
  1160  	}
  1161  	return n >= 19 // started tracking approvals in Go 1.19
  1162  }
  1163  
  1164  func (w *Walker) collectDeprecated() {
  1165  	isDeprecated := func(doc *ast.CommentGroup) bool {
  1166  		// Look for "Deprecated:" (case-sensitive) at the beginning (not middle) of a paragraph.
  1167  		// It's typically found in the last paragraph, but it's not required to be the last one.
  1168  		// The colon is typically followed by a space, but it can also be a newline, as was the
  1169  		// case at https://go.dev/pkg/go/build#AllowBinary for example.
  1170  		//
  1171  		// See https://go.dev/wiki/Deprecated and https://go.dev/ref/mod#go-mod-file-module-deprecation.
  1172  		text := doc.Text()
  1173  		return strings.HasPrefix(text, "Deprecated: ") || strings.Contains(text, "\n\nDeprecated: ") ||
  1174  			strings.HasPrefix(text, "Deprecated:\n") || strings.Contains(text, "\n\nDeprecated:\n")
  1175  	}
  1176  
  1177  	w.deprecated = make(map[token.Pos]bool)
  1178  	mark := func(id *ast.Ident) {
  1179  		if id != nil {
  1180  			w.deprecated[id.Pos()] = true
  1181  		}
  1182  	}
  1183  	for _, file := range w.current.Files {
  1184  		ast.Inspect(file, func(n ast.Node) bool {
  1185  			switch n := n.(type) {
  1186  			case *ast.File:
  1187  				if isDeprecated(n.Doc) {
  1188  					mark(n.Name)
  1189  				}
  1190  				return true
  1191  			case *ast.GenDecl:
  1192  				if isDeprecated(n.Doc) {
  1193  					for _, spec := range n.Specs {
  1194  						switch spec := spec.(type) {
  1195  						case *ast.ValueSpec:
  1196  							for _, id := range spec.Names {
  1197  								mark(id)
  1198  							}
  1199  						case *ast.TypeSpec:
  1200  							mark(spec.Name)
  1201  						}
  1202  					}
  1203  				}
  1204  				return true // look at specs
  1205  			case *ast.FuncDecl:
  1206  				if isDeprecated(n.Doc) {
  1207  					mark(n.Name)
  1208  				}
  1209  				return false
  1210  			case *ast.TypeSpec:
  1211  				if isDeprecated(n.Doc) {
  1212  					mark(n.Name)
  1213  				}
  1214  				return true // recurse into struct or interface type
  1215  			case *ast.StructType:
  1216  				return true // recurse into fields
  1217  			case *ast.InterfaceType:
  1218  				return true // recurse into methods
  1219  			case *ast.FieldList:
  1220  				return true // recurse into fields
  1221  			case *ast.ValueSpec:
  1222  				if isDeprecated(n.Doc) {
  1223  					for _, id := range n.Names {
  1224  						mark(id)
  1225  					}
  1226  				}
  1227  				return false
  1228  			case *ast.Field:
  1229  				if isDeprecated(n.Doc) {
  1230  					for _, id := range n.Names {
  1231  						mark(id)
  1232  					}
  1233  					if len(n.Names) == 0 {
  1234  						// embedded field T or *T?
  1235  						typ := n.Type
  1236  						if ptr, ok := typ.(*ast.StarExpr); ok {
  1237  							typ = ptr.X
  1238  						}
  1239  						if id, ok := typ.(*ast.Ident); ok {
  1240  							mark(id)
  1241  						}
  1242  					}
  1243  				}
  1244  				return false
  1245  			default:
  1246  				return false
  1247  			}
  1248  		})
  1249  	}
  1250  }
  1251  
  1252  func (w *Walker) isDeprecated(obj types.Object) bool {
  1253  	return w.deprecated[obj.Pos()]
  1254  }
  1255  

View as plain text