stdlib_test.go

     1  // Copyright 2013 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // This file tests types.Check by using it to
     6  // typecheck the standard library and tests.
     7  
     8  package types_test
     9  
    10  import (
    11  	"errors"
    12  	"fmt"
    13  	"go/ast"
    14  	"go/build"
    15  	"go/importer"
    16  	"go/parser"
    17  	"go/scanner"
    18  	"go/token"
    19  	"internal/testenv"
    20  	"os"
    21  	"path/filepath"
    22  	"runtime"
    23  	"slices"
    24  	"strings"
    25  	"sync"
    26  	"testing"
    27  	"time"
    28  
    29  	. "go/types"
    30  )
    31  
    32  // The cmd/*/internal packages may have been deleted as part of a binary
    33  // release. Import from source instead.
    34  //
    35  // (See https://golang.org/issue/43232 and
    36  // https://github.com/golang/build/blob/df58bbac082bc87c4a3cdfe336d1ffe60bbaa916/cmd/release/release.go#L533-L545.)
    37  //
    38  // Use the same importer for all std lib tests to
    39  // avoid repeated importing of the same packages.
    40  var stdLibImporter = importer.ForCompiler(token.NewFileSet(), "source", nil)
    41  
    42  func TestStdlib(t *testing.T) {
    43  	if testing.Short() {
    44  		t.Skip("skipping in short mode")
    45  	}
    46  
    47  	testenv.MustHaveGoBuild(t)
    48  	t.Parallel()
    49  
    50  	// Collect non-test files.
    51  	dirFiles := make(map[string][]string)
    52  	root := filepath.Join(testenv.GOROOT(t), "src")
    53  	walkPkgDirs(root, func(dir string, filenames []string) {
    54  		dirFiles[dir] = filenames
    55  	}, t.Error)
    56  
    57  	c := &stdlibChecker{
    58  		dirFiles: dirFiles,
    59  		pkgs:     make(map[string]*futurePackage),
    60  	}
    61  
    62  	start := time.Now()
    63  
    64  	// Though we read files while parsing, type-checking is otherwise CPU bound.
    65  	//
    66  	// This doesn't achieve great CPU utilization as many packages may block
    67  	// waiting for a common import, but in combination with the non-deterministic
    68  	// map iteration below this should provide decent coverage of concurrent
    69  	// type-checking (see golang/go#47729).
    70  	cpulimit := make(chan struct{}, runtime.GOMAXPROCS(0))
    71  	var wg sync.WaitGroup
    72  
    73  	for dir := range dirFiles {
    74  		cpulimit <- struct{}{}
    75  		wg.Add(1)
    76  		go func() {
    77  			defer func() {
    78  				wg.Done()
    79  				<-cpulimit
    80  			}()
    81  
    82  			_, err := c.getDirPackage(dir)
    83  			if err != nil {
    84  				t.Errorf("error checking %s: %v", dir, err)
    85  			}
    86  		}()
    87  	}
    88  
    89  	wg.Wait()
    90  
    91  	if testing.Verbose() {
    92  		fmt.Println(len(dirFiles), "packages typechecked in", time.Since(start))
    93  	}
    94  }
    95  
    96  // stdlibChecker implements concurrent type-checking of the packages defined by
    97  // dirFiles, which must define a closed set of packages (such as GOROOT/src).
    98  type stdlibChecker struct {
    99  	dirFiles map[string][]string // non-test files per directory; must be pre-populated
   100  
   101  	mu   sync.Mutex
   102  	pkgs map[string]*futurePackage // future cache of type-checking results
   103  }
   104  
   105  // A futurePackage is a future result of type-checking.
   106  type futurePackage struct {
   107  	done chan struct{} // guards pkg and err
   108  	pkg  *Package
   109  	err  error
   110  }
   111  
   112  func (c *stdlibChecker) Import(path string) (*Package, error) {
   113  	panic("unimplemented: use ImportFrom")
   114  }
   115  
   116  func (c *stdlibChecker) ImportFrom(path, dir string, _ ImportMode) (*Package, error) {
   117  	if path == "unsafe" {
   118  		// unsafe cannot be type checked normally.
   119  		return Unsafe, nil
   120  	}
   121  
   122  	p, err := build.Default.Import(path, dir, build.FindOnly)
   123  	if err != nil {
   124  		return nil, err
   125  	}
   126  
   127  	pkg, err := c.getDirPackage(p.Dir)
   128  	if pkg != nil {
   129  		// As long as pkg is non-nil, avoid redundant errors related to failed
   130  		// imports. TestStdlib will collect errors once for each package.
   131  		return pkg, nil
   132  	}
   133  	return nil, err
   134  }
   135  
   136  // getDirPackage gets the package defined in dir from the future cache.
   137  //
   138  // If this is the first goroutine requesting the package, getDirPackage
   139  // type-checks.
   140  func (c *stdlibChecker) getDirPackage(dir string) (*Package, error) {
   141  	c.mu.Lock()
   142  	fut, ok := c.pkgs[dir]
   143  	if !ok {
   144  		// First request for this package dir; type check.
   145  		fut = &futurePackage{
   146  			done: make(chan struct{}),
   147  		}
   148  		c.pkgs[dir] = fut
   149  		files, ok := c.dirFiles[dir]
   150  		c.mu.Unlock()
   151  		if !ok {
   152  			fut.err = fmt.Errorf("no files for %s", dir)
   153  		} else {
   154  			// Using dir as the package path here may be inconsistent with the behavior
   155  			// of a normal importer, but is sufficient as dir is by construction unique
   156  			// to this package.
   157  			fut.pkg, fut.err = typecheckFiles(dir, files, c)
   158  		}
   159  		close(fut.done)
   160  	} else {
   161  		// Otherwise, await the result.
   162  		c.mu.Unlock()
   163  		<-fut.done
   164  	}
   165  	return fut.pkg, fut.err
   166  }
   167  
   168  // firstComment returns the contents of the first non-empty comment in
   169  // the given file, "skip", or the empty string. No matter the present
   170  // comments, if any of them contains a build tag, the result is always
   171  // "skip". Only comments before the "package" token and within the first
   172  // 4K of the file are considered.
   173  func firstComment(filename string) string {
   174  	f, err := os.Open(filename)
   175  	if err != nil {
   176  		return ""
   177  	}
   178  	defer f.Close()
   179  
   180  	var src [4 << 10]byte // read at most 4KB
   181  	n, _ := f.Read(src[:])
   182  
   183  	var first string
   184  	var s scanner.Scanner
   185  	s.Init(testFSet.AddFile("", testFSet.Base(), n), src[:n], nil /* ignore errors */, scanner.ScanComments)
   186  	for {
   187  		_, tok, lit := s.Scan()
   188  		switch tok {
   189  		case token.COMMENT:
   190  			// remove trailing */ of multi-line comment
   191  			if lit[1] == '*' {
   192  				lit = lit[:len(lit)-2]
   193  			}
   194  			contents := strings.TrimSpace(lit[2:])
   195  			if strings.HasPrefix(contents, "go:build ") {
   196  				return "skip"
   197  			}
   198  			if first == "" {
   199  				first = contents // contents may be "" but that's ok
   200  			}
   201  			// continue as we may still see build tags
   202  
   203  		case token.PACKAGE, token.EOF:
   204  			return first
   205  		}
   206  	}
   207  }
   208  
   209  func testTestDir(t *testing.T, path string, ignore ...string) {
   210  	files, err := os.ReadDir(path)
   211  	if err != nil {
   212  		// cmd/distpack deletes GOROOT/test, so skip the test if it isn't present.
   213  		// cmd/distpack also requires GOROOT/VERSION to exist, so use that to
   214  		// suppress false-positive skips.
   215  		if _, err := os.Stat(filepath.Join(testenv.GOROOT(t), "test")); os.IsNotExist(err) {
   216  			if _, err := os.Stat(filepath.Join(testenv.GOROOT(t), "VERSION")); err == nil {
   217  				t.Skipf("skipping: GOROOT/test not present")
   218  			}
   219  		}
   220  		t.Fatal(err)
   221  	}
   222  
   223  	excluded := make(map[string]bool)
   224  	for _, filename := range ignore {
   225  		excluded[filename] = true
   226  	}
   227  
   228  	fset := token.NewFileSet()
   229  	for _, f := range files {
   230  		// filter directory contents
   231  		if f.IsDir() || !strings.HasSuffix(f.Name(), ".go") || excluded[f.Name()] {
   232  			continue
   233  		}
   234  
   235  		// get per-file instructions
   236  		expectErrors := false
   237  		filename := filepath.Join(path, f.Name())
   238  		goVersion := ""
   239  		if comment := firstComment(filename); comment != "" {
   240  			if strings.Contains(comment, "-goexperiment") {
   241  				continue // ignore this file
   242  			}
   243  			fields := strings.Fields(comment)
   244  			switch fields[0] {
   245  			case "skip", "compiledir":
   246  				continue // ignore this file
   247  			case "errorcheck":
   248  				expectErrors = true
   249  				for _, arg := range fields[1:] {
   250  					if arg == "-0" || arg == "-+" || arg == "-std" {
   251  						// Marked explicitly as not expecting errors (-0),
   252  						// or marked as compiling runtime/stdlib, which is only done
   253  						// to trigger runtime/stdlib-only error output.
   254  						// In both cases, the code should typecheck.
   255  						expectErrors = false
   256  						break
   257  					}
   258  					const prefix = "-lang="
   259  					if strings.HasPrefix(arg, prefix) {
   260  						goVersion = arg[len(prefix):]
   261  					}
   262  				}
   263  			}
   264  		}
   265  
   266  		// parse and type-check file
   267  		file, err := parser.ParseFile(fset, filename, nil, parser.SkipObjectResolution)
   268  		if err == nil {
   269  			conf := Config{
   270  				GoVersion: goVersion,
   271  				Importer:  stdLibImporter,
   272  			}
   273  			_, err = conf.Check(filename, fset, []*ast.File{file}, nil)
   274  		}
   275  
   276  		if expectErrors {
   277  			if err == nil {
   278  				t.Errorf("expected errors but found none in %s", filename)
   279  			}
   280  		} else {
   281  			if err != nil {
   282  				t.Error(err)
   283  			}
   284  		}
   285  	}
   286  }
   287  
   288  func TestStdTest(t *testing.T) {
   289  	testenv.MustHaveGoBuild(t)
   290  
   291  	if testing.Short() && testenv.Builder() == "" {
   292  		t.Skip("skipping in short mode")
   293  	}
   294  
   295  	testTestDir(t, filepath.Join(testenv.GOROOT(t), "test"),
   296  		"cmplxdivide.go", // also needs file cmplxdivide1.go - ignore
   297  		"directive.go",   // tests compiler rejection of bad directive placement - ignore
   298  		"directive2.go",  // tests compiler rejection of bad directive placement - ignore
   299  		"embedfunc.go",   // tests //go:embed
   300  		"embedvers.go",   // tests //go:embed
   301  		"linkname2.go",   // go/types doesn't check validity of //go:xxx directives
   302  		"linkname3.go",   // go/types doesn't check validity of //go:xxx directives
   303  	)
   304  }
   305  
   306  func TestStdFixed(t *testing.T) {
   307  	testenv.MustHaveGoBuild(t)
   308  
   309  	if testing.Short() && testenv.Builder() == "" {
   310  		t.Skip("skipping in short mode")
   311  	}
   312  
   313  	testTestDir(t, filepath.Join(testenv.GOROOT(t), "test", "fixedbugs"),
   314  		"bug248.go", "bug302.go", "bug369.go", // complex test instructions - ignore
   315  		"bug398.go",      // go/types doesn't check for anonymous interface cycles (go.dev/issue/56103)
   316  		"issue6889.go",   // gc-specific test
   317  		"issue11362.go",  // canonical import path check
   318  		"issue16369.go",  // go/types handles this correctly - not an issue
   319  		"issue18459.go",  // go/types doesn't check validity of //go:xxx directives
   320  		"issue18882.go",  // go/types doesn't check validity of //go:xxx directives
   321  		"issue20027.go",  // go/types does not have constraints on channel element size
   322  		"issue20529.go",  // go/types does not have constraints on stack size
   323  		"issue22200.go",  // go/types does not have constraints on stack size
   324  		"issue22200b.go", // go/types does not have constraints on stack size
   325  		"issue25507.go",  // go/types does not have constraints on stack size
   326  		"issue20780.go",  // go/types does not have constraints on stack size
   327  		"bug251.go",      // go.dev/issue/34333 which was exposed with fix for go.dev/issue/34151
   328  		"issue42058a.go", // go/types does not have constraints on channel element size
   329  		"issue42058b.go", // go/types does not have constraints on channel element size
   330  		"issue48097.go",  // go/types doesn't check validity of //go:xxx directives, and non-init bodyless function
   331  		"issue48230.go",  // go/types doesn't check validity of //go:xxx directives
   332  		"issue49767.go",  // go/types does not have constraints on channel element size
   333  		"issue49814.go",  // go/types does not have constraints on array size
   334  		"issue78355.go",  // go/types does not have constraints on map element size
   335  		"issue56103.go",  // anonymous interface cycles; will be a type checker error in 1.22
   336  		"issue52697.go",  // go/types does not have constraints on stack size
   337  
   338  		// These tests requires runtime/cgo.Incomplete, which is only available on some platforms.
   339  		// However, go/types does not know about build constraints.
   340  		"bug514.go",
   341  		"issue40954.go",
   342  		"issue42032.go",
   343  		"issue42076.go",
   344  		"issue46903.go",
   345  		"issue51733.go",
   346  		"notinheap2.go",
   347  		"notinheap3.go",
   348  	)
   349  }
   350  
   351  func TestStdKen(t *testing.T) {
   352  	testenv.MustHaveGoBuild(t)
   353  
   354  	testTestDir(t, filepath.Join(testenv.GOROOT(t), "test", "ken"))
   355  }
   356  
   357  // Package paths of excluded packages.
   358  var excluded = map[string]bool{
   359  	"builtin":                       true,
   360  	"cmd/compile/internal/ssa/_gen": true,
   361  	"crypto/internal/cryptotest/wycheproof/_schema": true,
   362  	"crypto/internal/cryptotest/x509limbo/_schema":  true,
   363  	"runtime/_mkmalloc":                             true,
   364  }
   365  
   366  func isExcluded(pkgPath string) bool {
   367  	if excluded[pkgPath] {
   368  		return true
   369  	}
   370  
   371  	// Submodules where not all dependencies are available.
   372  	// See go.dev/issue/46027.
   373  	if strings.HasPrefix(pkgPath, "simd/archsimd/_gen") {
   374  		return true
   375  	}
   376  	if slices.Contains(strings.Split(pkgPath, "/"), "_asm") {
   377  		return true
   378  	}
   379  
   380  	return false
   381  }
   382  
   383  // printPackageMu synchronizes the printing of type-checked package files in
   384  // the typecheckFiles function.
   385  //
   386  // Without synchronization, package files may be interleaved during concurrent
   387  // type-checking.
   388  var printPackageMu sync.Mutex
   389  
   390  // typecheckFiles typechecks the given package files.
   391  func typecheckFiles(path string, filenames []string, importer Importer) (*Package, error) {
   392  	fset := token.NewFileSet()
   393  
   394  	// Parse package files.
   395  	var files []*ast.File
   396  	for _, filename := range filenames {
   397  		file, err := parser.ParseFile(fset, filename, nil, parser.AllErrors|parser.SkipObjectResolution)
   398  		if err != nil {
   399  			return nil, err
   400  		}
   401  
   402  		files = append(files, file)
   403  	}
   404  
   405  	if testing.Verbose() {
   406  		printPackageMu.Lock()
   407  		fmt.Println("package", files[0].Name.Name)
   408  		for _, filename := range filenames {
   409  			fmt.Println("\t", filename)
   410  		}
   411  		printPackageMu.Unlock()
   412  	}
   413  
   414  	// Typecheck package files.
   415  	var errs []error
   416  	conf := Config{
   417  		Error: func(err error) {
   418  			errs = append(errs, err)
   419  		},
   420  		Importer: importer,
   421  	}
   422  	info := Info{
   423  		Defs:      make(map[*ast.Ident]Object),
   424  		Uses:      make(map[*ast.Ident]Object),
   425  		Implicits: make(map[ast.Node]Object),
   426  		Types:     make(map[ast.Expr]TypeAndValue),
   427  		Scopes:    make(map[ast.Node]*Scope),
   428  	}
   429  	pkg, _ := conf.Check(path, fset, files, &info)
   430  	err := errors.Join(errs...)
   431  	if err != nil {
   432  		return pkg, err
   433  	}
   434  
   435  	// Perform checks of API invariants.
   436  	if invErrs := checkInvariants(pkg, &info, false); len(invErrs) > 0 {
   437  		return pkg, fmt.Errorf("invariant failure: %v", errors.Join(invErrs...))
   438  	}
   439  
   440  	// All Objects have a package, except predeclared ones.
   441  	errorError := Universe.Lookup("error").Type().Underlying().(*Interface).ExplicitMethod(0) // (error).Error
   442  	for id, obj := range info.Uses {
   443  		predeclared := obj == Universe.Lookup(obj.Name()) || obj == errorError
   444  		if predeclared == (obj.Pkg() != nil) {
   445  			posn := fset.Position(id.Pos())
   446  			if predeclared {
   447  				return nil, fmt.Errorf("%s: predeclared object with package: %s", posn, obj)
   448  			} else {
   449  				return nil, fmt.Errorf("%s: user-defined object without package: %s", posn, obj)
   450  			}
   451  		}
   452  	}
   453  
   454  	return pkg, nil
   455  }
   456  
   457  // pkgFilenames returns the list of package filenames for the given directory.
   458  func pkgFilenames(dir string, includeTest bool) ([]string, error) {
   459  	ctxt := build.Default
   460  	ctxt.CgoEnabled = false
   461  	pkg, err := ctxt.ImportDir(dir, 0)
   462  	if err != nil {
   463  		if _, nogo := err.(*build.NoGoError); nogo {
   464  			return nil, nil // no *.go files, not an error
   465  		}
   466  		return nil, err
   467  	}
   468  	if isExcluded(pkg.ImportPath) {
   469  		return nil, nil
   470  	}
   471  	var filenames []string
   472  	for _, name := range pkg.GoFiles {
   473  		filenames = append(filenames, filepath.Join(pkg.Dir, name))
   474  	}
   475  	if includeTest {
   476  		for _, name := range pkg.TestGoFiles {
   477  			filenames = append(filenames, filepath.Join(pkg.Dir, name))
   478  		}
   479  	}
   480  	return filenames, nil
   481  }
   482  
   483  func walkPkgDirs(dir string, pkgh func(dir string, filenames []string), errh func(args ...any)) {
   484  	w := walker{pkgh, errh}
   485  	w.walk(dir)
   486  }
   487  
   488  type walker struct {
   489  	pkgh func(dir string, filenames []string)
   490  	errh func(args ...any)
   491  }
   492  
   493  func (w *walker) walk(dir string) {
   494  	files, err := os.ReadDir(dir)
   495  	if err != nil {
   496  		w.errh(err)
   497  		return
   498  	}
   499  
   500  	// apply pkgh to the files in directory dir
   501  
   502  	// Don't get test files as these packages are imported.
   503  	pkgFiles, err := pkgFilenames(dir, false)
   504  	if err != nil {
   505  		w.errh(err)
   506  		return
   507  	}
   508  	if pkgFiles != nil {
   509  		w.pkgh(dir, pkgFiles)
   510  	}
   511  
   512  	// traverse subdirectories, but don't walk into testdata
   513  	for _, f := range files {
   514  		if f.IsDir() && f.Name() != "testdata" {
   515  			w.walk(filepath.Join(dir, f.Name()))
   516  		}
   517  	}
   518  }
   519
View as plain text