script_test.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Script-driven tests.
     6  // See testdata/script/README for an overview.
     7  
     8  //go:generate go test cmd/go -v -run=TestScript/README --fixreadme
     9  
    10  package main_test
    11  
    12  import (
    13  	"bufio"
    14  	"bytes"
    15  	"context"
    16  	_ "embed"
    17  	"flag"
    18  	"internal/testenv"
    19  	"internal/txtar"
    20  	"net/url"
    21  	"os"
    22  	"path/filepath"
    23  	"runtime"
    24  	"strings"
    25  	"testing"
    26  	"time"
    27  
    28  	"cmd/go/internal/cfg"
    29  	"cmd/go/internal/gover"
    30  	"cmd/go/internal/vcweb/vcstest"
    31  	"cmd/internal/script"
    32  	"cmd/internal/script/scripttest"
    33  
    34  	"golang.org/x/telemetry/counter/countertest"
    35  )
    36  
    37  var testSum = flag.String("testsum", "", `may be tidy, listm, or listall. If set, TestScript generates a go.sum file at the beginning of each test and updates test files if they pass.`)
    38  
    39  // TestScript runs the tests in testdata/script/*.txt.
    40  func TestScript(t *testing.T) {
    41  	testenv.MustHaveGoBuild(t)
    42  	testenv.SkipIfShortAndSlow(t)
    43  
    44  	if testing.Short() && runtime.GOOS == "plan9" {
    45  		t.Skipf("skipping test in -short mode on %s", runtime.GOOS)
    46  	}
    47  
    48  	srv, err := vcstest.NewServer()
    49  	if err != nil {
    50  		t.Fatal(err)
    51  	}
    52  	t.Cleanup(func() {
    53  		if err := srv.Close(); err != nil {
    54  			t.Fatal(err)
    55  		}
    56  	})
    57  	certFile, err := srv.WriteCertificateFile()
    58  	if err != nil {
    59  		t.Fatal(err)
    60  	}
    61  
    62  	StartProxy()
    63  
    64  	var (
    65  		ctx         = context.Background()
    66  		gracePeriod = 100 * time.Millisecond
    67  	)
    68  	if deadline, ok := t.Deadline(); ok {
    69  		timeout := time.Until(deadline)
    70  
    71  		// If time allows, increase the termination grace period to 5% of the
    72  		// remaining time.
    73  		if gp := timeout / 20; gp > gracePeriod {
    74  			gracePeriod = gp
    75  		}
    76  
    77  		// When we run commands that execute subprocesses, we want to reserve two
    78  		// grace periods to clean up. We will send the first termination signal when
    79  		// the context expires, then wait one grace period for the process to
    80  		// produce whatever useful output it can (such as a stack trace). After the
    81  		// first grace period expires, we'll escalate to os.Kill, leaving the second
    82  		// grace period for the test function to record its output before the test
    83  		// process itself terminates.
    84  		timeout -= 2 * gracePeriod
    85  
    86  		var cancel context.CancelFunc
    87  		ctx, cancel = context.WithTimeout(ctx, timeout)
    88  		t.Cleanup(cancel)
    89  	}
    90  
    91  	env, err := scriptEnv(srv, certFile)
    92  	if err != nil {
    93  		t.Fatal(err)
    94  	}
    95  	engine := &script.Engine{
    96  		Conds: scriptConditions(t),
    97  		Cmds:  scriptCommands(quitSignal(), gracePeriod),
    98  		Quiet: !testing.Verbose(),
    99  	}
   100  
   101  	t.Run("README", func(t *testing.T) {
   102  		checkScriptReadme(t, engine, env)
   103  	})
   104  
   105  	files, err := filepath.Glob("testdata/script/*.txt")
   106  	if err != nil {
   107  		t.Fatal(err)
   108  	}
   109  	for _, file := range files {
   110  		file := file
   111  		name := strings.TrimSuffix(filepath.Base(file), ".txt")
   112  		t.Run(name, func(t *testing.T) {
   113  			t.Parallel()
   114  			StartProxy()
   115  
   116  			workdir, err := os.MkdirTemp(testTmpDir, name)
   117  			if err != nil {
   118  				t.Fatal(err)
   119  			}
   120  			if !*testWork {
   121  				defer removeAll(workdir)
   122  			}
   123  
   124  			s, err := script.NewState(tbContext(ctx, t), workdir, env)
   125  			if err != nil {
   126  				t.Fatal(err)
   127  			}
   128  
   129  			// Unpack archive.
   130  			a, err := txtar.ParseFile(file)
   131  			if err != nil {
   132  				t.Fatal(err)
   133  			}
   134  			telemetryDir := initScriptDirs(t, s)
   135  			if err := s.ExtractFiles(a); err != nil {
   136  				t.Fatal(err)
   137  			}
   138  
   139  			t.Log(time.Now().UTC().Format(time.RFC3339))
   140  			work, _ := s.LookupEnv("WORK")
   141  			t.Logf("$WORK=%s", work)
   142  
   143  			// With -testsum, if a go.mod file is present in the test's initial
   144  			// working directory, run 'go mod tidy'.
   145  			if *testSum != "" {
   146  				if updateSum(t, engine, s, a) {
   147  					defer func() {
   148  						if t.Failed() {
   149  							return
   150  						}
   151  						data := txtar.Format(a)
   152  						if err := os.WriteFile(file, data, 0666); err != nil {
   153  							t.Errorf("rewriting test file: %v", err)
   154  						}
   155  					}()
   156  				}
   157  			}
   158  
   159  			// Note: Do not use filepath.Base(file) here:
   160  			// editors that can jump to file:line references in the output
   161  			// will work better seeing the full path relative to cmd/go
   162  			// (where the "go test" command is usually run).
   163  			scripttest.Run(t, engine, s, file, bytes.NewReader(a.Comment))
   164  			checkCounters(t, telemetryDir)
   165  		})
   166  	}
   167  }
   168  
   169  // testingTBKey is the Context key for a testing.TB.
   170  type testingTBKey struct{}
   171  
   172  // tbContext returns a Context derived from ctx and associated with t.
   173  func tbContext(ctx context.Context, t testing.TB) context.Context {
   174  	return context.WithValue(ctx, testingTBKey{}, t)
   175  }
   176  
   177  // tbFromContext returns the testing.TB associated with ctx, if any.
   178  func tbFromContext(ctx context.Context) (testing.TB, bool) {
   179  	t := ctx.Value(testingTBKey{})
   180  	if t == nil {
   181  		return nil, false
   182  	}
   183  	return t.(testing.TB), true
   184  }
   185  
   186  // initScriptDirs creates the initial directory structure in s for unpacking a
   187  // cmd/go script.
   188  func initScriptDirs(t testing.TB, s *script.State) (telemetryDir string) {
   189  	must := func(err error) {
   190  		if err != nil {
   191  			t.Helper()
   192  			t.Fatal(err)
   193  		}
   194  	}
   195  
   196  	work := s.Getwd()
   197  	must(s.Setenv("WORK", work))
   198  
   199  	telemetryDir = filepath.Join(work, "telemetry")
   200  	must(os.MkdirAll(telemetryDir, 0777))
   201  	must(s.Setenv("TEST_TELEMETRY_DIR", filepath.Join(work, "telemetry")))
   202  
   203  	must(os.MkdirAll(filepath.Join(work, "tmp"), 0777))
   204  	must(s.Setenv(tempEnvName(), filepath.Join(work, "tmp")))
   205  
   206  	gopath := filepath.Join(work, "gopath")
   207  	must(s.Setenv("GOPATH", gopath))
   208  	gopathSrc := filepath.Join(gopath, "src")
   209  	must(os.MkdirAll(gopathSrc, 0777))
   210  	must(s.Chdir(gopathSrc))
   211  	return telemetryDir
   212  }
   213  
   214  func scriptEnv(srv *vcstest.Server, srvCertFile string) ([]string, error) {
   215  	httpURL, err := url.Parse(srv.HTTP.URL)
   216  	if err != nil {
   217  		return nil, err
   218  	}
   219  	httpsURL, err := url.Parse(srv.HTTPS.URL)
   220  	if err != nil {
   221  		return nil, err
   222  	}
   223  	env := []string{
   224  		pathEnvName() + "=" + testBin + string(filepath.ListSeparator) + os.Getenv(pathEnvName()),
   225  		homeEnvName() + "=/no-home",
   226  		"CCACHE_DISABLE=1", // ccache breaks with non-existent HOME
   227  		"GOARCH=" + runtime.GOARCH,
   228  		"TESTGO_GOHOSTARCH=" + goHostArch,
   229  		"GOCACHE=" + testGOCACHE,
   230  		"GOCOVERDIR=" + os.Getenv("GOCOVERDIR"),
   231  		"GODEBUG=" + os.Getenv("GODEBUG"),
   232  		"GOEXE=" + cfg.ExeSuffix,
   233  		"GOEXPERIMENT=" + os.Getenv("GOEXPERIMENT"),
   234  		"GOOS=" + runtime.GOOS,
   235  		"TESTGO_GOHOSTOS=" + goHostOS,
   236  		"GOPROXY=" + proxyURL,
   237  		"GOPRIVATE=",
   238  		"GOROOT=" + testGOROOT,
   239  		"GOTRACEBACK=system",
   240  		"TESTGONETWORK=panic", // allow only local connections by default; the [net] condition resets this
   241  		"TESTGO_GOROOT=" + testGOROOT,
   242  		"TESTGO_EXE=" + testGo,
   243  		"TESTGO_VCSTEST_HOST=" + httpURL.Host,
   244  		"TESTGO_VCSTEST_TLS_HOST=" + httpsURL.Host,
   245  		"TESTGO_VCSTEST_CERT=" + srvCertFile,
   246  		"TESTGONETWORK=panic", // cleared by the [net] condition
   247  		"GOSUMDB=" + testSumDBVerifierKey,
   248  		"GONOPROXY=",
   249  		"GONOSUMDB=",
   250  		"GOVCS=*:all",
   251  		"devnull=" + os.DevNull,
   252  		"goversion=" + gover.Local(),
   253  		"CMDGO_TEST_RUN_MAIN=true",
   254  		"HGRCPATH=",
   255  		"GOTOOLCHAIN=auto",
   256  		"newline=\n",
   257  	}
   258  
   259  	if testenv.Builder() != "" || os.Getenv("GIT_TRACE_CURL") == "1" {
   260  		// To help diagnose https://go.dev/issue/52545,
   261  		// enable tracing for Git HTTPS requests.
   262  		env = append(env,
   263  			"GIT_TRACE_CURL=1",
   264  			"GIT_TRACE_CURL_NO_DATA=1",
   265  			"GIT_REDACT_COOKIES=o,SSO,GSSO_Uberproxy")
   266  	}
   267  	if testing.Short() {
   268  		// VCS commands are always somewhat slow: they either require access to external hosts,
   269  		// or they require our intercepted vcs-test.golang.org to regenerate the repository.
   270  		// Require all tests that use VCS commands which require remote lookups to be skipped in
   271  		// short mode.
   272  		env = append(env, "TESTGOVCSREMOTE=panic")
   273  	}
   274  	if os.Getenv("CGO_ENABLED") != "" || runtime.GOOS != goHostOS || runtime.GOARCH != goHostArch {
   275  		// If the actual CGO_ENABLED might not match the cmd/go default, set it
   276  		// explicitly in the environment. Otherwise, leave it unset so that we also
   277  		// cover the default behaviors.
   278  		env = append(env, "CGO_ENABLED="+cgoEnabled)
   279  	}
   280  
   281  	for _, key := range extraEnvKeys {
   282  		if val, ok := os.LookupEnv(key); ok {
   283  			env = append(env, key+"="+val)
   284  		}
   285  	}
   286  
   287  	return env, nil
   288  }
   289  
   290  var extraEnvKeys = []string{
   291  	"SYSTEMROOT",         // must be preserved on Windows to find DLLs; golang.org/issue/25210
   292  	"WINDIR",             // must be preserved on Windows to be able to run PowerShell command; golang.org/issue/30711
   293  	"LD_LIBRARY_PATH",    // must be preserved on Unix systems to find shared libraries
   294  	"LIBRARY_PATH",       // allow override of non-standard static library paths
   295  	"C_INCLUDE_PATH",     // allow override non-standard include paths
   296  	"CC",                 // don't lose user settings when invoking cgo
   297  	"GO_TESTING_GOTOOLS", // for gccgo testing
   298  	"GCCGO",              // for gccgo testing
   299  	"GCCGOTOOLDIR",       // for gccgo testing
   300  }
   301  
   302  // updateSum runs 'go mod tidy', 'go list -mod=mod -m all', or
   303  // 'go list -mod=mod all' in the test's current directory if a file named
   304  // "go.mod" is present after the archive has been extracted. updateSum modifies
   305  // archive and returns true if go.mod or go.sum were changed.
   306  func updateSum(t testing.TB, e *script.Engine, s *script.State, archive *txtar.Archive) (rewrite bool) {
   307  	gomodIdx, gosumIdx := -1, -1
   308  	for i := range archive.Files {
   309  		switch archive.Files[i].Name {
   310  		case "go.mod":
   311  			gomodIdx = i
   312  		case "go.sum":
   313  			gosumIdx = i
   314  		}
   315  	}
   316  	if gomodIdx < 0 {
   317  		return false
   318  	}
   319  
   320  	var cmd string
   321  	switch *testSum {
   322  	case "tidy":
   323  		cmd = "go mod tidy"
   324  	case "listm":
   325  		cmd = "go list -m -mod=mod all"
   326  	case "listall":
   327  		cmd = "go list -mod=mod all"
   328  	default:
   329  		t.Fatalf(`unknown value for -testsum %q; may be "tidy", "listm", or "listall"`, *testSum)
   330  	}
   331  
   332  	log := new(strings.Builder)
   333  	err := e.Execute(s, "updateSum", bufio.NewReader(strings.NewReader(cmd)), log)
   334  	if log.Len() > 0 {
   335  		t.Logf("%s", log)
   336  	}
   337  	if err != nil {
   338  		t.Fatal(err)
   339  	}
   340  
   341  	newGomodData, err := os.ReadFile(s.Path("go.mod"))
   342  	if err != nil {
   343  		t.Fatalf("reading go.mod after -testsum: %v", err)
   344  	}
   345  	if !bytes.Equal(newGomodData, archive.Files[gomodIdx].Data) {
   346  		archive.Files[gomodIdx].Data = newGomodData
   347  		rewrite = true
   348  	}
   349  
   350  	newGosumData, err := os.ReadFile(s.Path("go.sum"))
   351  	if err != nil && !os.IsNotExist(err) {
   352  		t.Fatalf("reading go.sum after -testsum: %v", err)
   353  	}
   354  	switch {
   355  	case os.IsNotExist(err) && gosumIdx >= 0:
   356  		// go.sum was deleted.
   357  		rewrite = true
   358  		archive.Files = append(archive.Files[:gosumIdx], archive.Files[gosumIdx+1:]...)
   359  	case err == nil && gosumIdx < 0:
   360  		// go.sum was created.
   361  		rewrite = true
   362  		gosumIdx = gomodIdx + 1
   363  		archive.Files = append(archive.Files, txtar.File{})
   364  		copy(archive.Files[gosumIdx+1:], archive.Files[gosumIdx:])
   365  		archive.Files[gosumIdx] = txtar.File{Name: "go.sum", Data: newGosumData}
   366  	case err == nil && gosumIdx >= 0 && !bytes.Equal(newGosumData, archive.Files[gosumIdx].Data):
   367  		// go.sum was changed.
   368  		rewrite = true
   369  		archive.Files[gosumIdx].Data = newGosumData
   370  	}
   371  	return rewrite
   372  }
   373  
   374  func readCounters(t *testing.T, telemetryDir string) map[string]uint64 {
   375  	localDir := filepath.Join(telemetryDir, "local")
   376  	dirents, err := os.ReadDir(localDir)
   377  	if err != nil {
   378  		if os.IsNotExist(err) {
   379  			return nil // The Go command didn't ever run so the local dir wasn't created
   380  		}
   381  		t.Fatalf("reading telemetry local dir: %v", err)
   382  	}
   383  	totals := map[string]uint64{}
   384  	for _, dirent := range dirents {
   385  		if dirent.IsDir() || !strings.HasSuffix(dirent.Name(), ".count") {
   386  			// not a counter file
   387  			continue
   388  		}
   389  		counters, _, err := countertest.ReadFile(filepath.Join(localDir, dirent.Name()))
   390  		if err != nil {
   391  			t.Fatalf("reading counter file: %v", err)
   392  		}
   393  		for k, v := range counters {
   394  			totals[k] += v
   395  		}
   396  	}
   397  
   398  	return totals
   399  }
   400  
   401  func checkCounters(t *testing.T, telemetryDir string) {
   402  	counters := readCounters(t, telemetryDir)
   403  	if _, ok := scriptGoInvoked.Load(testing.TB(t)); ok {
   404  		if !disabledOnPlatform && len(counters) == 0 {
   405  			t.Fatal("go was invoked but no counters were incremented")
   406  		}
   407  	}
   408  }
   409  
   410  // Copied from https://go.googlesource.com/telemetry/+/5f08a0cbff3f/internal/telemetry/mode.go#122
   411  // TODO(go.dev/issues/66205): replace this with the public API once it becomes available.
   412  //
   413  // disabledOnPlatform indicates whether telemetry is disabled
   414  // due to bugs in the current platform.
   415  const disabledOnPlatform = false ||
   416  	// The following platforms could potentially be supported in the future:
   417  	runtime.GOOS == "openbsd" || // #60614
   418  	runtime.GOOS == "solaris" || // #60968 #60970
   419  	runtime.GOOS == "android" || // #60967
   420  	runtime.GOOS == "illumos" || // #65544
   421  	// These platforms fundamentally can't be supported:
   422  	runtime.GOOS == "js" || // #60971
   423  	runtime.GOOS == "wasip1" || // #60971
   424  	runtime.GOOS == "plan9" // https://github.com/golang/go/issues/57540#issuecomment-1470766639
   425
View as plain text