testing.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package testing provides support for automated testing of Go packages.
     6  // It is intended to be used in concert with the “go test” utility, which automates
     7  // execution of any function of the form
     8  //     func TestXxx(*testing.T)
     9  // where Xxx can be any alphanumeric string (but the first letter must not be in
    10  // [a-z]) and serves to identify the test routine.
    11  // These TestXxx routines should be declared within the package they are testing.
    12  //
    13  // Functions of the form
    14  //     func BenchmarkXxx(*testing.B)
    15  // are considered benchmarks, and are executed by go test when the -test.bench
    16  // flag is provided.
    17  //
    18  // A sample benchmark function looks like this:
    19  //     func BenchmarkHello(b *testing.B) {
    20  //         for i := 0; i < b.N; i++ {
    21  //             fmt.Sprintf("hello")
    22  //         }
    23  //     }
    24  // The benchmark package will vary b.N until the benchmark function lasts
    25  // long enough to be timed reliably. The output
    26  //     testing.BenchmarkHello    10000000    282 ns/op
    27  // means that the loop ran 10000000 times at a speed of 282 ns per loop.
    28  //
    29  // If a benchmark needs some expensive setup before running, the timer
    30  // may be stopped:
    31  //     func BenchmarkBigLen(b *testing.B) {
    32  //         b.StopTimer()
    33  //         big := NewBig()
    34  //         b.StartTimer()
    35  //         for i := 0; i < b.N; i++ {
    36  //             big.Len()
    37  //         }
    38  //     }
    39  package testing
    40  
    41  import (
    42  	"flag"
    43  	"fmt"
    44  	"os"
    45  	"runtime"
    46  	"runtime/pprof"
    47  	"strconv"
    48  	"strings"
    49  	"time"
    50  )
    51  
    52  var (
    53  	// The short flag requests that tests run more quickly, but its functionality
    54  	// is provided by test writers themselves. The testing package is just its
    55  	// home. The all.bash installation script sets it to make installation more
    56  	// efficient, but by default the flag is off so a plain "go test" will do a
    57  	// full test of the package.
    58  	short = flag.Bool("test.short", false, "run smaller test suite to save time")
    59  
    60  	// Report as tests are run; default is silent for success.
    61  	chatty         = flag.Bool("test.v", false, "verbose: print additional output")
    62  	match          = flag.String("test.run", "", "regular expression to select tests to run")
    63  	memProfile     = flag.String("test.memprofile", "", "write a memory profile to the named file after execution")
    64  	memProfileRate = flag.Int("test.memprofilerate", 0, "if >=0, sets runtime.MemProfileRate")
    65  	cpuProfile     = flag.String("test.cpuprofile", "", "write a cpu profile to the named file during execution")
    66  	timeout        = flag.Duration("test.timeout", 0, "if positive, sets an aggregate time limit for all tests")
    67  	cpuListStr     = flag.String("test.cpu", "", "comma-separated list of number of CPUs to use for each test")
    68  	parallel       = flag.Int("test.parallel", runtime.GOMAXPROCS(0), "maximum test parallelism")
    69  
    70  	cpuList []int
    71  )
    72  
    73  // common holds the elements common between T and B and
    74  // captures common methods such as Errorf.
    75  type common struct {
    76  	output   []byte    // Output generated by test or benchmark.
    77  	failed   bool      // Test or benchmark has failed.
    78  	start    time.Time // Time test or benchmark started
    79  	duration time.Duration
    80  	self     any      // To be sent on signal channel when done.
    81  	signal   chan any // Output for serial tests.
    82  }
    83  
    84  // Short reports whether the -test.short flag is set.
    85  func Short() bool {
    86  	return *short
    87  }
    88  
    89  // decorate inserts the final newline if needed and indentation tabs for formatting.
    90  // If addFileLine is true, it also prefixes the string with the file and line of the call site.
    91  func decorate(s string, addFileLine bool) string {
    92  	if addFileLine {
    93  		_, file, line, ok := runtime.Caller(3) // decorate + log + public function.
    94  		if ok {
    95  			// Truncate file name at last file name separator.
    96  			if index := strings.LastIndex(file, "/"); index >= 0 {
    97  				file = file[index+1:]
    98  			} else if index = strings.LastIndex(file, "\\"); index >= 0 {
    99  				file = file[index+1:]
   100  			}
   101  		} else {
   102  			file = "???"
   103  			line = 1
   104  		}
   105  		s = fmt.Sprintf("%s:%d: %s", file, line, s)
   106  	}
   107  	s = "\t" + s // Every line is indented at least one tab.
   108  	n := len(s)
   109  	if n > 0 && s[n-1] != '\n' {
   110  		s += "\n"
   111  		n++
   112  	}
   113  	for i := 0; i < n-1; i++ { // -1 to avoid final newline
   114  		if s[i] == '\n' {
   115  			// Second and subsequent lines are indented an extra tab.
   116  			return s[0:i+1] + "\t" + decorate(s[i+1:n], false)
   117  		}
   118  	}
   119  	return s
   120  }
   121  
   122  // T is a type passed to Test functions to manage test state and support formatted test logs.
   123  // Logs are accumulated during execution and dumped to standard error when done.
   124  type T struct {
   125  	common
   126  	name          string    // Name of test.
   127  	startParallel chan bool // Parallel tests will wait on this.
   128  }
   129  
   130  // Fail marks the function as having failed but continues execution.
   131  func (c *common) Fail() { c.failed = true }
   132  
   133  // Failed reports whether the function has failed.
   134  func (c *common) Failed() bool { return c.failed }
   135  
   136  // FailNow marks the function as having failed and stops its execution.
   137  // Execution will continue at the next Test.
   138  func (c *common) FailNow() {
   139  	c.Fail()
   140  
   141  	// Calling runtime.Goexit will exit the goroutine, which
   142  	// will run the deferred functions in this goroutine,
   143  	// which will eventually run the deferred lines in tRunner,
   144  	// which will signal to the test loop that this test is done.
   145  	//
   146  	// A previous version of this code said:
   147  	//
   148  	//	c.duration = ...
   149  	//	c.signal <- c.self
   150  	//	runtime.Goexit()
   151  	//
   152  	// This previous version duplicated code (those lines are in
   153  	// tRunner no matter what), but worse the goroutine teardown
   154  	// implicit in runtime.Goexit was not guaranteed to complete
   155  	// before the test exited. If a test deferred an important cleanup
   156  	// function (like removing temporary files), there was no guarantee
   157  	// it would run on a test failure. Because we send on c.signal during
   158  	// a top-of-stack deferred function now, we know that the send
   159  	// only happens after any other stacked defers have completed.
   160  	runtime.Goexit()
   161  }
   162  
   163  // log generates the output. It's always at the same stack depth.
   164  func (c *common) log(s string) {
   165  	c.output = append(c.output, decorate(s, true)...)
   166  }
   167  
   168  // Log formats its arguments using default formatting, analogous to Println(),
   169  // and records the text in the error log.
   170  func (c *common) Log(args ...any) { c.log(fmt.Sprintln(args...)) }
   171  
   172  // Logf formats its arguments according to the format, analogous to Printf(),
   173  // and records the text in the error log.
   174  func (c *common) Logf(format string, args ...any) { c.log(fmt.Sprintf(format, args...)) }
   175  
   176  // Error is equivalent to Log() followed by Fail().
   177  func (c *common) Error(args ...any) {
   178  	c.log(fmt.Sprintln(args...))
   179  	c.Fail()
   180  }
   181  
   182  // Errorf is equivalent to Logf() followed by Fail().
   183  func (c *common) Errorf(format string, args ...any) {
   184  	c.log(fmt.Sprintf(format, args...))
   185  	c.Fail()
   186  }
   187  
   188  // Fatal is equivalent to Log() followed by FailNow().
   189  func (c *common) Fatal(args ...any) {
   190  	c.log(fmt.Sprintln(args...))
   191  	c.FailNow()
   192  }
   193  
   194  // Fatalf is equivalent to Logf() followed by FailNow().
   195  func (c *common) Fatalf(format string, args ...any) {
   196  	c.log(fmt.Sprintf(format, args...))
   197  	c.FailNow()
   198  }
   199  
   200  // Parallel signals that this test is to be run in parallel with (and only with)
   201  // other parallel tests in this CPU group.
   202  func (t *T) Parallel() {
   203  	t.signal <- (*T)(nil) // Release main testing loop
   204  	<-t.startParallel     // Wait for serial tests to finish
   205  }
   206  
   207  // An internal type but exported because it is cross-package; part of the implementation
   208  // of go test.
   209  type InternalTest struct {
   210  	Name string
   211  	F    func(*T)
   212  }
   213  
   214  func tRunner(t *T, test *InternalTest) {
   215  	t.start = time.Now()
   216  
   217  	// When this goroutine is done, either because test.F(t)
   218  	// returned normally or because a test failure triggered
   219  	// a call to runtime.Goexit, record the duration and send
   220  	// a signal saying that the test is done.
   221  	defer func() {
   222  		t.duration = time.Since(t.start)
   223  		t.signal <- t
   224  	}()
   225  
   226  	test.F(t)
   227  }
   228  
   229  // An internal function but exported because it is cross-package; part of the implementation
   230  // of go test.
   231  func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) {
   232  	flag.Parse()
   233  	parseCpuList()
   234  
   235  	before()
   236  	startAlarm()
   237  	testOk := RunTests(matchString, tests)
   238  	exampleOk := RunExamples(examples)
   239  	if !testOk || !exampleOk {
   240  		fmt.Println("FAIL")
   241  		os.Exit(1)
   242  	}
   243  	fmt.Println("PASS")
   244  	stopAlarm()
   245  	RunBenchmarks(matchString, benchmarks)
   246  	after()
   247  }
   248  
   249  func (t *T) report() {
   250  	tstr := fmt.Sprintf("(%.2f seconds)", t.duration.Seconds())
   251  	format := "--- %s: %s %s\n%s"
   252  	if t.failed {
   253  		fmt.Printf(format, "FAIL", t.name, tstr, t.output)
   254  	} else if *chatty {
   255  		fmt.Printf(format, "PASS", t.name, tstr, t.output)
   256  	}
   257  }
   258  
   259  func RunTests(matchString func(pat, str string) (bool, error), tests []InternalTest) (ok bool) {
   260  	ok = true
   261  	if len(tests) == 0 {
   262  		fmt.Fprintln(os.Stderr, "testing: warning: no tests to run")
   263  		return
   264  	}
   265  	for _, procs := range cpuList {
   266  		runtime.GOMAXPROCS(procs)
   267  		// We build a new channel tree for each run of the loop.
   268  		// collector merges in one channel all the upstream signals from parallel tests.
   269  		// If all tests pump to the same channel, a bug can occur where a test
   270  		// kicks off a goroutine that Fails, yet the test still delivers a completion signal,
   271  		// which skews the counting.
   272  		var collector = make(chan any)
   273  
   274  		numParallel := 0
   275  		startParallel := make(chan bool)
   276  
   277  		for i := 0; i < len(tests); i++ {
   278  			matched, err := matchString(*match, tests[i].Name)
   279  			if err != nil {
   280  				fmt.Fprintf(os.Stderr, "testing: invalid regexp for -test.run: %s\n", err)
   281  				os.Exit(1)
   282  			}
   283  			if !matched {
   284  				continue
   285  			}
   286  			testName := tests[i].Name
   287  			if procs != 1 {
   288  				testName = fmt.Sprintf("%s-%d", tests[i].Name, procs)
   289  			}
   290  			t := &T{
   291  				common: common{
   292  					signal: make(chan any),
   293  				},
   294  				name:          testName,
   295  				startParallel: startParallel,
   296  			}
   297  			t.self = t
   298  			if *chatty {
   299  				fmt.Printf("=== RUN %s\n", t.name)
   300  			}
   301  			go tRunner(t, &tests[i])
   302  			out := (<-t.signal).(*T)
   303  			if out == nil { // Parallel run.
   304  				go func() {
   305  					collector <- <-t.signal
   306  				}()
   307  				numParallel++
   308  				continue
   309  			}
   310  			t.report()
   311  			ok = ok && !out.failed
   312  		}
   313  
   314  		running := 0
   315  		for numParallel+running > 0 {
   316  			if running < *parallel && numParallel > 0 {
   317  				startParallel <- true
   318  				running++
   319  				numParallel--
   320  				continue
   321  			}
   322  			t := (<-collector).(*T)
   323  			t.report()
   324  			ok = ok && !t.failed
   325  			running--
   326  		}
   327  	}
   328  	return
   329  }
   330  
   331  // before runs before all testing.
   332  func before() {
   333  	if *memProfileRate > 0 {
   334  		runtime.MemProfileRate = *memProfileRate
   335  	}
   336  	if *cpuProfile != "" {
   337  		f, err := os.Create(*cpuProfile)
   338  		if err != nil {
   339  			fmt.Fprintf(os.Stderr, "testing: %s", err)
   340  			return
   341  		}
   342  		if err := pprof.StartCPUProfile(f); err != nil {
   343  			fmt.Fprintf(os.Stderr, "testing: can't start cpu profile: %s", err)
   344  			f.Close()
   345  			return
   346  		}
   347  		// Could save f so after can call f.Close; not worth the effort.
   348  	}
   349  
   350  }
   351  
   352  // after runs after all testing.
   353  func after() {
   354  	if *cpuProfile != "" {
   355  		pprof.StopCPUProfile() // flushes profile to disk
   356  	}
   357  	if *memProfile != "" {
   358  		f, err := os.Create(*memProfile)
   359  		if err != nil {
   360  			fmt.Fprintf(os.Stderr, "testing: %s", err)
   361  			return
   362  		}
   363  		if err = pprof.WriteHeapProfile(f); err != nil {
   364  			fmt.Fprintf(os.Stderr, "testing: can't write %s: %s", *memProfile, err)
   365  		}
   366  		f.Close()
   367  	}
   368  }
   369  
   370  var timer *time.Timer
   371  
   372  // startAlarm starts an alarm if requested.
   373  func startAlarm() {
   374  	if *timeout > 0 {
   375  		timer = time.AfterFunc(*timeout, alarm)
   376  	}
   377  }
   378  
   379  // stopAlarm turns off the alarm.
   380  func stopAlarm() {
   381  	if *timeout > 0 {
   382  		timer.Stop()
   383  	}
   384  }
   385  
   386  // alarm is called if the timeout expires.
   387  func alarm() {
   388  	panic("test timed out")
   389  }
   390  
   391  func parseCpuList() {
   392  	if len(*cpuListStr) == 0 {
   393  		cpuList = append(cpuList, runtime.GOMAXPROCS(-1))
   394  	} else {
   395  		for _, val := range strings.Split(*cpuListStr, ",") {
   396  			cpu, err := strconv.Atoi(val)
   397  			if err != nil || cpu <= 0 {
   398  				fmt.Fprintf(os.Stderr, "testing: invalid value %q for -test.cpu", val)
   399  				os.Exit(1)
   400  			}
   401  			cpuList = append(cpuList, cpu)
   402  		}
   403  	}
   404  }
   405
View as plain text