Source file src/testing/sub_test.go

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package testing
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"regexp"
    11  	"runtime"
    12  	"slices"
    13  	"strings"
    14  	"sync"
    15  	"sync/atomic"
    16  	"time"
    17  )
    18  
    19  func init() {
    20  	// Make benchmark tests run 10x faster.
    21  	benchTime.d = 100 * time.Millisecond
    22  }
    23  
    24  func TestTestState(t *T) {
    25  	const (
    26  		add1 = 0
    27  		done = 1
    28  	)
    29  	type call struct {
    30  		typ int // run or done
    31  		// result from applying the call
    32  		running int
    33  		waiting int
    34  		started bool
    35  	}
    36  	testCases := []struct {
    37  		max int
    38  		run []call
    39  	}{{
    40  		max: 1,
    41  		run: []call{
    42  			{typ: add1, running: 1, waiting: 0, started: true},
    43  			{typ: done, running: 0, waiting: 0, started: false},
    44  		},
    45  	}, {
    46  		max: 1,
    47  		run: []call{
    48  			{typ: add1, running: 1, waiting: 0, started: true},
    49  			{typ: add1, running: 1, waiting: 1, started: false},
    50  			{typ: done, running: 1, waiting: 0, started: true},
    51  			{typ: done, running: 0, waiting: 0, started: false},
    52  			{typ: add1, running: 1, waiting: 0, started: true},
    53  		},
    54  	}, {
    55  		max: 3,
    56  		run: []call{
    57  			{typ: add1, running: 1, waiting: 0, started: true},
    58  			{typ: add1, running: 2, waiting: 0, started: true},
    59  			{typ: add1, running: 3, waiting: 0, started: true},
    60  			{typ: add1, running: 3, waiting: 1, started: false},
    61  			{typ: add1, running: 3, waiting: 2, started: false},
    62  			{typ: add1, running: 3, waiting: 3, started: false},
    63  			{typ: done, running: 3, waiting: 2, started: true},
    64  			{typ: add1, running: 3, waiting: 3, started: false},
    65  			{typ: done, running: 3, waiting: 2, started: true},
    66  			{typ: done, running: 3, waiting: 1, started: true},
    67  			{typ: done, running: 3, waiting: 0, started: true},
    68  			{typ: done, running: 2, waiting: 0, started: false},
    69  			{typ: done, running: 1, waiting: 0, started: false},
    70  			{typ: done, running: 0, waiting: 0, started: false},
    71  		},
    72  	}}
    73  	for i, tc := range testCases {
    74  		tstate := &testState{
    75  			startParallel: make(chan bool),
    76  			maxParallel:   tc.max,
    77  		}
    78  		for j, call := range tc.run {
    79  			doCall := func(f func()) chan bool {
    80  				done := make(chan bool)
    81  				go func() {
    82  					f()
    83  					done <- true
    84  				}()
    85  				return done
    86  			}
    87  			started := false
    88  			switch call.typ {
    89  			case add1:
    90  				signal := doCall(tstate.waitParallel)
    91  				select {
    92  				case <-signal:
    93  					started = true
    94  				case tstate.startParallel <- true:
    95  					<-signal
    96  				}
    97  			case done:
    98  				signal := doCall(tstate.release)
    99  				select {
   100  				case <-signal:
   101  				case <-tstate.startParallel:
   102  					started = true
   103  					<-signal
   104  				}
   105  			}
   106  			if started != call.started {
   107  				t.Errorf("%d:%d:started: got %v; want %v", i, j, started, call.started)
   108  			}
   109  			if tstate.running != call.running {
   110  				t.Errorf("%d:%d:running: got %v; want %v", i, j, tstate.running, call.running)
   111  			}
   112  			if tstate.numWaiting != call.waiting {
   113  				t.Errorf("%d:%d:waiting: got %v; want %v", i, j, tstate.numWaiting, call.waiting)
   114  			}
   115  		}
   116  	}
   117  }
   118  
   119  func TestTRun(t *T) {
   120  	realTest := t
   121  	testCases := []struct {
   122  		desc   string
   123  		ok     bool
   124  		maxPar int
   125  		chatty bool
   126  		json   bool
   127  		output string
   128  		f      func(*T)
   129  	}{{
   130  		desc:   "failnow skips future sequential and parallel tests at same level",
   131  		ok:     false,
   132  		maxPar: 1,
   133  		output: `
   134  --- FAIL: failnow skips future sequential and parallel tests at same level (N.NNs)
   135      --- FAIL: failnow skips future sequential and parallel tests at same level/#00 (N.NNs)
   136      `,
   137  		f: func(t *T) {
   138  			ranSeq := false
   139  			ranPar := false
   140  			t.Run("", func(t *T) {
   141  				t.Run("par", func(t *T) {
   142  					t.Parallel()
   143  					ranPar = true
   144  				})
   145  				t.Run("seq", func(t *T) {
   146  					ranSeq = true
   147  				})
   148  				t.FailNow()
   149  				t.Run("seq", func(t *T) {
   150  					realTest.Error("test must be skipped")
   151  				})
   152  				t.Run("par", func(t *T) {
   153  					t.Parallel()
   154  					realTest.Error("test must be skipped.")
   155  				})
   156  			})
   157  			if !ranPar {
   158  				realTest.Error("parallel test was not run")
   159  			}
   160  			if !ranSeq {
   161  				realTest.Error("sequential test was not run")
   162  			}
   163  		},
   164  	}, {
   165  		desc:   "failure in parallel test propagates upwards",
   166  		ok:     false,
   167  		maxPar: 1,
   168  		output: `
   169  --- FAIL: failure in parallel test propagates upwards (N.NNs)
   170      --- FAIL: failure in parallel test propagates upwards/#00 (N.NNs)
   171          --- FAIL: failure in parallel test propagates upwards/#00/par (N.NNs)
   172          `,
   173  		f: func(t *T) {
   174  			t.Run("", func(t *T) {
   175  				t.Parallel()
   176  				t.Run("par", func(t *T) {
   177  					t.Parallel()
   178  					t.Fail()
   179  				})
   180  			})
   181  		},
   182  	}, {
   183  		desc:   "skipping without message, chatty",
   184  		ok:     true,
   185  		chatty: true,
   186  		output: `
   187  === RUN   skipping without message, chatty
   188  --- SKIP: skipping without message, chatty (N.NNs)`,
   189  		f: func(t *T) { t.SkipNow() },
   190  	}, {
   191  		desc:   "chatty with recursion",
   192  		ok:     true,
   193  		chatty: true,
   194  		output: `
   195  === RUN   chatty with recursion
   196  === RUN   chatty with recursion/#00
   197  === RUN   chatty with recursion/#00/#00
   198  --- PASS: chatty with recursion (N.NNs)
   199      --- PASS: chatty with recursion/#00 (N.NNs)
   200          --- PASS: chatty with recursion/#00/#00 (N.NNs)`,
   201  		f: func(t *T) {
   202  			t.Run("", func(t *T) {
   203  				t.Run("", func(t *T) {})
   204  			})
   205  		},
   206  	}, {
   207  		desc:   "chatty with recursion and json",
   208  		ok:     false,
   209  		chatty: true,
   210  		json:   true,
   211  		output: `
   212  ^V=== RUN   chatty with recursion and json
   213  ^V=== RUN   chatty with recursion and json/#00
   214  ^V=== RUN   chatty with recursion and json/#00/#00
   215  ^V--- PASS: chatty with recursion and json/#00/#00 (N.NNs)
   216  ^V=== NAME  chatty with recursion and json/#00
   217  ^V=== RUN   chatty with recursion and json/#00/#01
   218      sub_test.go:NNN: skip
   219  ^V--- SKIP: chatty with recursion and json/#00/#01 (N.NNs)
   220  ^V=== NAME  chatty with recursion and json/#00
   221  ^V=== RUN   chatty with recursion and json/#00/#02
   222      sub_test.go:NNN: fail
   223  ^V--- FAIL: chatty with recursion and json/#00/#02 (N.NNs)
   224  ^V=== NAME  chatty with recursion and json/#00
   225  ^V--- FAIL: chatty with recursion and json/#00 (N.NNs)
   226  ^V=== NAME  chatty with recursion and json
   227  ^V--- FAIL: chatty with recursion and json (N.NNs)
   228  ^V=== NAME  `,
   229  		f: func(t *T) {
   230  			t.Run("", func(t *T) {
   231  				t.Run("", func(t *T) {})
   232  				t.Run("", func(t *T) { t.Skip("skip") })
   233  				t.Run("", func(t *T) { t.Fatal("fail") })
   234  			})
   235  		},
   236  	}, {
   237  		desc: "skipping without message, not chatty",
   238  		ok:   true,
   239  		f:    func(t *T) { t.SkipNow() },
   240  	}, {
   241  		desc: "skipping after error",
   242  		output: `
   243  --- FAIL: skipping after error (N.NNs)
   244      sub_test.go:NNN: an error
   245      sub_test.go:NNN: skipped`,
   246  		f: func(t *T) {
   247  			t.Error("an error")
   248  			t.Skip("skipped")
   249  		},
   250  	}, {
   251  		desc:   "use Run to locally synchronize parallelism",
   252  		ok:     true,
   253  		maxPar: 1,
   254  		f: func(t *T) {
   255  			var count uint32
   256  			t.Run("waitGroup", func(t *T) {
   257  				for i := 0; i < 4; i++ {
   258  					t.Run("par", func(t *T) {
   259  						t.Parallel()
   260  						atomic.AddUint32(&count, 1)
   261  					})
   262  				}
   263  			})
   264  			if count != 4 {
   265  				t.Errorf("count was %d; want 4", count)
   266  			}
   267  		},
   268  	}, {
   269  		desc: "alternate sequential and parallel",
   270  		// Sequential tests should partake in the counting of running threads.
   271  		// Otherwise, if one runs parallel subtests in sequential tests that are
   272  		// itself subtests of parallel tests, the counts can get askew.
   273  		ok:     true,
   274  		maxPar: 1,
   275  		f: func(t *T) {
   276  			t.Run("a", func(t *T) {
   277  				t.Parallel()
   278  				t.Run("b", func(t *T) {
   279  					// Sequential: ensure running count is decremented.
   280  					t.Run("c", func(t *T) {
   281  						t.Parallel()
   282  					})
   283  
   284  				})
   285  			})
   286  		},
   287  	}, {
   288  		desc: "alternate sequential and parallel 2",
   289  		// Sequential tests should partake in the counting of running threads.
   290  		// Otherwise, if one runs parallel subtests in sequential tests that are
   291  		// itself subtests of parallel tests, the counts can get askew.
   292  		ok:     true,
   293  		maxPar: 2,
   294  		f: func(t *T) {
   295  			for i := 0; i < 2; i++ {
   296  				t.Run("a", func(t *T) {
   297  					t.Parallel()
   298  					time.Sleep(time.Nanosecond)
   299  					for i := 0; i < 2; i++ {
   300  						t.Run("b", func(t *T) {
   301  							time.Sleep(time.Nanosecond)
   302  							for i := 0; i < 2; i++ {
   303  								t.Run("c", func(t *T) {
   304  									t.Parallel()
   305  									time.Sleep(time.Nanosecond)
   306  								})
   307  							}
   308  
   309  						})
   310  					}
   311  				})
   312  			}
   313  		},
   314  	}, {
   315  		desc:   "stress test",
   316  		ok:     true,
   317  		maxPar: 4,
   318  		f: func(t *T) {
   319  			t.Parallel()
   320  			for i := 0; i < 12; i++ {
   321  				t.Run("a", func(t *T) {
   322  					t.Parallel()
   323  					time.Sleep(time.Nanosecond)
   324  					for i := 0; i < 12; i++ {
   325  						t.Run("b", func(t *T) {
   326  							time.Sleep(time.Nanosecond)
   327  							for i := 0; i < 12; i++ {
   328  								t.Run("c", func(t *T) {
   329  									t.Parallel()
   330  									time.Sleep(time.Nanosecond)
   331  									t.Run("d1", func(t *T) {})
   332  									t.Run("d2", func(t *T) {})
   333  									t.Run("d3", func(t *T) {})
   334  									t.Run("d4", func(t *T) {})
   335  								})
   336  							}
   337  						})
   338  					}
   339  				})
   340  			}
   341  		},
   342  	}, {
   343  		desc:   "skip output",
   344  		ok:     true,
   345  		maxPar: 4,
   346  		f: func(t *T) {
   347  			t.Skip()
   348  		},
   349  	}, {
   350  		desc: "subtest calls error on parent",
   351  		ok:   false,
   352  		output: `
   353  --- FAIL: subtest calls error on parent (N.NNs)
   354      sub_test.go:NNN: first this
   355      sub_test.go:NNN: and now this!
   356      sub_test.go:NNN: oh, and this too`,
   357  		maxPar: 1,
   358  		f: func(t *T) {
   359  			t.Errorf("first this")
   360  			outer := t
   361  			t.Run("", func(t *T) {
   362  				outer.Errorf("and now this!")
   363  			})
   364  			t.Errorf("oh, and this too")
   365  		},
   366  	}, {
   367  		desc: "subtest calls fatal on parent",
   368  		ok:   false,
   369  		output: `
   370  --- FAIL: subtest calls fatal on parent (N.NNs)
   371      sub_test.go:NNN: first this
   372      sub_test.go:NNN: and now this!
   373      --- FAIL: subtest calls fatal on parent/#00 (N.NNs)
   374          testing.go:NNN: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test`,
   375  		maxPar: 1,
   376  		f: func(t *T) {
   377  			outer := t
   378  			t.Errorf("first this")
   379  			t.Run("", func(t *T) {
   380  				outer.Fatalf("and now this!")
   381  			})
   382  			t.Errorf("Should not reach here.")
   383  		},
   384  	}, {
   385  		desc: "subtest calls error on ancestor",
   386  		ok:   false,
   387  		output: `
   388  --- FAIL: subtest calls error on ancestor (N.NNs)
   389      sub_test.go:NNN: Report to ancestor
   390      --- FAIL: subtest calls error on ancestor/#00 (N.NNs)
   391          sub_test.go:NNN: Still do this
   392      sub_test.go:NNN: Also do this`,
   393  		maxPar: 1,
   394  		f: func(t *T) {
   395  			outer := t
   396  			t.Run("", func(t *T) {
   397  				t.Run("", func(t *T) {
   398  					outer.Errorf("Report to ancestor")
   399  				})
   400  				t.Errorf("Still do this")
   401  			})
   402  			t.Errorf("Also do this")
   403  		},
   404  	}, {
   405  		desc: "subtest calls fatal on ancestor",
   406  		ok:   false,
   407  		output: `
   408  --- FAIL: subtest calls fatal on ancestor (N.NNs)
   409      sub_test.go:NNN: Nope`,
   410  		maxPar: 1,
   411  		f: func(t *T) {
   412  			outer := t
   413  			t.Run("", func(t *T) {
   414  				for i := 0; i < 4; i++ {
   415  					t.Run("", func(t *T) {
   416  						outer.Fatalf("Nope")
   417  					})
   418  					t.Errorf("Don't do this")
   419  				}
   420  				t.Errorf("And neither do this")
   421  			})
   422  			t.Errorf("Nor this")
   423  		},
   424  	}, {
   425  		desc:   "panic on goroutine fail after test exit",
   426  		ok:     false,
   427  		maxPar: 4,
   428  		f: func(t *T) {
   429  			ch := make(chan bool)
   430  			t.Run("", func(t *T) {
   431  				go func() {
   432  					<-ch
   433  					defer func() {
   434  						if r := recover(); r == nil {
   435  							realTest.Errorf("expected panic")
   436  						}
   437  						ch <- true
   438  					}()
   439  					t.Errorf("failed after success")
   440  				}()
   441  			})
   442  			ch <- true
   443  			<-ch
   444  		},
   445  	}, {
   446  		desc: "log in finished sub test logs to parent",
   447  		ok:   false,
   448  		output: `
   449  		--- FAIL: log in finished sub test logs to parent (N.NNs)
   450      sub_test.go:NNN: message2
   451      sub_test.go:NNN: message1
   452      sub_test.go:NNN: error`,
   453  		maxPar: 1,
   454  		f: func(t *T) {
   455  			ch := make(chan bool)
   456  			t.Run("sub", func(t2 *T) {
   457  				go func() {
   458  					<-ch
   459  					t2.Log("message1")
   460  					ch <- true
   461  				}()
   462  			})
   463  			t.Log("message2")
   464  			ch <- true
   465  			<-ch
   466  			t.Errorf("error")
   467  		},
   468  	}, {
   469  		// A chatty test should always log with fmt.Print, even if the
   470  		// parent test has completed.
   471  		desc:   "log in finished sub test with chatty",
   472  		ok:     false,
   473  		chatty: true,
   474  		output: `
   475  		--- FAIL: log in finished sub test with chatty (N.NNs)`,
   476  		maxPar: 1,
   477  		f: func(t *T) {
   478  			ch := make(chan bool)
   479  			t.Run("sub", func(t2 *T) {
   480  				go func() {
   481  					<-ch
   482  					t2.Log("message1")
   483  					ch <- true
   484  				}()
   485  			})
   486  			t.Log("message2")
   487  			ch <- true
   488  			<-ch
   489  			t.Errorf("error")
   490  		},
   491  	}, {
   492  		// If a subtest panics we should run cleanups.
   493  		desc:   "cleanup when subtest panics",
   494  		ok:     false,
   495  		chatty: false,
   496  		output: `
   497  --- FAIL: cleanup when subtest panics (N.NNs)
   498      --- FAIL: cleanup when subtest panics/sub (N.NNs)
   499      sub_test.go:NNN: running cleanup`,
   500  		f: func(t *T) {
   501  			t.Cleanup(func() { t.Log("running cleanup") })
   502  			t.Run("sub", func(t2 *T) {
   503  				t2.FailNow()
   504  			})
   505  		},
   506  	}}
   507  	for _, tc := range testCases {
   508  		t.Run(tc.desc, func(t *T) {
   509  			tstate := newTestState(tc.maxPar, allMatcher())
   510  			buf := &strings.Builder{}
   511  			root := &T{
   512  				common: common{
   513  					signal:  make(chan bool),
   514  					barrier: make(chan bool),
   515  					name:    "",
   516  					w:       buf,
   517  				},
   518  				tstate: tstate,
   519  			}
   520  			if tc.chatty {
   521  				root.chatty = newChattyPrinter(root.w)
   522  				root.chatty.json = tc.json
   523  			}
   524  			ok := root.Run(tc.desc, tc.f)
   525  			tstate.release()
   526  
   527  			if ok != tc.ok {
   528  				t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, tc.ok)
   529  			}
   530  			if ok != !root.Failed() {
   531  				t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed())
   532  			}
   533  			if tstate.running != 0 || tstate.numWaiting != 0 {
   534  				t.Errorf("%s:running and waiting non-zero: got %d and %d", tc.desc, tstate.running, tstate.numWaiting)
   535  			}
   536  			got := strings.TrimSpace(buf.String())
   537  			want := strings.TrimSpace(tc.output)
   538  			re := makeRegexp(want)
   539  			if ok, err := regexp.MatchString(re, got); !ok || err != nil {
   540  				t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, got, want)
   541  			}
   542  		})
   543  	}
   544  }
   545  
   546  func TestBRun(t *T) {
   547  	work := func(b *B) {
   548  		for i := 0; i < b.N; i++ {
   549  			time.Sleep(time.Nanosecond)
   550  		}
   551  	}
   552  	testCases := []struct {
   553  		desc   string
   554  		failed bool
   555  		chatty bool
   556  		output string
   557  		f      func(*B)
   558  	}{{
   559  		desc: "simulate sequential run of subbenchmarks.",
   560  		f: func(b *B) {
   561  			b.Run("", func(b *B) { work(b) })
   562  			time1 := b.result.NsPerOp()
   563  			b.Run("", func(b *B) { work(b) })
   564  			time2 := b.result.NsPerOp()
   565  			if time1 >= time2 {
   566  				t.Errorf("no time spent in benchmark t1 >= t2 (%d >= %d)", time1, time2)
   567  			}
   568  		},
   569  	}, {
   570  		desc: "bytes set by all benchmarks",
   571  		f: func(b *B) {
   572  			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
   573  			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
   574  			if b.result.Bytes != 20 {
   575  				t.Errorf("bytes: got: %d; want 20", b.result.Bytes)
   576  			}
   577  		},
   578  	}, {
   579  		desc: "bytes set by some benchmarks",
   580  		// In this case the bytes result is meaningless, so it must be 0.
   581  		f: func(b *B) {
   582  			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
   583  			b.Run("", func(b *B) { work(b) })
   584  			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
   585  			if b.result.Bytes != 0 {
   586  				t.Errorf("bytes: got: %d; want 0", b.result.Bytes)
   587  			}
   588  		},
   589  	}, {
   590  		desc:   "failure carried over to root",
   591  		failed: true,
   592  		output: "--- FAIL: root",
   593  		f:      func(b *B) { b.Fail() },
   594  	}, {
   595  		desc:   "skipping without message, chatty",
   596  		chatty: true,
   597  		output: "--- SKIP: root",
   598  		f:      func(b *B) { b.SkipNow() },
   599  	}, {
   600  		desc:   "chatty with recursion",
   601  		chatty: true,
   602  		f: func(b *B) {
   603  			b.Run("", func(b *B) {
   604  				b.Run("", func(b *B) {})
   605  			})
   606  		},
   607  	}, {
   608  		desc: "skipping without message, not chatty",
   609  		f:    func(b *B) { b.SkipNow() },
   610  	}, {
   611  		desc:   "skipping after error",
   612  		failed: true,
   613  		output: `
   614  --- FAIL: root
   615      sub_test.go:NNN: an error
   616      sub_test.go:NNN: skipped`,
   617  		f: func(b *B) {
   618  			b.Error("an error")
   619  			b.Skip("skipped")
   620  		},
   621  	}, {
   622  		desc: "memory allocation",
   623  		f: func(b *B) {
   624  			const bufSize = 256
   625  			alloc := func(b *B) {
   626  				var buf [bufSize]byte
   627  				for i := 0; i < b.N; i++ {
   628  					_ = append([]byte(nil), buf[:]...)
   629  				}
   630  			}
   631  			b.Run("", func(b *B) {
   632  				alloc(b)
   633  				b.ReportAllocs()
   634  			})
   635  			b.Run("", func(b *B) {
   636  				alloc(b)
   637  				b.ReportAllocs()
   638  			})
   639  			// runtime.MemStats sometimes reports more allocations than the
   640  			// benchmark is responsible for. Luckily the point of this test is
   641  			// to ensure that the results are not underreported, so we can
   642  			// simply verify the lower bound.
   643  			if got := b.result.MemAllocs; got < 2 {
   644  				t.Errorf("MemAllocs was %v; want 2", got)
   645  			}
   646  			if got := b.result.MemBytes; got < 2*bufSize {
   647  				t.Errorf("MemBytes was %v; want %v", got, 2*bufSize)
   648  			}
   649  		},
   650  	}, {
   651  		desc: "cleanup is called",
   652  		f: func(b *B) {
   653  			var calls, cleanups, innerCalls, innerCleanups int
   654  			b.Run("", func(b *B) {
   655  				calls++
   656  				b.Cleanup(func() {
   657  					cleanups++
   658  				})
   659  				b.Run("", func(b *B) {
   660  					b.Cleanup(func() {
   661  						innerCleanups++
   662  					})
   663  					innerCalls++
   664  				})
   665  				work(b)
   666  			})
   667  			if calls == 0 || calls != cleanups {
   668  				t.Errorf("mismatched cleanups; got %d want %d", cleanups, calls)
   669  			}
   670  			if innerCalls == 0 || innerCalls != innerCleanups {
   671  				t.Errorf("mismatched cleanups; got %d want %d", cleanups, calls)
   672  			}
   673  		},
   674  	}, {
   675  		desc:   "cleanup is called on failure",
   676  		failed: true,
   677  		f: func(b *B) {
   678  			var calls, cleanups int
   679  			b.Run("", func(b *B) {
   680  				calls++
   681  				b.Cleanup(func() {
   682  					cleanups++
   683  				})
   684  				b.Fatalf("failure")
   685  			})
   686  			if calls == 0 || calls != cleanups {
   687  				t.Errorf("mismatched cleanups; got %d want %d", cleanups, calls)
   688  			}
   689  		},
   690  	}}
   691  	hideStdoutForTesting = true
   692  	defer func() {
   693  		hideStdoutForTesting = false
   694  	}()
   695  	for _, tc := range testCases {
   696  		t.Run(tc.desc, func(t *T) {
   697  			var ok bool
   698  			buf := &strings.Builder{}
   699  			// This is almost like the Benchmark function, except that we override
   700  			// the benchtime and catch the failure result of the subbenchmark.
   701  			root := &B{
   702  				common: common{
   703  					signal: make(chan bool),
   704  					name:   "root",
   705  					w:      buf,
   706  				},
   707  				benchFunc: func(b *B) { ok = b.Run("test", tc.f) }, // Use Run to catch failure.
   708  				benchTime: durationOrCountFlag{d: 1 * time.Microsecond},
   709  			}
   710  			if tc.chatty {
   711  				root.chatty = newChattyPrinter(root.w)
   712  			}
   713  			root.runN(1)
   714  			if ok != !tc.failed {
   715  				t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, !tc.failed)
   716  			}
   717  			if !ok != root.Failed() {
   718  				t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed())
   719  			}
   720  			// All tests are run as subtests
   721  			if root.result.N != 1 {
   722  				t.Errorf("%s: N for parent benchmark was %d; want 1", tc.desc, root.result.N)
   723  			}
   724  			got := strings.TrimSpace(buf.String())
   725  			want := strings.TrimSpace(tc.output)
   726  			re := makeRegexp(want)
   727  			if ok, err := regexp.MatchString(re, got); !ok || err != nil {
   728  				t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, got, want)
   729  			}
   730  		})
   731  	}
   732  }
   733  
   734  func makeRegexp(s string) string {
   735  	s = regexp.QuoteMeta(s)
   736  	s = strings.ReplaceAll(s, "^V", "\x16")
   737  	s = strings.ReplaceAll(s, ":NNN:", `:\d\d\d\d?:`)
   738  	s = strings.ReplaceAll(s, "N\\.NNs", `\d*\.\d*s`)
   739  	return s
   740  }
   741  
   742  func TestBenchmarkOutput(t *T) {
   743  	// Ensure Benchmark initialized common.w by invoking it with an error and
   744  	// normal case.
   745  	Benchmark(func(b *B) { b.Error("do not print this output") })
   746  	Benchmark(func(b *B) {})
   747  }
   748  
   749  func TestBenchmarkStartsFrom1(t *T) {
   750  	var first = true
   751  	Benchmark(func(b *B) {
   752  		if first && b.N != 1 {
   753  			panic(fmt.Sprintf("Benchmark() first N=%v; want 1", b.N))
   754  		}
   755  		first = false
   756  	})
   757  }
   758  
   759  func TestBenchmarkReadMemStatsBeforeFirstRun(t *T) {
   760  	var first = true
   761  	Benchmark(func(b *B) {
   762  		if first && (b.startAllocs == 0 || b.startBytes == 0) {
   763  			panic("ReadMemStats not called before first run")
   764  		}
   765  		first = false
   766  	})
   767  }
   768  
   769  type funcWriter struct {
   770  	write func([]byte) (int, error)
   771  }
   772  
   773  func (fw *funcWriter) Write(b []byte) (int, error) {
   774  	return fw.write(b)
   775  }
   776  
   777  func TestRacyOutput(t *T) {
   778  	var runs int32  // The number of running Writes
   779  	var races int32 // Incremented for each race detected
   780  	raceDetector := func(b []byte) (int, error) {
   781  		// Check if some other goroutine is concurrently calling Write.
   782  		if atomic.LoadInt32(&runs) > 0 {
   783  			atomic.AddInt32(&races, 1) // Race detected!
   784  		}
   785  		atomic.AddInt32(&runs, 1)
   786  		defer atomic.AddInt32(&runs, -1)
   787  		runtime.Gosched() // Increase probability of a race
   788  		return len(b), nil
   789  	}
   790  
   791  	root := &T{
   792  		common: common{w: &funcWriter{raceDetector}},
   793  		tstate: newTestState(1, allMatcher()),
   794  	}
   795  	root.chatty = newChattyPrinter(root.w)
   796  	root.Run("", func(t *T) {
   797  		var wg sync.WaitGroup
   798  		for i := 0; i < 100; i++ {
   799  			wg.Add(1)
   800  			go func(i int) {
   801  				defer wg.Done()
   802  				t.Run(fmt.Sprint(i), func(t *T) {
   803  					t.Logf("testing run %d", i)
   804  				})
   805  			}(i)
   806  		}
   807  		wg.Wait()
   808  	})
   809  
   810  	if races > 0 {
   811  		t.Errorf("detected %d racy Writes", races)
   812  	}
   813  }
   814  
   815  // The late log message did not include the test name.  Issue 29388.
   816  func TestLogAfterComplete(t *T) {
   817  	tstate := newTestState(1, allMatcher())
   818  	var buf bytes.Buffer
   819  	t1 := &T{
   820  		common: common{
   821  			// Use a buffered channel so that tRunner can write
   822  			// to it although nothing is reading from it.
   823  			signal: make(chan bool, 1),
   824  			w:      &buf,
   825  		},
   826  		tstate: tstate,
   827  	}
   828  
   829  	c1 := make(chan bool)
   830  	c2 := make(chan string)
   831  	tRunner(t1, func(t *T) {
   832  		t.Run("TestLateLog", func(t *T) {
   833  			go func() {
   834  				defer close(c2)
   835  				defer func() {
   836  					p := recover()
   837  					if p == nil {
   838  						c2 <- "subtest did not panic"
   839  						return
   840  					}
   841  					s, ok := p.(string)
   842  					if !ok {
   843  						c2 <- fmt.Sprintf("subtest panic with unexpected value %v", p)
   844  						return
   845  					}
   846  					const want = "Log in goroutine after TestLateLog has completed: log after test"
   847  					if !strings.Contains(s, want) {
   848  						c2 <- fmt.Sprintf("subtest panic %q does not contain %q", s, want)
   849  					}
   850  				}()
   851  
   852  				<-c1
   853  				t.Log("log after test")
   854  			}()
   855  		})
   856  	})
   857  	close(c1)
   858  
   859  	if s := <-c2; s != "" {
   860  		t.Error(s)
   861  	}
   862  }
   863  
   864  func TestBenchmark(t *T) {
   865  	if Short() {
   866  		t.Skip("skipping in short mode")
   867  	}
   868  	res := Benchmark(func(b *B) {
   869  		for i := 0; i < 5; i++ {
   870  			b.Run("", func(b *B) {
   871  				for i := 0; i < b.N; i++ {
   872  					time.Sleep(time.Millisecond)
   873  				}
   874  			})
   875  		}
   876  	})
   877  	if res.NsPerOp() < 4000000 {
   878  		t.Errorf("want >5ms; got %v", time.Duration(res.NsPerOp()))
   879  	}
   880  }
   881  
   882  func TestCleanup(t *T) {
   883  	var cleanups []int
   884  	t.Run("test", func(t *T) {
   885  		t.Cleanup(func() { cleanups = append(cleanups, 1) })
   886  		t.Cleanup(func() { cleanups = append(cleanups, 2) })
   887  	})
   888  	if got, want := cleanups, []int{2, 1}; !slices.Equal(got, want) {
   889  		t.Errorf("unexpected cleanup record; got %v want %v", got, want)
   890  	}
   891  }
   892  
   893  func TestConcurrentCleanup(t *T) {
   894  	cleanups := 0
   895  	t.Run("test", func(t *T) {
   896  		var wg sync.WaitGroup
   897  		wg.Add(2)
   898  		for i := 0; i < 2; i++ {
   899  			i := i
   900  			go func() {
   901  				t.Cleanup(func() {
   902  					// Although the calls to Cleanup are concurrent, the functions passed
   903  					// to Cleanup should be called sequentially, in some nondeterministic
   904  					// order based on when the Cleanup calls happened to be scheduled.
   905  					// So these assignments to the cleanups variable should not race.
   906  					cleanups |= 1 << i
   907  				})
   908  				wg.Done()
   909  			}()
   910  		}
   911  		wg.Wait()
   912  	})
   913  	if cleanups != 1|2 {
   914  		t.Errorf("unexpected cleanup; got %d want 3", cleanups)
   915  	}
   916  }
   917  
   918  func TestCleanupCalledEvenAfterGoexit(t *T) {
   919  	cleanups := 0
   920  	t.Run("test", func(t *T) {
   921  		t.Cleanup(func() {
   922  			cleanups++
   923  		})
   924  		t.Cleanup(func() {
   925  			runtime.Goexit()
   926  		})
   927  	})
   928  	if cleanups != 1 {
   929  		t.Errorf("unexpected cleanup count; got %d want 1", cleanups)
   930  	}
   931  }
   932  
   933  func TestRunCleanup(t *T) {
   934  	outerCleanup := 0
   935  	innerCleanup := 0
   936  	t.Run("test", func(t *T) {
   937  		t.Cleanup(func() { outerCleanup++ })
   938  		t.Run("x", func(t *T) {
   939  			t.Cleanup(func() { innerCleanup++ })
   940  		})
   941  	})
   942  	if innerCleanup != 1 {
   943  		t.Errorf("unexpected inner cleanup count; got %d want 1", innerCleanup)
   944  	}
   945  	if outerCleanup != 1 {
   946  		t.Errorf("unexpected outer cleanup count; got %d want 0", outerCleanup)
   947  	}
   948  }
   949  
   950  func TestCleanupParallelSubtests(t *T) {
   951  	ranCleanup := 0
   952  	t.Run("test", func(t *T) {
   953  		t.Cleanup(func() { ranCleanup++ })
   954  		t.Run("x", func(t *T) {
   955  			t.Parallel()
   956  			if ranCleanup > 0 {
   957  				t.Error("outer cleanup ran before parallel subtest")
   958  			}
   959  		})
   960  	})
   961  	if ranCleanup != 1 {
   962  		t.Errorf("unexpected cleanup count; got %d want 1", ranCleanup)
   963  	}
   964  }
   965  
   966  func TestNestedCleanup(t *T) {
   967  	ranCleanup := 0
   968  	t.Run("test", func(t *T) {
   969  		t.Cleanup(func() {
   970  			if ranCleanup != 2 {
   971  				t.Errorf("unexpected cleanup count in first cleanup: got %d want 2", ranCleanup)
   972  			}
   973  			ranCleanup++
   974  		})
   975  		t.Cleanup(func() {
   976  			if ranCleanup != 0 {
   977  				t.Errorf("unexpected cleanup count in second cleanup: got %d want 0", ranCleanup)
   978  			}
   979  			ranCleanup++
   980  			t.Cleanup(func() {
   981  				if ranCleanup != 1 {
   982  					t.Errorf("unexpected cleanup count in nested cleanup: got %d want 1", ranCleanup)
   983  				}
   984  				ranCleanup++
   985  			})
   986  		})
   987  	})
   988  	if ranCleanup != 3 {
   989  		t.Errorf("unexpected cleanup count: got %d want 3", ranCleanup)
   990  	}
   991  }
   992  

View as plain text