...
Run Format

Source file src/testing/sub_test.go

Documentation: testing

     1  // Copyright 2016 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package testing
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"regexp"
    11  	"runtime"
    12  	"strings"
    13  	"sync"
    14  	"sync/atomic"
    15  	"time"
    16  )
    17  
    18  func init() {
    19  	// Make benchmark tests run 10* faster.
    20  	*benchTime = 100 * time.Millisecond
    21  }
    22  
    23  func TestTestContext(t *T) {
    24  	const (
    25  		add1 = 0
    26  		done = 1
    27  	)
    28  	// After each of the calls are applied to the context, the
    29  	type call struct {
    30  		typ int // run or done
    31  		// result from applying the call
    32  		running int
    33  		waiting int
    34  		started bool
    35  	}
    36  	testCases := []struct {
    37  		max int
    38  		run []call
    39  	}{{
    40  		max: 1,
    41  		run: []call{
    42  			{typ: add1, running: 1, waiting: 0, started: true},
    43  			{typ: done, running: 0, waiting: 0, started: false},
    44  		},
    45  	}, {
    46  		max: 1,
    47  		run: []call{
    48  			{typ: add1, running: 1, waiting: 0, started: true},
    49  			{typ: add1, running: 1, waiting: 1, started: false},
    50  			{typ: done, running: 1, waiting: 0, started: true},
    51  			{typ: done, running: 0, waiting: 0, started: false},
    52  			{typ: add1, running: 1, waiting: 0, started: true},
    53  		},
    54  	}, {
    55  		max: 3,
    56  		run: []call{
    57  			{typ: add1, running: 1, waiting: 0, started: true},
    58  			{typ: add1, running: 2, waiting: 0, started: true},
    59  			{typ: add1, running: 3, waiting: 0, started: true},
    60  			{typ: add1, running: 3, waiting: 1, started: false},
    61  			{typ: add1, running: 3, waiting: 2, started: false},
    62  			{typ: add1, running: 3, waiting: 3, started: false},
    63  			{typ: done, running: 3, waiting: 2, started: true},
    64  			{typ: add1, running: 3, waiting: 3, started: false},
    65  			{typ: done, running: 3, waiting: 2, started: true},
    66  			{typ: done, running: 3, waiting: 1, started: true},
    67  			{typ: done, running: 3, waiting: 0, started: true},
    68  			{typ: done, running: 2, waiting: 0, started: false},
    69  			{typ: done, running: 1, waiting: 0, started: false},
    70  			{typ: done, running: 0, waiting: 0, started: false},
    71  		},
    72  	}}
    73  	for i, tc := range testCases {
    74  		ctx := &testContext{
    75  			startParallel: make(chan bool),
    76  			maxParallel:   tc.max,
    77  		}
    78  		for j, call := range tc.run {
    79  			doCall := func(f func()) chan bool {
    80  				done := make(chan bool)
    81  				go func() {
    82  					f()
    83  					done <- true
    84  				}()
    85  				return done
    86  			}
    87  			started := false
    88  			switch call.typ {
    89  			case add1:
    90  				signal := doCall(ctx.waitParallel)
    91  				select {
    92  				case <-signal:
    93  					started = true
    94  				case ctx.startParallel <- true:
    95  					<-signal
    96  				}
    97  			case done:
    98  				signal := doCall(ctx.release)
    99  				select {
   100  				case <-signal:
   101  				case <-ctx.startParallel:
   102  					started = true
   103  					<-signal
   104  				}
   105  			}
   106  			if started != call.started {
   107  				t.Errorf("%d:%d:started: got %v; want %v", i, j, started, call.started)
   108  			}
   109  			if ctx.running != call.running {
   110  				t.Errorf("%d:%d:running: got %v; want %v", i, j, ctx.running, call.running)
   111  			}
   112  			if ctx.numWaiting != call.waiting {
   113  				t.Errorf("%d:%d:waiting: got %v; want %v", i, j, ctx.numWaiting, call.waiting)
   114  			}
   115  		}
   116  	}
   117  }
   118  
   119  func TestTRun(t *T) {
   120  	realTest := t
   121  	testCases := []struct {
   122  		desc   string
   123  		ok     bool
   124  		maxPar int
   125  		chatty bool
   126  		output string
   127  		f      func(*T)
   128  	}{{
   129  		desc:   "failnow skips future sequential and parallel tests at same level",
   130  		ok:     false,
   131  		maxPar: 1,
   132  		output: `
   133  --- FAIL: failnow skips future sequential and parallel tests at same level (N.NNs)
   134      --- FAIL: failnow skips future sequential and parallel tests at same level/#00 (N.NNs)
   135      `,
   136  		f: func(t *T) {
   137  			ranSeq := false
   138  			ranPar := false
   139  			t.Run("", func(t *T) {
   140  				t.Run("par", func(t *T) {
   141  					t.Parallel()
   142  					ranPar = true
   143  				})
   144  				t.Run("seq", func(t *T) {
   145  					ranSeq = true
   146  				})
   147  				t.FailNow()
   148  				t.Run("seq", func(t *T) {
   149  					realTest.Error("test must be skipped")
   150  				})
   151  				t.Run("par", func(t *T) {
   152  					t.Parallel()
   153  					realTest.Error("test must be skipped.")
   154  				})
   155  			})
   156  			if !ranPar {
   157  				realTest.Error("parallel test was not run")
   158  			}
   159  			if !ranSeq {
   160  				realTest.Error("sequential test was not run")
   161  			}
   162  		},
   163  	}, {
   164  		desc:   "failure in parallel test propagates upwards",
   165  		ok:     false,
   166  		maxPar: 1,
   167  		output: `
   168  --- FAIL: failure in parallel test propagates upwards (N.NNs)
   169      --- FAIL: failure in parallel test propagates upwards/#00 (N.NNs)
   170          --- FAIL: failure in parallel test propagates upwards/#00/par (N.NNs)
   171  		`,
   172  		f: func(t *T) {
   173  			t.Run("", func(t *T) {
   174  				t.Parallel()
   175  				t.Run("par", func(t *T) {
   176  					t.Parallel()
   177  					t.Fail()
   178  				})
   179  			})
   180  		},
   181  	}, {
   182  		desc:   "skipping without message, chatty",
   183  		ok:     true,
   184  		chatty: true,
   185  		output: `
   186  === RUN   skipping without message, chatty
   187  --- SKIP: skipping without message, chatty (N.NNs)`,
   188  		f: func(t *T) { t.SkipNow() },
   189  	}, {
   190  		desc:   "chatty with recursion",
   191  		ok:     true,
   192  		chatty: true,
   193  		output: `
   194  === RUN   chatty with recursion
   195  === RUN   chatty with recursion/#00
   196  === RUN   chatty with recursion/#00/#00
   197  --- PASS: chatty with recursion (N.NNs)
   198      --- PASS: chatty with recursion/#00 (N.NNs)
   199          --- PASS: chatty with recursion/#00/#00 (N.NNs)`,
   200  		f: func(t *T) {
   201  			t.Run("", func(t *T) {
   202  				t.Run("", func(t *T) {})
   203  			})
   204  		},
   205  	}, {
   206  		desc: "skipping without message, not chatty",
   207  		ok:   true,
   208  		f:    func(t *T) { t.SkipNow() },
   209  	}, {
   210  		desc: "skipping after error",
   211  		output: `
   212  --- FAIL: skipping after error (N.NNs)
   213  	sub_test.go:NNN: an error
   214  	sub_test.go:NNN: skipped`,
   215  		f: func(t *T) {
   216  			t.Error("an error")
   217  			t.Skip("skipped")
   218  		},
   219  	}, {
   220  		desc:   "use Run to locally synchronize parallelism",
   221  		ok:     true,
   222  		maxPar: 1,
   223  		f: func(t *T) {
   224  			var count uint32
   225  			t.Run("waitGroup", func(t *T) {
   226  				for i := 0; i < 4; i++ {
   227  					t.Run("par", func(t *T) {
   228  						t.Parallel()
   229  						atomic.AddUint32(&count, 1)
   230  					})
   231  				}
   232  			})
   233  			if count != 4 {
   234  				t.Errorf("count was %d; want 4", count)
   235  			}
   236  		},
   237  	}, {
   238  		desc: "alternate sequential and parallel",
   239  		// Sequential tests should partake in the counting of running threads.
   240  		// Otherwise, if one runs parallel subtests in sequential tests that are
   241  		// itself subtests of parallel tests, the counts can get askew.
   242  		ok:     true,
   243  		maxPar: 1,
   244  		f: func(t *T) {
   245  			t.Run("a", func(t *T) {
   246  				t.Parallel()
   247  				t.Run("b", func(t *T) {
   248  					// Sequential: ensure running count is decremented.
   249  					t.Run("c", func(t *T) {
   250  						t.Parallel()
   251  					})
   252  
   253  				})
   254  			})
   255  		},
   256  	}, {
   257  		desc: "alternate sequential and parallel 2",
   258  		// Sequential tests should partake in the counting of running threads.
   259  		// Otherwise, if one runs parallel subtests in sequential tests that are
   260  		// itself subtests of parallel tests, the counts can get askew.
   261  		ok:     true,
   262  		maxPar: 2,
   263  		f: func(t *T) {
   264  			for i := 0; i < 2; i++ {
   265  				t.Run("a", func(t *T) {
   266  					t.Parallel()
   267  					time.Sleep(time.Nanosecond)
   268  					for i := 0; i < 2; i++ {
   269  						t.Run("b", func(t *T) {
   270  							time.Sleep(time.Nanosecond)
   271  							for i := 0; i < 2; i++ {
   272  								t.Run("c", func(t *T) {
   273  									t.Parallel()
   274  									time.Sleep(time.Nanosecond)
   275  								})
   276  							}
   277  
   278  						})
   279  					}
   280  				})
   281  			}
   282  		},
   283  	}, {
   284  		desc:   "stress test",
   285  		ok:     true,
   286  		maxPar: 4,
   287  		f: func(t *T) {
   288  			t.Parallel()
   289  			for i := 0; i < 12; i++ {
   290  				t.Run("a", func(t *T) {
   291  					t.Parallel()
   292  					time.Sleep(time.Nanosecond)
   293  					for i := 0; i < 12; i++ {
   294  						t.Run("b", func(t *T) {
   295  							time.Sleep(time.Nanosecond)
   296  							for i := 0; i < 12; i++ {
   297  								t.Run("c", func(t *T) {
   298  									t.Parallel()
   299  									time.Sleep(time.Nanosecond)
   300  									t.Run("d1", func(t *T) {})
   301  									t.Run("d2", func(t *T) {})
   302  									t.Run("d3", func(t *T) {})
   303  									t.Run("d4", func(t *T) {})
   304  								})
   305  							}
   306  						})
   307  					}
   308  				})
   309  			}
   310  		},
   311  	}, {
   312  		desc:   "skip output",
   313  		ok:     true,
   314  		maxPar: 4,
   315  		f: func(t *T) {
   316  			t.Skip()
   317  		},
   318  	}, {
   319  		desc:   "panic on goroutine fail after test exit",
   320  		ok:     false,
   321  		maxPar: 4,
   322  		f: func(t *T) {
   323  			ch := make(chan bool)
   324  			t.Run("", func(t *T) {
   325  				go func() {
   326  					<-ch
   327  					defer func() {
   328  						if r := recover(); r == nil {
   329  							realTest.Errorf("expected panic")
   330  						}
   331  						ch <- true
   332  					}()
   333  					t.Errorf("failed after success")
   334  				}()
   335  			})
   336  			ch <- true
   337  			<-ch
   338  		},
   339  	}}
   340  	for _, tc := range testCases {
   341  		ctx := newTestContext(tc.maxPar, newMatcher(regexp.MatchString, "", ""))
   342  		buf := &bytes.Buffer{}
   343  		root := &T{
   344  			common: common{
   345  				signal: make(chan bool),
   346  				name:   "Test",
   347  				w:      buf,
   348  				chatty: tc.chatty,
   349  			},
   350  			context: ctx,
   351  		}
   352  		ok := root.Run(tc.desc, tc.f)
   353  		ctx.release()
   354  
   355  		if ok != tc.ok {
   356  			t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, tc.ok)
   357  		}
   358  		if ok != !root.Failed() {
   359  			t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed())
   360  		}
   361  		if ctx.running != 0 || ctx.numWaiting != 0 {
   362  			t.Errorf("%s:running and waiting non-zero: got %d and %d", tc.desc, ctx.running, ctx.numWaiting)
   363  		}
   364  		got := strings.TrimSpace(buf.String())
   365  		want := strings.TrimSpace(tc.output)
   366  		re := makeRegexp(want)
   367  		if ok, err := regexp.MatchString(re, got); !ok || err != nil {
   368  			t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, got, want)
   369  		}
   370  	}
   371  }
   372  
   373  func TestBRun(t *T) {
   374  	work := func(b *B) {
   375  		for i := 0; i < b.N; i++ {
   376  			time.Sleep(time.Nanosecond)
   377  		}
   378  	}
   379  	testCases := []struct {
   380  		desc   string
   381  		failed bool
   382  		chatty bool
   383  		output string
   384  		f      func(*B)
   385  	}{{
   386  		desc: "simulate sequential run of subbenchmarks.",
   387  		f: func(b *B) {
   388  			b.Run("", func(b *B) { work(b) })
   389  			time1 := b.result.NsPerOp()
   390  			b.Run("", func(b *B) { work(b) })
   391  			time2 := b.result.NsPerOp()
   392  			if time1 >= time2 {
   393  				t.Errorf("no time spent in benchmark t1 >= t2 (%d >= %d)", time1, time2)
   394  			}
   395  		},
   396  	}, {
   397  		desc: "bytes set by all benchmarks",
   398  		f: func(b *B) {
   399  			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
   400  			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
   401  			if b.result.Bytes != 20 {
   402  				t.Errorf("bytes: got: %d; want 20", b.result.Bytes)
   403  			}
   404  		},
   405  	}, {
   406  		desc: "bytes set by some benchmarks",
   407  		// In this case the bytes result is meaningless, so it must be 0.
   408  		f: func(b *B) {
   409  			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
   410  			b.Run("", func(b *B) { work(b) })
   411  			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
   412  			if b.result.Bytes != 0 {
   413  				t.Errorf("bytes: got: %d; want 0", b.result.Bytes)
   414  			}
   415  		},
   416  	}, {
   417  		desc:   "failure carried over to root",
   418  		failed: true,
   419  		output: "--- FAIL: root",
   420  		f:      func(b *B) { b.Fail() },
   421  	}, {
   422  		desc:   "skipping without message, chatty",
   423  		chatty: true,
   424  		output: "--- SKIP: root",
   425  		f:      func(b *B) { b.SkipNow() },
   426  	}, {
   427  		desc:   "skipping with message, chatty",
   428  		chatty: true,
   429  		output: `
   430  --- SKIP: root
   431  	sub_test.go:NNN: skipping`,
   432  		f: func(b *B) { b.Skip("skipping") },
   433  	}, {
   434  		desc:   "chatty with recursion",
   435  		chatty: true,
   436  		f: func(b *B) {
   437  			b.Run("", func(b *B) {
   438  				b.Run("", func(b *B) {})
   439  			})
   440  		},
   441  	}, {
   442  		desc: "skipping without message, not chatty",
   443  		f:    func(b *B) { b.SkipNow() },
   444  	}, {
   445  		desc:   "skipping after error",
   446  		failed: true,
   447  		output: `
   448  --- FAIL: root
   449  	sub_test.go:NNN: an error
   450  	sub_test.go:NNN: skipped`,
   451  		f: func(b *B) {
   452  			b.Error("an error")
   453  			b.Skip("skipped")
   454  		},
   455  	}, {
   456  		desc: "memory allocation",
   457  		f: func(b *B) {
   458  			const bufSize = 256
   459  			alloc := func(b *B) {
   460  				var buf [bufSize]byte
   461  				for i := 0; i < b.N; i++ {
   462  					_ = append([]byte(nil), buf[:]...)
   463  				}
   464  			}
   465  			b.Run("", func(b *B) {
   466  				alloc(b)
   467  				b.ReportAllocs()
   468  			})
   469  			b.Run("", func(b *B) {
   470  				alloc(b)
   471  				b.ReportAllocs()
   472  			})
   473  			// runtime.MemStats sometimes reports more allocations than the
   474  			// benchmark is responsible for. Luckily the point of this test is
   475  			// to ensure that the results are not underreported, so we can
   476  			// simply verify the lower bound.
   477  			if got := b.result.MemAllocs; got < 2 {
   478  				t.Errorf("MemAllocs was %v; want 2", got)
   479  			}
   480  			if got := b.result.MemBytes; got < 2*bufSize {
   481  				t.Errorf("MemBytes was %v; want %v", got, 2*bufSize)
   482  			}
   483  		},
   484  	}}
   485  	for _, tc := range testCases {
   486  		var ok bool
   487  		buf := &bytes.Buffer{}
   488  		// This is almost like the Benchmark function, except that we override
   489  		// the benchtime and catch the failure result of the subbenchmark.
   490  		root := &B{
   491  			common: common{
   492  				signal: make(chan bool),
   493  				name:   "root",
   494  				w:      buf,
   495  				chatty: tc.chatty,
   496  			},
   497  			benchFunc: func(b *B) { ok = b.Run("test", tc.f) }, // Use Run to catch failure.
   498  			benchTime: time.Microsecond,
   499  		}
   500  		root.runN(1)
   501  		if ok != !tc.failed {
   502  			t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, !tc.failed)
   503  		}
   504  		if !ok != root.Failed() {
   505  			t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed())
   506  		}
   507  		// All tests are run as subtests
   508  		if root.result.N != 1 {
   509  			t.Errorf("%s: N for parent benchmark was %d; want 1", tc.desc, root.result.N)
   510  		}
   511  		got := strings.TrimSpace(buf.String())
   512  		want := strings.TrimSpace(tc.output)
   513  		re := makeRegexp(want)
   514  		if ok, err := regexp.MatchString(re, got); !ok || err != nil {
   515  			t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, got, want)
   516  		}
   517  	}
   518  }
   519  
   520  func makeRegexp(s string) string {
   521  	s = strings.Replace(s, ":NNN:", `:\d\d\d:`, -1)
   522  	s = strings.Replace(s, "(N.NNs)", `\(\d*\.\d*s\)`, -1)
   523  	return s
   524  }
   525  
   526  func TestBenchmarkOutput(t *T) {
   527  	// Ensure Benchmark initialized common.w by invoking it with an error and
   528  	// normal case.
   529  	Benchmark(func(b *B) { b.Error("do not print this output") })
   530  	Benchmark(func(b *B) {})
   531  }
   532  
   533  func TestBenchmarkStartsFrom1(t *T) {
   534  	var first = true
   535  	Benchmark(func(b *B) {
   536  		if first && b.N != 1 {
   537  			panic(fmt.Sprintf("Benchmark() first N=%v; want 1", b.N))
   538  		}
   539  		first = false
   540  	})
   541  }
   542  
   543  func TestBenchmarkReadMemStatsBeforeFirstRun(t *T) {
   544  	var first = true
   545  	Benchmark(func(b *B) {
   546  		if first && (b.startAllocs == 0 || b.startBytes == 0) {
   547  			panic(fmt.Sprintf("ReadMemStats not called before first run"))
   548  		}
   549  		first = false
   550  	})
   551  }
   552  
   553  func TestParallelSub(t *T) {
   554  	c := make(chan int)
   555  	block := make(chan int)
   556  	for i := 0; i < 10; i++ {
   557  		go func(i int) {
   558  			<-block
   559  			t.Run(fmt.Sprint(i), func(t *T) {})
   560  			c <- 1
   561  		}(i)
   562  	}
   563  	close(block)
   564  	for i := 0; i < 10; i++ {
   565  		<-c
   566  	}
   567  }
   568  
   569  type funcWriter func([]byte) (int, error)
   570  
   571  func (fw funcWriter) Write(b []byte) (int, error) { return fw(b) }
   572  
   573  func TestRacyOutput(t *T) {
   574  	var runs int32  // The number of running Writes
   575  	var races int32 // Incremented for each race detected
   576  	raceDetector := func(b []byte) (int, error) {
   577  		// Check if some other goroutine is concurrently calling Write.
   578  		if atomic.LoadInt32(&runs) > 0 {
   579  			atomic.AddInt32(&races, 1) // Race detected!
   580  		}
   581  		atomic.AddInt32(&runs, 1)
   582  		defer atomic.AddInt32(&runs, -1)
   583  		runtime.Gosched() // Increase probability of a race
   584  		return len(b), nil
   585  	}
   586  
   587  	var wg sync.WaitGroup
   588  	root := &T{
   589  		common:  common{w: funcWriter(raceDetector), chatty: true},
   590  		context: newTestContext(1, newMatcher(regexp.MatchString, "", "")),
   591  	}
   592  	root.Run("", func(t *T) {
   593  		for i := 0; i < 100; i++ {
   594  			wg.Add(1)
   595  			go func(i int) {
   596  				defer wg.Done()
   597  				t.Run(fmt.Sprint(i), func(t *T) {
   598  					t.Logf("testing run %d", i)
   599  				})
   600  			}(i)
   601  		}
   602  	})
   603  	wg.Wait()
   604  
   605  	if races > 0 {
   606  		t.Errorf("detected %d racy Writes", races)
   607  	}
   608  }
   609  
   610  func TestBenchmark(t *T) {
   611  	res := Benchmark(func(b *B) {
   612  		for i := 0; i < 5; i++ {
   613  			b.Run("", func(b *B) {
   614  				for i := 0; i < b.N; i++ {
   615  					time.Sleep(time.Millisecond)
   616  				}
   617  			})
   618  		}
   619  	})
   620  	if res.NsPerOp() < 4000000 {
   621  		t.Errorf("want >5ms; got %v", time.Duration(res.NsPerOp()))
   622  	}
   623  }
   624  

View as plain text