...
Run Format

Source file src/testing/sub_test.go

Documentation: testing

  // Copyright 2016 The Go Authors. All rights reserved.
  // Use of this source code is governed by a BSD-style
  // license that can be found in the LICENSE file.
  
  package testing
  
  import (
  	"bytes"
  	"fmt"
  	"regexp"
  	"runtime"
  	"strings"
  	"sync"
  	"sync/atomic"
  	"time"
  )
  
  func init() {
  	// Make benchmark tests run 10* faster.
  	*benchTime = 100 * time.Millisecond
  }
  
  func TestTestContext(t *T) {
  	const (
  		add1 = 0
  		done = 1
  	)
  	// After each of the calls are applied to the context, the
  	type call struct {
  		typ int // run or done
  		// result from applying the call
  		running int
  		waiting int
  		started bool
  	}
  	testCases := []struct {
  		max int
  		run []call
  	}{{
  		max: 1,
  		run: []call{
  			{typ: add1, running: 1, waiting: 0, started: true},
  			{typ: done, running: 0, waiting: 0, started: false},
  		},
  	}, {
  		max: 1,
  		run: []call{
  			{typ: add1, running: 1, waiting: 0, started: true},
  			{typ: add1, running: 1, waiting: 1, started: false},
  			{typ: done, running: 1, waiting: 0, started: true},
  			{typ: done, running: 0, waiting: 0, started: false},
  			{typ: add1, running: 1, waiting: 0, started: true},
  		},
  	}, {
  		max: 3,
  		run: []call{
  			{typ: add1, running: 1, waiting: 0, started: true},
  			{typ: add1, running: 2, waiting: 0, started: true},
  			{typ: add1, running: 3, waiting: 0, started: true},
  			{typ: add1, running: 3, waiting: 1, started: false},
  			{typ: add1, running: 3, waiting: 2, started: false},
  			{typ: add1, running: 3, waiting: 3, started: false},
  			{typ: done, running: 3, waiting: 2, started: true},
  			{typ: add1, running: 3, waiting: 3, started: false},
  			{typ: done, running: 3, waiting: 2, started: true},
  			{typ: done, running: 3, waiting: 1, started: true},
  			{typ: done, running: 3, waiting: 0, started: true},
  			{typ: done, running: 2, waiting: 0, started: false},
  			{typ: done, running: 1, waiting: 0, started: false},
  			{typ: done, running: 0, waiting: 0, started: false},
  		},
  	}}
  	for i, tc := range testCases {
  		ctx := &testContext{
  			startParallel: make(chan bool),
  			maxParallel:   tc.max,
  		}
  		for j, call := range tc.run {
  			doCall := func(f func()) chan bool {
  				done := make(chan bool)
  				go func() {
  					f()
  					done <- true
  				}()
  				return done
  			}
  			started := false
  			switch call.typ {
  			case add1:
  				signal := doCall(ctx.waitParallel)
  				select {
  				case <-signal:
  					started = true
  				case ctx.startParallel <- true:
  					<-signal
  				}
  			case done:
  				signal := doCall(ctx.release)
  				select {
  				case <-signal:
  				case <-ctx.startParallel:
  					started = true
  					<-signal
  				}
  			}
  			if started != call.started {
  				t.Errorf("%d:%d:started: got %v; want %v", i, j, started, call.started)
  			}
  			if ctx.running != call.running {
  				t.Errorf("%d:%d:running: got %v; want %v", i, j, ctx.running, call.running)
  			}
  			if ctx.numWaiting != call.waiting {
  				t.Errorf("%d:%d:waiting: got %v; want %v", i, j, ctx.numWaiting, call.waiting)
  			}
  		}
  	}
  }
  
  func TestTRun(t *T) {
  	realTest := t
  	testCases := []struct {
  		desc   string
  		ok     bool
  		maxPar int
  		chatty bool
  		output string
  		f      func(*T)
  	}{{
  		desc:   "failnow skips future sequential and parallel tests at same level",
  		ok:     false,
  		maxPar: 1,
  		output: `
  --- FAIL: failnow skips future sequential and parallel tests at same level (N.NNs)
      --- FAIL: failnow skips future sequential and parallel tests at same level/#00 (N.NNs)
      `,
  		f: func(t *T) {
  			ranSeq := false
  			ranPar := false
  			t.Run("", func(t *T) {
  				t.Run("par", func(t *T) {
  					t.Parallel()
  					ranPar = true
  				})
  				t.Run("seq", func(t *T) {
  					ranSeq = true
  				})
  				t.FailNow()
  				t.Run("seq", func(t *T) {
  					realTest.Error("test must be skipped")
  				})
  				t.Run("par", func(t *T) {
  					t.Parallel()
  					realTest.Error("test must be skipped.")
  				})
  			})
  			if !ranPar {
  				realTest.Error("parallel test was not run")
  			}
  			if !ranSeq {
  				realTest.Error("sequential test was not run")
  			}
  		},
  	}, {
  		desc:   "failure in parallel test propagates upwards",
  		ok:     false,
  		maxPar: 1,
  		output: `
  --- FAIL: failure in parallel test propagates upwards (N.NNs)
      --- FAIL: failure in parallel test propagates upwards/#00 (N.NNs)
          --- FAIL: failure in parallel test propagates upwards/#00/par (N.NNs)
  		`,
  		f: func(t *T) {
  			t.Run("", func(t *T) {
  				t.Parallel()
  				t.Run("par", func(t *T) {
  					t.Parallel()
  					t.Fail()
  				})
  			})
  		},
  	}, {
  		desc:   "skipping without message, chatty",
  		ok:     true,
  		chatty: true,
  		output: `
  === RUN   skipping without message, chatty
  --- SKIP: skipping without message, chatty (N.NNs)`,
  		f: func(t *T) { t.SkipNow() },
  	}, {
  		desc:   "chatty with recursion",
  		ok:     true,
  		chatty: true,
  		output: `
  === RUN   chatty with recursion
  === RUN   chatty with recursion/#00
  === RUN   chatty with recursion/#00/#00
  --- PASS: chatty with recursion (N.NNs)
      --- PASS: chatty with recursion/#00 (N.NNs)
          --- PASS: chatty with recursion/#00/#00 (N.NNs)`,
  		f: func(t *T) {
  			t.Run("", func(t *T) {
  				t.Run("", func(t *T) {})
  			})
  		},
  	}, {
  		desc: "skipping without message, not chatty",
  		ok:   true,
  		f:    func(t *T) { t.SkipNow() },
  	}, {
  		desc: "skipping after error",
  		output: `
  --- FAIL: skipping after error (N.NNs)
  	sub_test.go:NNN: an error
  	sub_test.go:NNN: skipped`,
  		f: func(t *T) {
  			t.Error("an error")
  			t.Skip("skipped")
  		},
  	}, {
  		desc:   "use Run to locally synchronize parallelism",
  		ok:     true,
  		maxPar: 1,
  		f: func(t *T) {
  			var count uint32
  			t.Run("waitGroup", func(t *T) {
  				for i := 0; i < 4; i++ {
  					t.Run("par", func(t *T) {
  						t.Parallel()
  						atomic.AddUint32(&count, 1)
  					})
  				}
  			})
  			if count != 4 {
  				t.Errorf("count was %d; want 4", count)
  			}
  		},
  	}, {
  		desc: "alternate sequential and parallel",
  		// Sequential tests should partake in the counting of running threads.
  		// Otherwise, if one runs parallel subtests in sequential tests that are
  		// itself subtests of parallel tests, the counts can get askew.
  		ok:     true,
  		maxPar: 1,
  		f: func(t *T) {
  			t.Run("a", func(t *T) {
  				t.Parallel()
  				t.Run("b", func(t *T) {
  					// Sequential: ensure running count is decremented.
  					t.Run("c", func(t *T) {
  						t.Parallel()
  					})
  
  				})
  			})
  		},
  	}, {
  		desc: "alternate sequential and parallel 2",
  		// Sequential tests should partake in the counting of running threads.
  		// Otherwise, if one runs parallel subtests in sequential tests that are
  		// itself subtests of parallel tests, the counts can get askew.
  		ok:     true,
  		maxPar: 2,
  		f: func(t *T) {
  			for i := 0; i < 2; i++ {
  				t.Run("a", func(t *T) {
  					t.Parallel()
  					time.Sleep(time.Nanosecond)
  					for i := 0; i < 2; i++ {
  						t.Run("b", func(t *T) {
  							time.Sleep(time.Nanosecond)
  							for i := 0; i < 2; i++ {
  								t.Run("c", func(t *T) {
  									t.Parallel()
  									time.Sleep(time.Nanosecond)
  								})
  							}
  
  						})
  					}
  				})
  			}
  		},
  	}, {
  		desc:   "stress test",
  		ok:     true,
  		maxPar: 4,
  		f: func(t *T) {
  			t.Parallel()
  			for i := 0; i < 12; i++ {
  				t.Run("a", func(t *T) {
  					t.Parallel()
  					time.Sleep(time.Nanosecond)
  					for i := 0; i < 12; i++ {
  						t.Run("b", func(t *T) {
  							time.Sleep(time.Nanosecond)
  							for i := 0; i < 12; i++ {
  								t.Run("c", func(t *T) {
  									t.Parallel()
  									time.Sleep(time.Nanosecond)
  									t.Run("d1", func(t *T) {})
  									t.Run("d2", func(t *T) {})
  									t.Run("d3", func(t *T) {})
  									t.Run("d4", func(t *T) {})
  								})
  							}
  						})
  					}
  				})
  			}
  		},
  	}, {
  		desc:   "skip output",
  		ok:     true,
  		maxPar: 4,
  		f: func(t *T) {
  			t.Skip()
  		},
  	}, {
  		desc:   "panic on goroutine fail after test exit",
  		ok:     false,
  		maxPar: 4,
  		f: func(t *T) {
  			ch := make(chan bool)
  			t.Run("", func(t *T) {
  				go func() {
  					<-ch
  					defer func() {
  						if r := recover(); r == nil {
  							realTest.Errorf("expected panic")
  						}
  						ch <- true
  					}()
  					t.Errorf("failed after success")
  				}()
  			})
  			ch <- true
  			<-ch
  		},
  	}}
  	for _, tc := range testCases {
  		ctx := newTestContext(tc.maxPar, newMatcher(regexp.MatchString, "", ""))
  		buf := &bytes.Buffer{}
  		root := &T{
  			common: common{
  				signal: make(chan bool),
  				name:   "Test",
  				w:      buf,
  				chatty: tc.chatty,
  			},
  			context: ctx,
  		}
  		ok := root.Run(tc.desc, tc.f)
  		ctx.release()
  
  		if ok != tc.ok {
  			t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, tc.ok)
  		}
  		if ok != !root.Failed() {
  			t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed())
  		}
  		if ctx.running != 0 || ctx.numWaiting != 0 {
  			t.Errorf("%s:running and waiting non-zero: got %d and %d", tc.desc, ctx.running, ctx.numWaiting)
  		}
  		got := strings.TrimSpace(buf.String())
  		want := strings.TrimSpace(tc.output)
  		re := makeRegexp(want)
  		if ok, err := regexp.MatchString(re, got); !ok || err != nil {
  			t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, got, want)
  		}
  	}
  }
  
  func TestBRun(t *T) {
  	work := func(b *B) {
  		for i := 0; i < b.N; i++ {
  			time.Sleep(time.Nanosecond)
  		}
  	}
  	testCases := []struct {
  		desc   string
  		failed bool
  		chatty bool
  		output string
  		f      func(*B)
  	}{{
  		desc: "simulate sequential run of subbenchmarks.",
  		f: func(b *B) {
  			b.Run("", func(b *B) { work(b) })
  			time1 := b.result.NsPerOp()
  			b.Run("", func(b *B) { work(b) })
  			time2 := b.result.NsPerOp()
  			if time1 >= time2 {
  				t.Errorf("no time spent in benchmark t1 >= t2 (%d >= %d)", time1, time2)
  			}
  		},
  	}, {
  		desc: "bytes set by all benchmarks",
  		f: func(b *B) {
  			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
  			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
  			if b.result.Bytes != 20 {
  				t.Errorf("bytes: got: %d; want 20", b.result.Bytes)
  			}
  		},
  	}, {
  		desc: "bytes set by some benchmarks",
  		// In this case the bytes result is meaningless, so it must be 0.
  		f: func(b *B) {
  			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
  			b.Run("", func(b *B) { work(b) })
  			b.Run("", func(b *B) { b.SetBytes(10); work(b) })
  			if b.result.Bytes != 0 {
  				t.Errorf("bytes: got: %d; want 0", b.result.Bytes)
  			}
  		},
  	}, {
  		desc:   "failure carried over to root",
  		failed: true,
  		output: "--- FAIL: root",
  		f:      func(b *B) { b.Fail() },
  	}, {
  		desc:   "skipping without message, chatty",
  		chatty: true,
  		output: "--- SKIP: root",
  		f:      func(b *B) { b.SkipNow() },
  	}, {
  		desc:   "skipping with message, chatty",
  		chatty: true,
  		output: `
  --- SKIP: root
  	sub_test.go:NNN: skipping`,
  		f: func(b *B) { b.Skip("skipping") },
  	}, {
  		desc:   "chatty with recursion",
  		chatty: true,
  		f: func(b *B) {
  			b.Run("", func(b *B) {
  				b.Run("", func(b *B) {})
  			})
  		},
  	}, {
  		desc: "skipping without message, not chatty",
  		f:    func(b *B) { b.SkipNow() },
  	}, {
  		desc:   "skipping after error",
  		failed: true,
  		output: `
  --- FAIL: root
  	sub_test.go:NNN: an error
  	sub_test.go:NNN: skipped`,
  		f: func(b *B) {
  			b.Error("an error")
  			b.Skip("skipped")
  		},
  	}, {
  		desc: "memory allocation",
  		f: func(b *B) {
  			const bufSize = 256
  			alloc := func(b *B) {
  				var buf [bufSize]byte
  				for i := 0; i < b.N; i++ {
  					_ = append([]byte(nil), buf[:]...)
  				}
  			}
  			b.Run("", func(b *B) {
  				alloc(b)
  				b.ReportAllocs()
  			})
  			b.Run("", func(b *B) {
  				alloc(b)
  				b.ReportAllocs()
  			})
  			// runtime.MemStats sometimes reports more allocations than the
  			// benchmark is responsible for. Luckily the point of this test is
  			// to ensure that the results are not underreported, so we can
  			// simply verify the lower bound.
  			if got := b.result.MemAllocs; got < 2 {
  				t.Errorf("MemAllocs was %v; want 2", got)
  			}
  			if got := b.result.MemBytes; got < 2*bufSize {
  				t.Errorf("MemBytes was %v; want %v", got, 2*bufSize)
  			}
  		},
  	}}
  	for _, tc := range testCases {
  		var ok bool
  		buf := &bytes.Buffer{}
  		// This is almost like the Benchmark function, except that we override
  		// the benchtime and catch the failure result of the subbenchmark.
  		root := &B{
  			common: common{
  				signal: make(chan bool),
  				name:   "root",
  				w:      buf,
  				chatty: tc.chatty,
  			},
  			benchFunc: func(b *B) { ok = b.Run("test", tc.f) }, // Use Run to catch failure.
  			benchTime: time.Microsecond,
  		}
  		root.runN(1)
  		if ok != !tc.failed {
  			t.Errorf("%s:ok: got %v; want %v", tc.desc, ok, !tc.failed)
  		}
  		if !ok != root.Failed() {
  			t.Errorf("%s:root failed: got %v; want %v", tc.desc, !ok, root.Failed())
  		}
  		// All tests are run as subtests
  		if root.result.N != 1 {
  			t.Errorf("%s: N for parent benchmark was %d; want 1", tc.desc, root.result.N)
  		}
  		got := strings.TrimSpace(buf.String())
  		want := strings.TrimSpace(tc.output)
  		re := makeRegexp(want)
  		if ok, err := regexp.MatchString(re, got); !ok || err != nil {
  			t.Errorf("%s:output:\ngot:\n%s\nwant:\n%s", tc.desc, got, want)
  		}
  	}
  }
  
  func makeRegexp(s string) string {
  	s = strings.Replace(s, ":NNN:", `:\d\d\d:`, -1)
  	s = strings.Replace(s, "(N.NNs)", `\(\d*\.\d*s\)`, -1)
  	return s
  }
  
  func TestBenchmarkOutput(t *T) {
  	// Ensure Benchmark initialized common.w by invoking it with an error and
  	// normal case.
  	Benchmark(func(b *B) { b.Error("do not print this output") })
  	Benchmark(func(b *B) {})
  }
  
  func TestBenchmarkStartsFrom1(t *T) {
  	var first = true
  	Benchmark(func(b *B) {
  		if first && b.N != 1 {
  			panic(fmt.Sprintf("Benchmark() first N=%v; want 1", b.N))
  		}
  		first = false
  	})
  }
  
  func TestBenchmarkReadMemStatsBeforeFirstRun(t *T) {
  	var first = true
  	Benchmark(func(b *B) {
  		if first && (b.startAllocs == 0 || b.startBytes == 0) {
  			panic(fmt.Sprintf("ReadMemStats not called before first run"))
  		}
  		first = false
  	})
  }
  
  func TestParallelSub(t *T) {
  	c := make(chan int)
  	block := make(chan int)
  	for i := 0; i < 10; i++ {
  		go func(i int) {
  			<-block
  			t.Run(fmt.Sprint(i), func(t *T) {})
  			c <- 1
  		}(i)
  	}
  	close(block)
  	for i := 0; i < 10; i++ {
  		<-c
  	}
  }
  
  type funcWriter func([]byte) (int, error)
  
  func (fw funcWriter) Write(b []byte) (int, error) { return fw(b) }
  
  func TestRacyOutput(t *T) {
  	var runs int32  // The number of running Writes
  	var races int32 // Incremented for each race detected
  	raceDetector := func(b []byte) (int, error) {
  		// Check if some other goroutine is concurrently calling Write.
  		if atomic.LoadInt32(&runs) > 0 {
  			atomic.AddInt32(&races, 1) // Race detected!
  		}
  		atomic.AddInt32(&runs, 1)
  		defer atomic.AddInt32(&runs, -1)
  		runtime.Gosched() // Increase probability of a race
  		return len(b), nil
  	}
  
  	var wg sync.WaitGroup
  	root := &T{
  		common:  common{w: funcWriter(raceDetector), chatty: true},
  		context: newTestContext(1, newMatcher(regexp.MatchString, "", "")),
  	}
  	root.Run("", func(t *T) {
  		for i := 0; i < 100; i++ {
  			wg.Add(1)
  			go func(i int) {
  				defer wg.Done()
  				t.Run(fmt.Sprint(i), func(t *T) {
  					t.Logf("testing run %d", i)
  				})
  			}(i)
  		}
  	})
  	wg.Wait()
  
  	if races > 0 {
  		t.Errorf("detected %d racy Writes", races)
  	}
  }
  
  func TestBenchmark(t *T) {
  	res := Benchmark(func(b *B) {
  		for i := 0; i < 5; i++ {
  			b.Run("", func(b *B) {
  				for i := 0; i < b.N; i++ {
  					time.Sleep(time.Millisecond)
  				}
  			})
  		}
  	})
  	if res.NsPerOp() < 4000000 {
  		t.Errorf("want >5ms; got %v", time.Duration(res.NsPerOp()))
  	}
  }
  

View as plain text