Source file src/pkg/testing/testing.go

     1	// Copyright 2009 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// Package testing provides support for automated testing of Go packages.
     6	// It is intended to be used in concert with the ``go test'' command, which automates
     7	// execution of any function of the form
     8	//     func TestXxx(*testing.T)
     9	// where Xxx can be any alphanumeric string (but the first letter must not be in
    10	// [a-z]) and serves to identify the test routine.
    11	// These TestXxx routines should be declared within the package they are testing.
    12	//
    13	// Tests and benchmarks may be skipped if not applicable like this:
    14	//     func TestTimeConsuming(t *testing.T) {
    15	//         if testing.Short() {
    16	//             t.Skip("skipping test in short mode.")
    17	//         }
    18	//         ...
    19	//     }
    20	//
    21	// Benchmarks
    22	//
    23	// Functions of the form
    24	//     func BenchmarkXxx(*testing.B)
    25	// are considered benchmarks, and are executed by the "go test" command when
    26	// the -test.bench flag is provided. Benchmarks are run sequentially.
    27	//
    28	// For a description of the testing flags, see
    29	// http://golang.org/cmd/go/#Description_of_testing_flags.
    30	//
    31	// A sample benchmark function looks like this:
    32	//     func BenchmarkHello(b *testing.B) {
    33	//         for i := 0; i < b.N; i++ {
    34	//             fmt.Sprintf("hello")
    35	//         }
    36	//     }
    37	//
    38	// The benchmark function must run the target code b.N times.
    39	// The benchmark package will vary b.N until the benchmark function lasts
    40	// long enough to be timed reliably.  The output
    41	//     BenchmarkHello    10000000    282 ns/op
    42	// means that the loop ran 10000000 times at a speed of 282 ns per loop.
    43	//
    44	// If a benchmark needs some expensive setup before running, the timer
    45	// may be reset:
    46	//     func BenchmarkBigLen(b *testing.B) {
    47	//         big := NewBig()
    48	//         b.ResetTimer()
    49	//         for i := 0; i < b.N; i++ {
    50	//             big.Len()
    51	//         }
    52	//     }
    53	//
    54	// Examples
    55	//
    56	// The package also runs and verifies example code. Example functions may
    57	// include a concluding line comment that begins with "Output:" and is compared with
    58	// the standard output of the function when the tests are run. (The comparison
    59	// ignores leading and trailing space.) These are examples of an example:
    60	//
    61	//     func ExampleHello() {
    62	//             fmt.Println("hello")
    63	//             // Output: hello
    64	//     }
    65	//
    66	//     func ExampleSalutations() {
    67	//             fmt.Println("hello, and")
    68	//             fmt.Println("goodbye")
    69	//             // Output:
    70	//             // hello, and
    71	//             // goodbye
    72	//     }
    73	//
    74	// Example functions without output comments are compiled but not executed.
    75	//
    76	// The naming convention to declare examples for a function F, a type T and
    77	// method M on type T are:
    78	//
    79	//     func ExampleF() { ... }
    80	//     func ExampleT() { ... }
    81	//     func ExampleT_M() { ... }
    82	//
    83	// Multiple example functions for a type/function/method may be provided by
    84	// appending a distinct suffix to the name. The suffix must start with a
    85	// lower-case letter.
    86	//
    87	//     func ExampleF_suffix() { ... }
    88	//     func ExampleT_suffix() { ... }
    89	//     func ExampleT_M_suffix() { ... }
    90	//
    91	// The entire test file is presented as the example when it contains a single
    92	// example function, at least one other function, type, variable, or constant
    93	// declaration, and no test or benchmark functions.
    94	package testing
    95	
    96	import (
    97		"bytes"
    98		"flag"
    99		"fmt"
   100		"os"
   101		"runtime"
   102		"runtime/pprof"
   103		"strconv"
   104		"strings"
   105		"sync"
   106		"time"
   107	)
   108	
   109	var (
   110		// The short flag requests that tests run more quickly, but its functionality
   111		// is provided by test writers themselves.  The testing package is just its
   112		// home.  The all.bash installation script sets it to make installation more
   113		// efficient, but by default the flag is off so a plain "go test" will do a
   114		// full test of the package.
   115		short = flag.Bool("test.short", false, "run smaller test suite to save time")
   116	
   117		// Report as tests are run; default is silent for success.
   118		chatty           = flag.Bool("test.v", false, "verbose: print additional output")
   119		match            = flag.String("test.run", "", "regular expression to select tests and examples to run")
   120		memProfile       = flag.String("test.memprofile", "", "write a memory profile to the named file after execution")
   121		memProfileRate   = flag.Int("test.memprofilerate", 0, "if >=0, sets runtime.MemProfileRate")
   122		cpuProfile       = flag.String("test.cpuprofile", "", "write a cpu profile to the named file during execution")
   123		blockProfile     = flag.String("test.blockprofile", "", "write a goroutine blocking profile to the named file after execution")
   124		blockProfileRate = flag.Int("test.blockprofilerate", 1, "if >= 0, calls runtime.SetBlockProfileRate()")
   125		timeout          = flag.Duration("test.timeout", 0, "if positive, sets an aggregate time limit for all tests")
   126		cpuListStr       = flag.String("test.cpu", "", "comma-separated list of number of CPUs to use for each test")
   127		parallel         = flag.Int("test.parallel", runtime.GOMAXPROCS(0), "maximum test parallelism")
   128	
   129		haveExamples bool // are there examples?
   130	
   131		cpuList []int
   132	)
   133	
   134	// common holds the elements common between T and B and
   135	// captures common methods such as Errorf.
   136	type common struct {
   137		mu      sync.RWMutex // guards output and failed
   138		output  []byte       // Output generated by test or benchmark.
   139		failed  bool         // Test or benchmark has failed.
   140		skipped bool         // Test of benchmark has been skipped.
   141	
   142		start    time.Time // Time test or benchmark started
   143		duration time.Duration
   144		self     interface{}      // To be sent on signal channel when done.
   145		signal   chan interface{} // Output for serial tests.
   146	}
   147	
   148	// Short reports whether the -test.short flag is set.
   149	func Short() bool {
   150		return *short
   151	}
   152	
   153	// Verbose reports whether the -test.v flag is set.
   154	func Verbose() bool {
   155		return *chatty
   156	}
   157	
   158	// decorate prefixes the string with the file and line of the call site
   159	// and inserts the final newline if needed and indentation tabs for formatting.
   160	func decorate(s string) string {
   161		_, file, line, ok := runtime.Caller(3) // decorate + log + public function.
   162		if ok {
   163			// Truncate file name at last file name separator.
   164			if index := strings.LastIndex(file, "/"); index >= 0 {
   165				file = file[index+1:]
   166			} else if index = strings.LastIndex(file, "\\"); index >= 0 {
   167				file = file[index+1:]
   168			}
   169		} else {
   170			file = "???"
   171			line = 1
   172		}
   173		buf := new(bytes.Buffer)
   174		// Every line is indented at least one tab.
   175		buf.WriteByte('\t')
   176		fmt.Fprintf(buf, "%s:%d: ", file, line)
   177		lines := strings.Split(s, "\n")
   178		if l := len(lines); l > 1 && lines[l-1] == "" {
   179			lines = lines[:l-1]
   180		}
   181		for i, line := range lines {
   182			if i > 0 {
   183				// Second and subsequent lines are indented an extra tab.
   184				buf.WriteString("\n\t\t")
   185			}
   186			buf.WriteString(line)
   187		}
   188		buf.WriteByte('\n')
   189		return buf.String()
   190	}
   191	
   192	// T is a type passed to Test functions to manage test state and support formatted test logs.
   193	// Logs are accumulated during execution and dumped to standard error when done.
   194	type T struct {
   195		common
   196		name          string    // Name of test.
   197		startParallel chan bool // Parallel tests will wait on this.
   198	}
   199	
   200	// Fail marks the function as having failed but continues execution.
   201	func (c *common) Fail() {
   202		c.mu.Lock()
   203		defer c.mu.Unlock()
   204		c.failed = true
   205	}
   206	
   207	// Failed reports whether the function has failed.
   208	func (c *common) Failed() bool {
   209		c.mu.RLock()
   210		defer c.mu.RUnlock()
   211		return c.failed
   212	}
   213	
   214	// FailNow marks the function as having failed and stops its execution.
   215	// Execution will continue at the next test or benchmark.
   216	// FailNow must be called from the goroutine running the
   217	// test or benchmark function, not from other goroutines
   218	// created during the test. Calling FailNow does not stop
   219	// those other goroutines.
   220	func (c *common) FailNow() {
   221		c.Fail()
   222	
   223		// Calling runtime.Goexit will exit the goroutine, which
   224		// will run the deferred functions in this goroutine,
   225		// which will eventually run the deferred lines in tRunner,
   226		// which will signal to the test loop that this test is done.
   227		//
   228		// A previous version of this code said:
   229		//
   230		//	c.duration = ...
   231		//	c.signal <- c.self
   232		//	runtime.Goexit()
   233		//
   234		// This previous version duplicated code (those lines are in
   235		// tRunner no matter what), but worse the goroutine teardown
   236		// implicit in runtime.Goexit was not guaranteed to complete
   237		// before the test exited.  If a test deferred an important cleanup
   238		// function (like removing temporary files), there was no guarantee
   239		// it would run on a test failure.  Because we send on c.signal during
   240		// a top-of-stack deferred function now, we know that the send
   241		// only happens after any other stacked defers have completed.
   242		runtime.Goexit()
   243	}
   244	
   245	// log generates the output. It's always at the same stack depth.
   246	func (c *common) log(s string) {
   247		c.mu.Lock()
   248		defer c.mu.Unlock()
   249		c.output = append(c.output, decorate(s)...)
   250	}
   251	
   252	// Log formats its arguments using default formatting, analogous to Println,
   253	// and records the text in the error log. The text will be printed only if
   254	// the test fails or the -test.v flag is set.
   255	func (c *common) Log(args ...interface{}) { c.log(fmt.Sprintln(args...)) }
   256	
   257	// Logf formats its arguments according to the format, analogous to Printf,
   258	// and records the text in the error log. The text will be printed only if
   259	// the test fails or the -test.v flag is set.
   260	func (c *common) Logf(format string, args ...interface{}) { c.log(fmt.Sprintf(format, args...)) }
   261	
   262	// Error is equivalent to Log followed by Fail.
   263	func (c *common) Error(args ...interface{}) {
   264		c.log(fmt.Sprintln(args...))
   265		c.Fail()
   266	}
   267	
   268	// Errorf is equivalent to Logf followed by Fail.
   269	func (c *common) Errorf(format string, args ...interface{}) {
   270		c.log(fmt.Sprintf(format, args...))
   271		c.Fail()
   272	}
   273	
   274	// Fatal is equivalent to Log followed by FailNow.
   275	func (c *common) Fatal(args ...interface{}) {
   276		c.log(fmt.Sprintln(args...))
   277		c.FailNow()
   278	}
   279	
   280	// Fatalf is equivalent to Logf followed by FailNow.
   281	func (c *common) Fatalf(format string, args ...interface{}) {
   282		c.log(fmt.Sprintf(format, args...))
   283		c.FailNow()
   284	}
   285	
   286	// Skip is equivalent to Log followed by SkipNow.
   287	func (c *common) Skip(args ...interface{}) {
   288		c.log(fmt.Sprintln(args...))
   289		c.SkipNow()
   290	}
   291	
   292	// Skipf is equivalent to Logf followed by SkipNow.
   293	func (c *common) Skipf(format string, args ...interface{}) {
   294		c.log(fmt.Sprintf(format, args...))
   295		c.SkipNow()
   296	}
   297	
   298	// SkipNow marks the test as having been skipped and stops its execution.
   299	// Execution will continue at the next test or benchmark. See also FailNow.
   300	// SkipNow must be called from the goroutine running the test, not from
   301	// other goroutines created during the test. Calling SkipNow does not stop
   302	// those other goroutines.
   303	func (c *common) SkipNow() {
   304		c.skip()
   305		runtime.Goexit()
   306	}
   307	
   308	func (c *common) skip() {
   309		c.mu.Lock()
   310		defer c.mu.Unlock()
   311		c.skipped = true
   312	}
   313	
   314	// Skipped reports whether the test was skipped.
   315	func (c *common) Skipped() bool {
   316		c.mu.RLock()
   317		defer c.mu.RUnlock()
   318		return c.skipped
   319	}
   320	
   321	// Parallel signals that this test is to be run in parallel with (and only with)
   322	// other parallel tests.
   323	func (t *T) Parallel() {
   324		t.signal <- (*T)(nil) // Release main testing loop
   325		<-t.startParallel     // Wait for serial tests to finish
   326	}
   327	
   328	// An internal type but exported because it is cross-package; part of the implementation
   329	// of the "go test" command.
   330	type InternalTest struct {
   331		Name string
   332		F    func(*T)
   333	}
   334	
   335	func tRunner(t *T, test *InternalTest) {
   336		t.start = time.Now()
   337	
   338		// When this goroutine is done, either because test.F(t)
   339		// returned normally or because a test failure triggered
   340		// a call to runtime.Goexit, record the duration and send
   341		// a signal saying that the test is done.
   342		defer func() {
   343			t.duration = time.Now().Sub(t.start)
   344			// If the test panicked, print any test output before dying.
   345			if err := recover(); err != nil {
   346				t.Fail()
   347				t.report()
   348				panic(err)
   349			}
   350			t.signal <- t
   351		}()
   352	
   353		test.F(t)
   354	}
   355	
   356	// An internal function but exported because it is cross-package; part of the implementation
   357	// of the "go test" command.
   358	func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) {
   359		flag.Parse()
   360		parseCpuList()
   361	
   362		before()
   363		startAlarm()
   364		haveExamples = len(examples) > 0
   365		testOk := RunTests(matchString, tests)
   366		exampleOk := RunExamples(matchString, examples)
   367		if !testOk || !exampleOk {
   368			fmt.Println("FAIL")
   369			os.Exit(1)
   370		}
   371		fmt.Println("PASS")
   372		stopAlarm()
   373		RunBenchmarks(matchString, benchmarks)
   374		after()
   375	}
   376	
   377	func (t *T) report() {
   378		tstr := fmt.Sprintf("(%.2f seconds)", t.duration.Seconds())
   379		format := "--- %s: %s %s\n%s"
   380		if t.Failed() {
   381			fmt.Printf(format, "FAIL", t.name, tstr, t.output)
   382		} else if *chatty {
   383			if t.Skipped() {
   384				fmt.Printf(format, "SKIP", t.name, tstr, t.output)
   385			} else {
   386				fmt.Printf(format, "PASS", t.name, tstr, t.output)
   387			}
   388		}
   389	}
   390	
   391	func RunTests(matchString func(pat, str string) (bool, error), tests []InternalTest) (ok bool) {
   392		ok = true
   393		if len(tests) == 0 && !haveExamples {
   394			fmt.Fprintln(os.Stderr, "testing: warning: no tests to run")
   395			return
   396		}
   397		for _, procs := range cpuList {
   398			runtime.GOMAXPROCS(procs)
   399			// We build a new channel tree for each run of the loop.
   400			// collector merges in one channel all the upstream signals from parallel tests.
   401			// If all tests pump to the same channel, a bug can occur where a test
   402			// kicks off a goroutine that Fails, yet the test still delivers a completion signal,
   403			// which skews the counting.
   404			var collector = make(chan interface{})
   405	
   406			numParallel := 0
   407			startParallel := make(chan bool)
   408	
   409			for i := 0; i < len(tests); i++ {
   410				matched, err := matchString(*match, tests[i].Name)
   411				if err != nil {
   412					fmt.Fprintf(os.Stderr, "testing: invalid regexp for -test.run: %s\n", err)
   413					os.Exit(1)
   414				}
   415				if !matched {
   416					continue
   417				}
   418				testName := tests[i].Name
   419				if procs != 1 {
   420					testName = fmt.Sprintf("%s-%d", tests[i].Name, procs)
   421				}
   422				t := &T{
   423					common: common{
   424						signal: make(chan interface{}),
   425					},
   426					name:          testName,
   427					startParallel: startParallel,
   428				}
   429				t.self = t
   430				if *chatty {
   431					fmt.Printf("=== RUN %s\n", t.name)
   432				}
   433				go tRunner(t, &tests[i])
   434				out := (<-t.signal).(*T)
   435				if out == nil { // Parallel run.
   436					go func() {
   437						collector <- <-t.signal
   438					}()
   439					numParallel++
   440					continue
   441				}
   442				t.report()
   443				ok = ok && !out.Failed()
   444			}
   445	
   446			running := 0
   447			for numParallel+running > 0 {
   448				if running < *parallel && numParallel > 0 {
   449					startParallel <- true
   450					running++
   451					numParallel--
   452					continue
   453				}
   454				t := (<-collector).(*T)
   455				t.report()
   456				ok = ok && !t.Failed()
   457				running--
   458			}
   459		}
   460		return
   461	}
   462	
   463	// before runs before all testing.
   464	func before() {
   465		if *memProfileRate > 0 {
   466			runtime.MemProfileRate = *memProfileRate
   467		}
   468		if *cpuProfile != "" {
   469			f, err := os.Create(*cpuProfile)
   470			if err != nil {
   471				fmt.Fprintf(os.Stderr, "testing: %s", err)
   472				return
   473			}
   474			if err := pprof.StartCPUProfile(f); err != nil {
   475				fmt.Fprintf(os.Stderr, "testing: can't start cpu profile: %s", err)
   476				f.Close()
   477				return
   478			}
   479			// Could save f so after can call f.Close; not worth the effort.
   480		}
   481		if *blockProfile != "" && *blockProfileRate >= 0 {
   482			runtime.SetBlockProfileRate(*blockProfileRate)
   483		}
   484	}
   485	
   486	// after runs after all testing.
   487	func after() {
   488		if *cpuProfile != "" {
   489			pprof.StopCPUProfile() // flushes profile to disk
   490		}
   491		if *memProfile != "" {
   492			f, err := os.Create(*memProfile)
   493			if err != nil {
   494				fmt.Fprintf(os.Stderr, "testing: %s", err)
   495				return
   496			}
   497			if err = pprof.WriteHeapProfile(f); err != nil {
   498				fmt.Fprintf(os.Stderr, "testing: can't write %s: %s", *memProfile, err)
   499			}
   500			f.Close()
   501		}
   502		if *blockProfile != "" && *blockProfileRate >= 0 {
   503			f, err := os.Create(*blockProfile)
   504			if err != nil {
   505				fmt.Fprintf(os.Stderr, "testing: %s", err)
   506				return
   507			}
   508			if err = pprof.Lookup("block").WriteTo(f, 0); err != nil {
   509				fmt.Fprintf(os.Stderr, "testing: can't write %s: %s", *blockProfile, err)
   510			}
   511			f.Close()
   512		}
   513	}
   514	
   515	var timer *time.Timer
   516	
   517	// startAlarm starts an alarm if requested.
   518	func startAlarm() {
   519		if *timeout > 0 {
   520			timer = time.AfterFunc(*timeout, alarm)
   521		}
   522	}
   523	
   524	// stopAlarm turns off the alarm.
   525	func stopAlarm() {
   526		if *timeout > 0 {
   527			timer.Stop()
   528		}
   529	}
   530	
   531	// alarm is called if the timeout expires.
   532	func alarm() {
   533		panic("test timed out")
   534	}
   535	
   536	func parseCpuList() {
   537		if len(*cpuListStr) == 0 {
   538			cpuList = append(cpuList, runtime.GOMAXPROCS(-1))
   539		} else {
   540			for _, val := range strings.Split(*cpuListStr, ",") {
   541				cpu, err := strconv.Atoi(val)
   542				if err != nil || cpu <= 0 {
   543					fmt.Fprintf(os.Stderr, "testing: invalid value %q for -test.cpu", val)
   544					os.Exit(1)
   545				}
   546				cpuList = append(cpuList, cpu)
   547			}
   548		}
   549	}

View as plain text