...
Run Format

Source file src/pkg/testing/testing.go

     1	// Copyright 2009 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// Package testing provides support for automated testing of Go packages.
     6	// It is intended to be used in concert with the ``go test'' command, which automates
     7	// execution of any function of the form
     8	//     func TestXxx(*testing.T)
     9	// where Xxx can be any alphanumeric string (but the first letter must not be in
    10	// [a-z]) and serves to identify the test routine.
    11	//
    12	// Within these functions, use the Error, Fail or related methods to signal failure.
    13	//
    14	// To write a new test suite, create a file whose name ends _test.go that
    15	// contains the TestXxx functions as described here. Put the file in the same
    16	// package as the one being tested. The file will be excluded from regular
    17	// package builds but will be included when the ``go test'' command is run.
    18	// For more detail, run ``go help test'' and ``go help testflag''.
    19	//
    20	// Tests and benchmarks may be skipped if not applicable with a call to
    21	// the Skip method of *T and *B:
    22	//     func TestTimeConsuming(t *testing.T) {
    23	//         if testing.Short() {
    24	//             t.Skip("skipping test in short mode.")
    25	//         }
    26	//         ...
    27	//     }
    28	//
    29	// Benchmarks
    30	//
    31	// Functions of the form
    32	//     func BenchmarkXxx(*testing.B)
    33	// are considered benchmarks, and are executed by the "go test" command when
    34	// its -bench flag is provided. Benchmarks are run sequentially.
    35	//
    36	// For a description of the testing flags, see
    37	// http://golang.org/cmd/go/#hdr-Description_of_testing_flags.
    38	//
    39	// A sample benchmark function looks like this:
    40	//     func BenchmarkHello(b *testing.B) {
    41	//         for i := 0; i < b.N; i++ {
    42	//             fmt.Sprintf("hello")
    43	//         }
    44	//     }
    45	//
    46	// The benchmark function must run the target code b.N times.
    47	// The benchmark package will vary b.N until the benchmark function lasts
    48	// long enough to be timed reliably.  The output
    49	//     BenchmarkHello    10000000    282 ns/op
    50	// means that the loop ran 10000000 times at a speed of 282 ns per loop.
    51	//
    52	// If a benchmark needs some expensive setup before running, the timer
    53	// may be reset:
    54	//
    55	//     func BenchmarkBigLen(b *testing.B) {
    56	//         big := NewBig()
    57	//         b.ResetTimer()
    58	//         for i := 0; i < b.N; i++ {
    59	//             big.Len()
    60	//         }
    61	//     }
    62	//
    63	// If a benchmark needs to test performance in a parallel setting, it may use
    64	// the RunParallel helper function; such benchmarks are intended to be used with
    65	// the go test -cpu flag:
    66	//
    67	//     func BenchmarkTemplateParallel(b *testing.B) {
    68	//         templ := template.Must(template.New("test").Parse("Hello, {{.}}!"))
    69	//         b.RunParallel(func(pb *testing.PB) {
    70	//             var buf bytes.Buffer
    71	//             for pb.Next() {
    72	//                 buf.Reset()
    73	//                 templ.Execute(&buf, "World")
    74	//             }
    75	//         })
    76	//     }
    77	//
    78	// Examples
    79	//
    80	// The package also runs and verifies example code. Example functions may
    81	// include a concluding line comment that begins with "Output:" and is compared with
    82	// the standard output of the function when the tests are run. (The comparison
    83	// ignores leading and trailing space.) These are examples of an example:
    84	//
    85	//     func ExampleHello() {
    86	//             fmt.Println("hello")
    87	//             // Output: hello
    88	//     }
    89	//
    90	//     func ExampleSalutations() {
    91	//             fmt.Println("hello, and")
    92	//             fmt.Println("goodbye")
    93	//             // Output:
    94	//             // hello, and
    95	//             // goodbye
    96	//     }
    97	//
    98	// Example functions without output comments are compiled but not executed.
    99	//
   100	// The naming convention to declare examples for the package, a function F, a type T and
   101	// method M on type T are:
   102	//
   103	//     func Example() { ... }
   104	//     func ExampleF() { ... }
   105	//     func ExampleT() { ... }
   106	//     func ExampleT_M() { ... }
   107	//
   108	// Multiple example functions for a package/type/function/method may be provided by
   109	// appending a distinct suffix to the name. The suffix must start with a
   110	// lower-case letter.
   111	//
   112	//     func Example_suffix() { ... }
   113	//     func ExampleF_suffix() { ... }
   114	//     func ExampleT_suffix() { ... }
   115	//     func ExampleT_M_suffix() { ... }
   116	//
   117	// The entire test file is presented as the example when it contains a single
   118	// example function, at least one other function, type, variable, or constant
   119	// declaration, and no test or benchmark functions.
   120	package testing
   121	
   122	import (
   123		"bytes"
   124		"flag"
   125		"fmt"
   126		"os"
   127		"runtime"
   128		"runtime/pprof"
   129		"strconv"
   130		"strings"
   131		"sync"
   132		"time"
   133	)
   134	
   135	var (
   136		// The short flag requests that tests run more quickly, but its functionality
   137		// is provided by test writers themselves.  The testing package is just its
   138		// home.  The all.bash installation script sets it to make installation more
   139		// efficient, but by default the flag is off so a plain "go test" will do a
   140		// full test of the package.
   141		short = flag.Bool("test.short", false, "run smaller test suite to save time")
   142	
   143		// The directory in which to create profile files and the like. When run from
   144		// "go test", the binary always runs in the source directory for the package;
   145		// this flag lets "go test" tell the binary to write the files in the directory where
   146		// the "go test" command is run.
   147		outputDir = flag.String("test.outputdir", "", "directory in which to write profiles")
   148	
   149		// Report as tests are run; default is silent for success.
   150		chatty           = flag.Bool("test.v", false, "verbose: print additional output")
   151		coverProfile     = flag.String("test.coverprofile", "", "write a coverage profile to the named file after execution")
   152		match            = flag.String("test.run", "", "regular expression to select tests and examples to run")
   153		memProfile       = flag.String("test.memprofile", "", "write a memory profile to the named file after execution")
   154		memProfileRate   = flag.Int("test.memprofilerate", 0, "if >=0, sets runtime.MemProfileRate")
   155		cpuProfile       = flag.String("test.cpuprofile", "", "write a cpu profile to the named file during execution")
   156		blockProfile     = flag.String("test.blockprofile", "", "write a goroutine blocking profile to the named file after execution")
   157		blockProfileRate = flag.Int("test.blockprofilerate", 1, "if >= 0, calls runtime.SetBlockProfileRate()")
   158		timeout          = flag.Duration("test.timeout", 0, "if positive, sets an aggregate time limit for all tests")
   159		cpuListStr       = flag.String("test.cpu", "", "comma-separated list of number of CPUs to use for each test")
   160		parallel         = flag.Int("test.parallel", runtime.GOMAXPROCS(0), "maximum test parallelism")
   161	
   162		haveExamples bool // are there examples?
   163	
   164		cpuList []int
   165	)
   166	
   167	// common holds the elements common between T and B and
   168	// captures common methods such as Errorf.
   169	type common struct {
   170		mu       sync.RWMutex // guards output and failed
   171		output   []byte       // Output generated by test or benchmark.
   172		failed   bool         // Test or benchmark has failed.
   173		skipped  bool         // Test of benchmark has been skipped.
   174		finished bool
   175	
   176		start    time.Time // Time test or benchmark started
   177		duration time.Duration
   178		self     interface{}      // To be sent on signal channel when done.
   179		signal   chan interface{} // Output for serial tests.
   180	}
   181	
   182	// Short reports whether the -test.short flag is set.
   183	func Short() bool {
   184		return *short
   185	}
   186	
   187	// Verbose reports whether the -test.v flag is set.
   188	func Verbose() bool {
   189		return *chatty
   190	}
   191	
   192	// decorate prefixes the string with the file and line of the call site
   193	// and inserts the final newline if needed and indentation tabs for formatting.
   194	func decorate(s string) string {
   195		_, file, line, ok := runtime.Caller(3) // decorate + log + public function.
   196		if ok {
   197			// Truncate file name at last file name separator.
   198			if index := strings.LastIndex(file, "/"); index >= 0 {
   199				file = file[index+1:]
   200			} else if index = strings.LastIndex(file, "\\"); index >= 0 {
   201				file = file[index+1:]
   202			}
   203		} else {
   204			file = "???"
   205			line = 1
   206		}
   207		buf := new(bytes.Buffer)
   208		// Every line is indented at least one tab.
   209		buf.WriteByte('\t')
   210		fmt.Fprintf(buf, "%s:%d: ", file, line)
   211		lines := strings.Split(s, "\n")
   212		if l := len(lines); l > 1 && lines[l-1] == "" {
   213			lines = lines[:l-1]
   214		}
   215		for i, line := range lines {
   216			if i > 0 {
   217				// Second and subsequent lines are indented an extra tab.
   218				buf.WriteString("\n\t\t")
   219			}
   220			buf.WriteString(line)
   221		}
   222		buf.WriteByte('\n')
   223		return buf.String()
   224	}
   225	
   226	// TB is the interface common to T and B.
   227	type TB interface {
   228		Error(args ...interface{})
   229		Errorf(format string, args ...interface{})
   230		Fail()
   231		FailNow()
   232		Failed() bool
   233		Fatal(args ...interface{})
   234		Fatalf(format string, args ...interface{})
   235		Log(args ...interface{})
   236		Logf(format string, args ...interface{})
   237		Skip(args ...interface{})
   238		SkipNow()
   239		Skipf(format string, args ...interface{})
   240		Skipped() bool
   241	
   242		// A private method to prevent users implementing the
   243		// interface and so future additions to it will not
   244		// violate Go 1 compatibility.
   245		private()
   246	}
   247	
   248	var _ TB = (*T)(nil)
   249	var _ TB = (*B)(nil)
   250	
   251	// T is a type passed to Test functions to manage test state and support formatted test logs.
   252	// Logs are accumulated during execution and dumped to standard error when done.
   253	type T struct {
   254		common
   255		name          string    // Name of test.
   256		startParallel chan bool // Parallel tests will wait on this.
   257	}
   258	
   259	func (c *common) private() {}
   260	
   261	// Fail marks the function as having failed but continues execution.
   262	func (c *common) Fail() {
   263		c.mu.Lock()
   264		defer c.mu.Unlock()
   265		c.failed = true
   266	}
   267	
   268	// Failed reports whether the function has failed.
   269	func (c *common) Failed() bool {
   270		c.mu.RLock()
   271		defer c.mu.RUnlock()
   272		return c.failed
   273	}
   274	
   275	// FailNow marks the function as having failed and stops its execution.
   276	// Execution will continue at the next test or benchmark.
   277	// FailNow must be called from the goroutine running the
   278	// test or benchmark function, not from other goroutines
   279	// created during the test. Calling FailNow does not stop
   280	// those other goroutines.
   281	func (c *common) FailNow() {
   282		c.Fail()
   283	
   284		// Calling runtime.Goexit will exit the goroutine, which
   285		// will run the deferred functions in this goroutine,
   286		// which will eventually run the deferred lines in tRunner,
   287		// which will signal to the test loop that this test is done.
   288		//
   289		// A previous version of this code said:
   290		//
   291		//	c.duration = ...
   292		//	c.signal <- c.self
   293		//	runtime.Goexit()
   294		//
   295		// This previous version duplicated code (those lines are in
   296		// tRunner no matter what), but worse the goroutine teardown
   297		// implicit in runtime.Goexit was not guaranteed to complete
   298		// before the test exited.  If a test deferred an important cleanup
   299		// function (like removing temporary files), there was no guarantee
   300		// it would run on a test failure.  Because we send on c.signal during
   301		// a top-of-stack deferred function now, we know that the send
   302		// only happens after any other stacked defers have completed.
   303		c.finished = true
   304		runtime.Goexit()
   305	}
   306	
   307	// log generates the output. It's always at the same stack depth.
   308	func (c *common) log(s string) {
   309		c.mu.Lock()
   310		defer c.mu.Unlock()
   311		c.output = append(c.output, decorate(s)...)
   312	}
   313	
   314	// Log formats its arguments using default formatting, analogous to Println,
   315	// and records the text in the error log. The text will be printed only if
   316	// the test fails or the -test.v flag is set.
   317	func (c *common) Log(args ...interface{}) { c.log(fmt.Sprintln(args...)) }
   318	
   319	// Logf formats its arguments according to the format, analogous to Printf,
   320	// and records the text in the error log. The text will be printed only if
   321	// the test fails or the -test.v flag is set.
   322	func (c *common) Logf(format string, args ...interface{}) { c.log(fmt.Sprintf(format, args...)) }
   323	
   324	// Error is equivalent to Log followed by Fail.
   325	func (c *common) Error(args ...interface{}) {
   326		c.log(fmt.Sprintln(args...))
   327		c.Fail()
   328	}
   329	
   330	// Errorf is equivalent to Logf followed by Fail.
   331	func (c *common) Errorf(format string, args ...interface{}) {
   332		c.log(fmt.Sprintf(format, args...))
   333		c.Fail()
   334	}
   335	
   336	// Fatal is equivalent to Log followed by FailNow.
   337	func (c *common) Fatal(args ...interface{}) {
   338		c.log(fmt.Sprintln(args...))
   339		c.FailNow()
   340	}
   341	
   342	// Fatalf is equivalent to Logf followed by FailNow.
   343	func (c *common) Fatalf(format string, args ...interface{}) {
   344		c.log(fmt.Sprintf(format, args...))
   345		c.FailNow()
   346	}
   347	
   348	// Skip is equivalent to Log followed by SkipNow.
   349	func (c *common) Skip(args ...interface{}) {
   350		c.log(fmt.Sprintln(args...))
   351		c.SkipNow()
   352	}
   353	
   354	// Skipf is equivalent to Logf followed by SkipNow.
   355	func (c *common) Skipf(format string, args ...interface{}) {
   356		c.log(fmt.Sprintf(format, args...))
   357		c.SkipNow()
   358	}
   359	
   360	// SkipNow marks the test as having been skipped and stops its execution.
   361	// Execution will continue at the next test or benchmark. See also FailNow.
   362	// SkipNow must be called from the goroutine running the test, not from
   363	// other goroutines created during the test. Calling SkipNow does not stop
   364	// those other goroutines.
   365	func (c *common) SkipNow() {
   366		c.skip()
   367		c.finished = true
   368		runtime.Goexit()
   369	}
   370	
   371	func (c *common) skip() {
   372		c.mu.Lock()
   373		defer c.mu.Unlock()
   374		c.skipped = true
   375	}
   376	
   377	// Skipped reports whether the test was skipped.
   378	func (c *common) Skipped() bool {
   379		c.mu.RLock()
   380		defer c.mu.RUnlock()
   381		return c.skipped
   382	}
   383	
   384	// Parallel signals that this test is to be run in parallel with (and only with)
   385	// other parallel tests.
   386	func (t *T) Parallel() {
   387		t.signal <- (*T)(nil) // Release main testing loop
   388		<-t.startParallel     // Wait for serial tests to finish
   389		// Assuming Parallel is the first thing a test does, which is reasonable,
   390		// reinitialize the test's start time because it's actually starting now.
   391		t.start = time.Now()
   392	}
   393	
   394	// An internal type but exported because it is cross-package; part of the implementation
   395	// of the "go test" command.
   396	type InternalTest struct {
   397		Name string
   398		F    func(*T)
   399	}
   400	
   401	func tRunner(t *T, test *InternalTest) {
   402		// When this goroutine is done, either because test.F(t)
   403		// returned normally or because a test failure triggered
   404		// a call to runtime.Goexit, record the duration and send
   405		// a signal saying that the test is done.
   406		defer func() {
   407			t.duration = time.Now().Sub(t.start)
   408			// If the test panicked, print any test output before dying.
   409			err := recover()
   410			if !t.finished && err == nil {
   411				err = fmt.Errorf("test executed panic(nil) or runtime.Goexit")
   412			}
   413			if err != nil {
   414				t.Fail()
   415				t.report()
   416				panic(err)
   417			}
   418			t.signal <- t
   419		}()
   420	
   421		t.start = time.Now()
   422		test.F(t)
   423		t.finished = true
   424	}
   425	
   426	// An internal function but exported because it is cross-package; part of the implementation
   427	// of the "go test" command.
   428	func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) {
   429		flag.Parse()
   430		parseCpuList()
   431	
   432		before()
   433		startAlarm()
   434		haveExamples = len(examples) > 0
   435		testOk := RunTests(matchString, tests)
   436		exampleOk := RunExamples(matchString, examples)
   437		stopAlarm()
   438		if !testOk || !exampleOk {
   439			fmt.Println("FAIL")
   440			after()
   441			os.Exit(1)
   442		}
   443		fmt.Println("PASS")
   444		RunBenchmarks(matchString, benchmarks)
   445		after()
   446	}
   447	
   448	func (t *T) report() {
   449		tstr := fmt.Sprintf("(%.2f seconds)", t.duration.Seconds())
   450		format := "--- %s: %s %s\n%s"
   451		if t.Failed() {
   452			fmt.Printf(format, "FAIL", t.name, tstr, t.output)
   453		} else if *chatty {
   454			if t.Skipped() {
   455				fmt.Printf(format, "SKIP", t.name, tstr, t.output)
   456			} else {
   457				fmt.Printf(format, "PASS", t.name, tstr, t.output)
   458			}
   459		}
   460	}
   461	
   462	func RunTests(matchString func(pat, str string) (bool, error), tests []InternalTest) (ok bool) {
   463		ok = true
   464		if len(tests) == 0 && !haveExamples {
   465			fmt.Fprintln(os.Stderr, "testing: warning: no tests to run")
   466			return
   467		}
   468		for _, procs := range cpuList {
   469			runtime.GOMAXPROCS(procs)
   470			// We build a new channel tree for each run of the loop.
   471			// collector merges in one channel all the upstream signals from parallel tests.
   472			// If all tests pump to the same channel, a bug can occur where a test
   473			// kicks off a goroutine that Fails, yet the test still delivers a completion signal,
   474			// which skews the counting.
   475			var collector = make(chan interface{})
   476	
   477			numParallel := 0
   478			startParallel := make(chan bool)
   479	
   480			for i := 0; i < len(tests); i++ {
   481				matched, err := matchString(*match, tests[i].Name)
   482				if err != nil {
   483					fmt.Fprintf(os.Stderr, "testing: invalid regexp for -test.run: %s\n", err)
   484					os.Exit(1)
   485				}
   486				if !matched {
   487					continue
   488				}
   489				testName := tests[i].Name
   490				if procs != 1 {
   491					testName = fmt.Sprintf("%s-%d", tests[i].Name, procs)
   492				}
   493				t := &T{
   494					common: common{
   495						signal: make(chan interface{}),
   496					},
   497					name:          testName,
   498					startParallel: startParallel,
   499				}
   500				t.self = t
   501				if *chatty {
   502					fmt.Printf("=== RUN %s\n", t.name)
   503				}
   504				go tRunner(t, &tests[i])
   505				out := (<-t.signal).(*T)
   506				if out == nil { // Parallel run.
   507					go func() {
   508						collector <- <-t.signal
   509					}()
   510					numParallel++
   511					continue
   512				}
   513				t.report()
   514				ok = ok && !out.Failed()
   515			}
   516	
   517			running := 0
   518			for numParallel+running > 0 {
   519				if running < *parallel && numParallel > 0 {
   520					startParallel <- true
   521					running++
   522					numParallel--
   523					continue
   524				}
   525				t := (<-collector).(*T)
   526				t.report()
   527				ok = ok && !t.Failed()
   528				running--
   529			}
   530		}
   531		return
   532	}
   533	
   534	// before runs before all testing.
   535	func before() {
   536		if *memProfileRate > 0 {
   537			runtime.MemProfileRate = *memProfileRate
   538		}
   539		if *cpuProfile != "" {
   540			f, err := os.Create(toOutputDir(*cpuProfile))
   541			if err != nil {
   542				fmt.Fprintf(os.Stderr, "testing: %s", err)
   543				return
   544			}
   545			if err := pprof.StartCPUProfile(f); err != nil {
   546				fmt.Fprintf(os.Stderr, "testing: can't start cpu profile: %s", err)
   547				f.Close()
   548				return
   549			}
   550			// Could save f so after can call f.Close; not worth the effort.
   551		}
   552		if *blockProfile != "" && *blockProfileRate >= 0 {
   553			runtime.SetBlockProfileRate(*blockProfileRate)
   554		}
   555		if *coverProfile != "" && cover.Mode == "" {
   556			fmt.Fprintf(os.Stderr, "testing: cannot use -test.coverprofile because test binary was not built with coverage enabled\n")
   557			os.Exit(2)
   558		}
   559	}
   560	
   561	// after runs after all testing.
   562	func after() {
   563		if *cpuProfile != "" {
   564			pprof.StopCPUProfile() // flushes profile to disk
   565		}
   566		if *memProfile != "" {
   567			f, err := os.Create(toOutputDir(*memProfile))
   568			if err != nil {
   569				fmt.Fprintf(os.Stderr, "testing: %s\n", err)
   570				os.Exit(2)
   571			}
   572			if err = pprof.WriteHeapProfile(f); err != nil {
   573				fmt.Fprintf(os.Stderr, "testing: can't write %s: %s\n", *memProfile, err)
   574				os.Exit(2)
   575			}
   576			f.Close()
   577		}
   578		if *blockProfile != "" && *blockProfileRate >= 0 {
   579			f, err := os.Create(toOutputDir(*blockProfile))
   580			if err != nil {
   581				fmt.Fprintf(os.Stderr, "testing: %s\n", err)
   582				os.Exit(2)
   583			}
   584			if err = pprof.Lookup("block").WriteTo(f, 0); err != nil {
   585				fmt.Fprintf(os.Stderr, "testing: can't write %s: %s\n", *blockProfile, err)
   586				os.Exit(2)
   587			}
   588			f.Close()
   589		}
   590		if cover.Mode != "" {
   591			coverReport()
   592		}
   593	}
   594	
   595	// toOutputDir returns the file name relocated, if required, to outputDir.
   596	// Simple implementation to avoid pulling in path/filepath.
   597	func toOutputDir(path string) string {
   598		if *outputDir == "" || path == "" {
   599			return path
   600		}
   601		if runtime.GOOS == "windows" {
   602			// On Windows, it's clumsy, but we can be almost always correct
   603			// by just looking for a drive letter and a colon.
   604			// Absolute paths always have a drive letter (ignoring UNC).
   605			// Problem: if path == "C:A" and outputdir == "C:\Go" it's unclear
   606			// what to do, but even then path/filepath doesn't help.
   607			// TODO: Worth doing better? Probably not, because we're here only
   608			// under the management of go test.
   609			if len(path) >= 2 {
   610				letter, colon := path[0], path[1]
   611				if ('a' <= letter && letter <= 'z' || 'A' <= letter && letter <= 'Z') && colon == ':' {
   612					// If path starts with a drive letter we're stuck with it regardless.
   613					return path
   614				}
   615			}
   616		}
   617		if os.IsPathSeparator(path[0]) {
   618			return path
   619		}
   620		return fmt.Sprintf("%s%c%s", *outputDir, os.PathSeparator, path)
   621	}
   622	
   623	var timer *time.Timer
   624	
   625	// startAlarm starts an alarm if requested.
   626	func startAlarm() {
   627		if *timeout > 0 {
   628			timer = time.AfterFunc(*timeout, func() {
   629				panic(fmt.Sprintf("test timed out after %v", *timeout))
   630			})
   631		}
   632	}
   633	
   634	// stopAlarm turns off the alarm.
   635	func stopAlarm() {
   636		if *timeout > 0 {
   637			timer.Stop()
   638		}
   639	}
   640	
   641	func parseCpuList() {
   642		for _, val := range strings.Split(*cpuListStr, ",") {
   643			val = strings.TrimSpace(val)
   644			if val == "" {
   645				continue
   646			}
   647			cpu, err := strconv.Atoi(val)
   648			if err != nil || cpu <= 0 {
   649				fmt.Fprintf(os.Stderr, "testing: invalid value %q for -test.cpu\n", val)
   650				os.Exit(1)
   651			}
   652			cpuList = append(cpuList, cpu)
   653		}
   654		if cpuList == nil {
   655			cpuList = append(cpuList, runtime.GOMAXPROCS(-1))
   656		}
   657	}
   658	

View as plain text