Run Format

Source file src/pkg/testing/testing.go

     1	// Copyright 2009 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// Package testing provides support for automated testing of Go packages.
     6	// It is intended to be used in concert with the ``go test'' command, which automates
     7	// execution of any function of the form
     8	//     func TestXxx(*testing.T)
     9	// where Xxx can be any alphanumeric string (but the first letter must not be in
    10	// [a-z]) and serves to identify the test routine.
    11	// These TestXxx routines should be declared within the package they are testing.
    12	//
    13	// Tests and benchmarks may be skipped if not applicable like this:
    14	//     func TestTimeConsuming(t *testing.T) {
    15	//         if testing.Short() {
    16	//             t.Skip("skipping test in short mode.")
    17	//         }
    18	//         ...
    19	//     }
    20	//
    21	// Benchmarks
    22	//
    23	// Functions of the form
    24	//     func BenchmarkXxx(*testing.B)
    25	// are considered benchmarks, and are executed by the "go test" command when
    26	// its -bench flag is provided. Benchmarks are run sequentially.
    27	//
    28	// For a description of the testing flags, see
    29	// http://golang.org/cmd/go/#hdr-Description_of_testing_flags.
    30	//
    31	// A sample benchmark function looks like this:
    32	//     func BenchmarkHello(b *testing.B) {
    33	//         for i := 0; i < b.N; i++ {
    34	//             fmt.Sprintf("hello")
    35	//         }
    36	//     }
    37	//
    38	// The benchmark function must run the target code b.N times.
    39	// The benchmark package will vary b.N until the benchmark function lasts
    40	// long enough to be timed reliably.  The output
    41	//     BenchmarkHello    10000000    282 ns/op
    42	// means that the loop ran 10000000 times at a speed of 282 ns per loop.
    43	//
    44	// If a benchmark needs some expensive setup before running, the timer
    45	// may be reset:
    46	//     func BenchmarkBigLen(b *testing.B) {
    47	//         big := NewBig()
    48	//         b.ResetTimer()
    49	//         for i := 0; i < b.N; i++ {
    50	//             big.Len()
    51	//         }
    52	//     }
    53	//
    54	// Examples
    55	//
    56	// The package also runs and verifies example code. Example functions may
    57	// include a concluding line comment that begins with "Output:" and is compared with
    58	// the standard output of the function when the tests are run. (The comparison
    59	// ignores leading and trailing space.) These are examples of an example:
    60	//
    61	//     func ExampleHello() {
    62	//             fmt.Println("hello")
    63	//             // Output: hello
    64	//     }
    65	//
    66	//     func ExampleSalutations() {
    67	//             fmt.Println("hello, and")
    68	//             fmt.Println("goodbye")
    69	//             // Output:
    70	//             // hello, and
    71	//             // goodbye
    72	//     }
    73	//
    74	// Example functions without output comments are compiled but not executed.
    75	//
    76	// The naming convention to declare examples for the package, a function F, a type T and
    77	// method M on type T are:
    78	//
    79	//     func Example() { ... }
    80	//     func ExampleF() { ... }
    81	//     func ExampleT() { ... }
    82	//     func ExampleT_M() { ... }
    83	//
    84	// Multiple example functions for a package/type/function/method may be provided by
    85	// appending a distinct suffix to the name. The suffix must start with a
    86	// lower-case letter.
    87	//
    88	//     func Example_suffix() { ... }
    89	//     func ExampleF_suffix() { ... }
    90	//     func ExampleT_suffix() { ... }
    91	//     func ExampleT_M_suffix() { ... }
    92	//
    93	// The entire test file is presented as the example when it contains a single
    94	// example function, at least one other function, type, variable, or constant
    95	// declaration, and no test or benchmark functions.
    96	package testing
    97	
    98	import (
    99		"bytes"
   100		"flag"
   101		"fmt"
   102		"os"
   103		"runtime"
   104		"runtime/pprof"
   105		"strconv"
   106		"strings"
   107		"sync"
   108		"time"
   109	)
   110	
   111	var (
   112		// The short flag requests that tests run more quickly, but its functionality
   113		// is provided by test writers themselves.  The testing package is just its
   114		// home.  The all.bash installation script sets it to make installation more
   115		// efficient, but by default the flag is off so a plain "go test" will do a
   116		// full test of the package.
   117		short = flag.Bool("test.short", false, "run smaller test suite to save time")
   118	
   119		// The directory in which to create profile files and the like. When run from
   120		// "go test", the binary always runs in the source directory for the package;
   121		// this flag lets "go test" tell the binary to write the files in the directory where
   122		// the "go test" command is run.
   123		outputDir = flag.String("test.outputdir", "", "directory in which to write profiles")
   124	
   125		// Report as tests are run; default is silent for success.
   126		chatty           = flag.Bool("test.v", false, "verbose: print additional output")
   127		coverProfile     = flag.String("test.coverprofile", "", "write a coverage profile to the named file after execution")
   128		match            = flag.String("test.run", "", "regular expression to select tests and examples to run")
   129		memProfile       = flag.String("test.memprofile", "", "write a memory profile to the named file after execution")
   130		memProfileRate   = flag.Int("test.memprofilerate", 0, "if >=0, sets runtime.MemProfileRate")
   131		cpuProfile       = flag.String("test.cpuprofile", "", "write a cpu profile to the named file during execution")
   132		blockProfile     = flag.String("test.blockprofile", "", "write a goroutine blocking profile to the named file after execution")
   133		blockProfileRate = flag.Int("test.blockprofilerate", 1, "if >= 0, calls runtime.SetBlockProfileRate()")
   134		timeout          = flag.Duration("test.timeout", 0, "if positive, sets an aggregate time limit for all tests")
   135		cpuListStr       = flag.String("test.cpu", "", "comma-separated list of number of CPUs to use for each test")
   136		parallel         = flag.Int("test.parallel", runtime.GOMAXPROCS(0), "maximum test parallelism")
   137	
   138		haveExamples bool // are there examples?
   139	
   140		cpuList []int
   141	)
   142	
   143	// common holds the elements common between T and B and
   144	// captures common methods such as Errorf.
   145	type common struct {
   146		mu      sync.RWMutex // guards output and failed
   147		output  []byte       // Output generated by test or benchmark.
   148		failed  bool         // Test or benchmark has failed.
   149		skipped bool         // Test of benchmark has been skipped.
   150	
   151		start    time.Time // Time test or benchmark started
   152		duration time.Duration
   153		self     interface{}      // To be sent on signal channel when done.
   154		signal   chan interface{} // Output for serial tests.
   155	}
   156	
   157	// Short reports whether the -test.short flag is set.
   158	func Short() bool {
   159		return *short
   160	}
   161	
   162	// Verbose reports whether the -test.v flag is set.
   163	func Verbose() bool {
   164		return *chatty
   165	}
   166	
   167	// decorate prefixes the string with the file and line of the call site
   168	// and inserts the final newline if needed and indentation tabs for formatting.
   169	func decorate(s string) string {
   170		_, file, line, ok := runtime.Caller(3) // decorate + log + public function.
   171		if ok {
   172			// Truncate file name at last file name separator.
   173			if index := strings.LastIndex(file, "/"); index >= 0 {
   174				file = file[index+1:]
   175			} else if index = strings.LastIndex(file, "\\"); index >= 0 {
   176				file = file[index+1:]
   177			}
   178		} else {
   179			file = "???"
   180			line = 1
   181		}
   182		buf := new(bytes.Buffer)
   183		// Every line is indented at least one tab.
   184		buf.WriteByte('\t')
   185		fmt.Fprintf(buf, "%s:%d: ", file, line)
   186		lines := strings.Split(s, "\n")
   187		if l := len(lines); l > 1 && lines[l-1] == "" {
   188			lines = lines[:l-1]
   189		}
   190		for i, line := range lines {
   191			if i > 0 {
   192				// Second and subsequent lines are indented an extra tab.
   193				buf.WriteString("\n\t\t")
   194			}
   195			buf.WriteString(line)
   196		}
   197		buf.WriteByte('\n')
   198		return buf.String()
   199	}
   200	
   201	// TB is the interface common to T and B.
   202	type TB interface {
   203		Error(args ...interface{})
   204		Errorf(format string, args ...interface{})
   205		Fail()
   206		FailNow()
   207		Failed() bool
   208		Fatal(args ...interface{})
   209		Fatalf(format string, args ...interface{})
   210		Log(args ...interface{})
   211		Logf(format string, args ...interface{})
   212		Skip(args ...interface{})
   213		SkipNow()
   214		Skipf(format string, args ...interface{})
   215		Skipped() bool
   216	
   217		// A private method to prevent users implementing the
   218		// interface and so future additions to it will not
   219		// violate Go 1 compatibility.
   220		private()
   221	}
   222	
   223	var _ TB = (*T)(nil)
   224	var _ TB = (*B)(nil)
   225	
   226	// T is a type passed to Test functions to manage test state and support formatted test logs.
   227	// Logs are accumulated during execution and dumped to standard error when done.
   228	type T struct {
   229		common
   230		name          string    // Name of test.
   231		startParallel chan bool // Parallel tests will wait on this.
   232	}
   233	
   234	func (c *common) private() {}
   235	
   236	// Fail marks the function as having failed but continues execution.
   237	func (c *common) Fail() {
   238		c.mu.Lock()
   239		defer c.mu.Unlock()
   240		c.failed = true
   241	}
   242	
   243	// Failed reports whether the function has failed.
   244	func (c *common) Failed() bool {
   245		c.mu.RLock()
   246		defer c.mu.RUnlock()
   247		return c.failed
   248	}
   249	
   250	// FailNow marks the function as having failed and stops its execution.
   251	// Execution will continue at the next test or benchmark.
   252	// FailNow must be called from the goroutine running the
   253	// test or benchmark function, not from other goroutines
   254	// created during the test. Calling FailNow does not stop
   255	// those other goroutines.
   256	func (c *common) FailNow() {
   257		c.Fail()
   258	
   259		// Calling runtime.Goexit will exit the goroutine, which
   260		// will run the deferred functions in this goroutine,
   261		// which will eventually run the deferred lines in tRunner,
   262		// which will signal to the test loop that this test is done.
   263		//
   264		// A previous version of this code said:
   265		//
   266		//	c.duration = ...
   267		//	c.signal <- c.self
   268		//	runtime.Goexit()
   269		//
   270		// This previous version duplicated code (those lines are in
   271		// tRunner no matter what), but worse the goroutine teardown
   272		// implicit in runtime.Goexit was not guaranteed to complete
   273		// before the test exited.  If a test deferred an important cleanup
   274		// function (like removing temporary files), there was no guarantee
   275		// it would run on a test failure.  Because we send on c.signal during
   276		// a top-of-stack deferred function now, we know that the send
   277		// only happens after any other stacked defers have completed.
   278		runtime.Goexit()
   279	}
   280	
   281	// log generates the output. It's always at the same stack depth.
   282	func (c *common) log(s string) {
   283		c.mu.Lock()
   284		defer c.mu.Unlock()
   285		c.output = append(c.output, decorate(s)...)
   286	}
   287	
   288	// Log formats its arguments using default formatting, analogous to Println,
   289	// and records the text in the error log. The text will be printed only if
   290	// the test fails or the -test.v flag is set.
   291	func (c *common) Log(args ...interface{}) { c.log(fmt.Sprintln(args...)) }
   292	
   293	// Logf formats its arguments according to the format, analogous to Printf,
   294	// and records the text in the error log. The text will be printed only if
   295	// the test fails or the -test.v flag is set.
   296	func (c *common) Logf(format string, args ...interface{}) { c.log(fmt.Sprintf(format, args...)) }
   297	
   298	// Error is equivalent to Log followed by Fail.
   299	func (c *common) Error(args ...interface{}) {
   300		c.log(fmt.Sprintln(args...))
   301		c.Fail()
   302	}
   303	
   304	// Errorf is equivalent to Logf followed by Fail.
   305	func (c *common) Errorf(format string, args ...interface{}) {
   306		c.log(fmt.Sprintf(format, args...))
   307		c.Fail()
   308	}
   309	
   310	// Fatal is equivalent to Log followed by FailNow.
   311	func (c *common) Fatal(args ...interface{}) {
   312		c.log(fmt.Sprintln(args...))
   313		c.FailNow()
   314	}
   315	
   316	// Fatalf is equivalent to Logf followed by FailNow.
   317	func (c *common) Fatalf(format string, args ...interface{}) {
   318		c.log(fmt.Sprintf(format, args...))
   319		c.FailNow()
   320	}
   321	
   322	// Skip is equivalent to Log followed by SkipNow.
   323	func (c *common) Skip(args ...interface{}) {
   324		c.log(fmt.Sprintln(args...))
   325		c.SkipNow()
   326	}
   327	
   328	// Skipf is equivalent to Logf followed by SkipNow.
   329	func (c *common) Skipf(format string, args ...interface{}) {
   330		c.log(fmt.Sprintf(format, args...))
   331		c.SkipNow()
   332	}
   333	
   334	// SkipNow marks the test as having been skipped and stops its execution.
   335	// Execution will continue at the next test or benchmark. See also FailNow.
   336	// SkipNow must be called from the goroutine running the test, not from
   337	// other goroutines created during the test. Calling SkipNow does not stop
   338	// those other goroutines.
   339	func (c *common) SkipNow() {
   340		c.skip()
   341		runtime.Goexit()
   342	}
   343	
   344	func (c *common) skip() {
   345		c.mu.Lock()
   346		defer c.mu.Unlock()
   347		c.skipped = true
   348	}
   349	
   350	// Skipped reports whether the test was skipped.
   351	func (c *common) Skipped() bool {
   352		c.mu.RLock()
   353		defer c.mu.RUnlock()
   354		return c.skipped
   355	}
   356	
   357	// Parallel signals that this test is to be run in parallel with (and only with)
   358	// other parallel tests.
   359	func (t *T) Parallel() {
   360		t.signal <- (*T)(nil) // Release main testing loop
   361		<-t.startParallel     // Wait for serial tests to finish
   362		// Assuming Parallel is the first thing a test does, which is reasonable,
   363		// reinitialize the test's start time because it's actually starting now.
   364		t.start = time.Now()
   365	}
   366	
   367	// An internal type but exported because it is cross-package; part of the implementation
   368	// of the "go test" command.
   369	type InternalTest struct {
   370		Name string
   371		F    func(*T)
   372	}
   373	
   374	func tRunner(t *T, test *InternalTest) {
   375		// When this goroutine is done, either because test.F(t)
   376		// returned normally or because a test failure triggered
   377		// a call to runtime.Goexit, record the duration and send
   378		// a signal saying that the test is done.
   379		defer func() {
   380			t.duration = time.Now().Sub(t.start)
   381			// If the test panicked, print any test output before dying.
   382			if err := recover(); err != nil {
   383				t.Fail()
   384				t.report()
   385				panic(err)
   386			}
   387			t.signal <- t
   388		}()
   389	
   390		t.start = time.Now()
   391		test.F(t)
   392	}
   393	
   394	// An internal function but exported because it is cross-package; part of the implementation
   395	// of the "go test" command.
   396	func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) {
   397		flag.Parse()
   398		parseCpuList()
   399	
   400		before()
   401		startAlarm()
   402		haveExamples = len(examples) > 0
   403		testOk := RunTests(matchString, tests)
   404		exampleOk := RunExamples(matchString, examples)
   405		stopAlarm()
   406		if !testOk || !exampleOk {
   407			fmt.Println("FAIL")
   408			os.Exit(1)
   409		}
   410		fmt.Println("PASS")
   411		RunBenchmarks(matchString, benchmarks)
   412		after()
   413	}
   414	
   415	func (t *T) report() {
   416		tstr := fmt.Sprintf("(%.2f seconds)", t.duration.Seconds())
   417		format := "--- %s: %s %s\n%s"
   418		if t.Failed() {
   419			fmt.Printf(format, "FAIL", t.name, tstr, t.output)
   420		} else if *chatty {
   421			if t.Skipped() {
   422				fmt.Printf(format, "SKIP", t.name, tstr, t.output)
   423			} else {
   424				fmt.Printf(format, "PASS", t.name, tstr, t.output)
   425			}
   426		}
   427	}
   428	
   429	func RunTests(matchString func(pat, str string) (bool, error), tests []InternalTest) (ok bool) {
   430		ok = true
   431		if len(tests) == 0 && !haveExamples {
   432			fmt.Fprintln(os.Stderr, "testing: warning: no tests to run")
   433			return
   434		}
   435		for _, procs := range cpuList {
   436			runtime.GOMAXPROCS(procs)
   437			// We build a new channel tree for each run of the loop.
   438			// collector merges in one channel all the upstream signals from parallel tests.
   439			// If all tests pump to the same channel, a bug can occur where a test
   440			// kicks off a goroutine that Fails, yet the test still delivers a completion signal,
   441			// which skews the counting.
   442			var collector = make(chan interface{})
   443	
   444			numParallel := 0
   445			startParallel := make(chan bool)
   446	
   447			for i := 0; i < len(tests); i++ {
   448				matched, err := matchString(*match, tests[i].Name)
   449				if err != nil {
   450					fmt.Fprintf(os.Stderr, "testing: invalid regexp for -test.run: %s\n", err)
   451					os.Exit(1)
   452				}
   453				if !matched {
   454					continue
   455				}
   456				testName := tests[i].Name
   457				if procs != 1 {
   458					testName = fmt.Sprintf("%s-%d", tests[i].Name, procs)
   459				}
   460				t := &T{
   461					common: common{
   462						signal: make(chan interface{}),
   463					},
   464					name:          testName,
   465					startParallel: startParallel,
   466				}
   467				t.self = t
   468				if *chatty {
   469					fmt.Printf("=== RUN %s\n", t.name)
   470				}
   471				go tRunner(t, &tests[i])
   472				out := (<-t.signal).(*T)
   473				if out == nil { // Parallel run.
   474					go func() {
   475						collector <- <-t.signal
   476					}()
   477					numParallel++
   478					continue
   479				}
   480				t.report()
   481				ok = ok && !out.Failed()
   482			}
   483	
   484			running := 0
   485			for numParallel+running > 0 {
   486				if running < *parallel && numParallel > 0 {
   487					startParallel <- true
   488					running++
   489					numParallel--
   490					continue
   491				}
   492				t := (<-collector).(*T)
   493				t.report()
   494				ok = ok && !t.Failed()
   495				running--
   496			}
   497		}
   498		return
   499	}
   500	
   501	// before runs before all testing.
   502	func before() {
   503		if *memProfileRate > 0 {
   504			runtime.MemProfileRate = *memProfileRate
   505		}
   506		if *cpuProfile != "" {
   507			f, err := os.Create(toOutputDir(*cpuProfile))
   508			if err != nil {
   509				fmt.Fprintf(os.Stderr, "testing: %s", err)
   510				return
   511			}
   512			if err := pprof.StartCPUProfile(f); err != nil {
   513				fmt.Fprintf(os.Stderr, "testing: can't start cpu profile: %s", err)
   514				f.Close()
   515				return
   516			}
   517			// Could save f so after can call f.Close; not worth the effort.
   518		}
   519		if *blockProfile != "" && *blockProfileRate >= 0 {
   520			runtime.SetBlockProfileRate(*blockProfileRate)
   521		}
   522		if *coverProfile != "" && cover.Mode == "" {
   523			fmt.Fprintf(os.Stderr, "testing: cannot use -test.coverprofile because test binary was not built with coverage enabled\n")
   524			os.Exit(2)
   525		}
   526	}
   527	
   528	// after runs after all testing.
   529	func after() {
   530		if *cpuProfile != "" {
   531			pprof.StopCPUProfile() // flushes profile to disk
   532		}
   533		if *memProfile != "" {
   534			f, err := os.Create(toOutputDir(*memProfile))
   535			if err != nil {
   536				fmt.Fprintf(os.Stderr, "testing: %s\n", err)
   537				os.Exit(2)
   538			}
   539			if err = pprof.WriteHeapProfile(f); err != nil {
   540				fmt.Fprintf(os.Stderr, "testing: can't write %s: %s\n", *memProfile, err)
   541				os.Exit(2)
   542			}
   543			f.Close()
   544		}
   545		if *blockProfile != "" && *blockProfileRate >= 0 {
   546			f, err := os.Create(toOutputDir(*blockProfile))
   547			if err != nil {
   548				fmt.Fprintf(os.Stderr, "testing: %s\n", err)
   549				os.Exit(2)
   550			}
   551			if err = pprof.Lookup("block").WriteTo(f, 0); err != nil {
   552				fmt.Fprintf(os.Stderr, "testing: can't write %s: %s\n", *blockProfile, err)
   553				os.Exit(2)
   554			}
   555			f.Close()
   556		}
   557		if cover.Mode != "" {
   558			coverReport()
   559		}
   560	}
   561	
   562	// toOutputDir returns the file name relocated, if required, to outputDir.
   563	// Simple implementation to avoid pulling in path/filepath.
   564	func toOutputDir(path string) string {
   565		if *outputDir == "" || path == "" {
   566			return path
   567		}
   568		if runtime.GOOS == "windows" {
   569			// On Windows, it's clumsy, but we can be almost always correct
   570			// by just looking for a drive letter and a colon.
   571			// Absolute paths always have a drive letter (ignoring UNC).
   572			// Problem: if path == "C:A" and outputdir == "C:\Go" it's unclear
   573			// what to do, but even then path/filepath doesn't help.
   574			// TODO: Worth doing better? Probably not, because we're here only
   575			// under the management of go test.
   576			if len(path) >= 2 {
   577				letter, colon := path[0], path[1]
   578				if ('a' <= letter && letter <= 'z' || 'A' <= letter && letter <= 'Z') && colon == ':' {
   579					// If path starts with a drive letter we're stuck with it regardless.
   580					return path
   581				}
   582			}
   583		}
   584		if os.IsPathSeparator(path[0]) {
   585			return path
   586		}
   587		return fmt.Sprintf("%s%c%s", *outputDir, os.PathSeparator, path)
   588	}
   589	
   590	var timer *time.Timer
   591	
   592	// startAlarm starts an alarm if requested.
   593	func startAlarm() {
   594		if *timeout > 0 {
   595			timer = time.AfterFunc(*timeout, func() {
   596				panic(fmt.Sprintf("test timed out after %v", *timeout))
   597			})
   598		}
   599	}
   600	
   601	// stopAlarm turns off the alarm.
   602	func stopAlarm() {
   603		if *timeout > 0 {
   604			timer.Stop()
   605		}
   606	}
   607	
   608	func parseCpuList() {
   609		for _, val := range strings.Split(*cpuListStr, ",") {
   610			val = strings.TrimSpace(val)
   611			if val == "" {
   612				continue
   613			}
   614			cpu, err := strconv.Atoi(val)
   615			if err != nil || cpu <= 0 {
   616				fmt.Fprintf(os.Stderr, "testing: invalid value %q for -test.cpu\n", val)
   617				os.Exit(1)
   618			}
   619			cpuList = append(cpuList, cpu)
   620		}
   621		if cpuList == nil {
   622			cpuList = append(cpuList, runtime.GOMAXPROCS(-1))
   623		}
   624	}

View as plain text