1 # Shuffle order of tests and benchmarks
2
3 [short] skip 'builds and repeatedly runs a test binary'
4
5 # Run tests
6 go test -v foo_test.go
7 ! stdout '-test.shuffle '
8 stdout '(?s)TestOne(.*)TestTwo(.*)TestThree'
9
10 go test -v -shuffle=off foo_test.go
11 ! stdout '-test.shuffle '
12 stdout '(?s)TestOne(.*)TestTwo(.*)TestThree'
13
14 go test -v -shuffle=42 foo_test.go
15 stdout '^-test.shuffle 42'
16 stdout '(?s)TestThree(.*)TestOne(.*)TestTwo'
17
18 go test -v -shuffle=0 foo_test.go
19 stdout '^-test.shuffle 0'
20 stdout '(?s)TestTwo(.*)TestOne(.*)TestThree'
21
22 go test -v -shuffle -1 foo_test.go
23 stdout '^-test.shuffle -1'
24 stdout '(?s)TestThree(.*)TestOne(.*)TestTwo'
25
26 go test -v -shuffle=on foo_test.go
27 stdout '^-test.shuffle '
28 stdout '(?s)=== RUN TestOne(.*)--- PASS: TestOne'
29 stdout '(?s)=== RUN TestTwo(.*)--- PASS: TestTwo'
30 stdout '(?s)=== RUN TestThree(.*)--- PASS: TestThree'
31
32
33 # Run tests and benchmarks
34 go test -v -bench=. foo_test.go
35 ! stdout '-test.shuffle '
36 stdout '(?s)TestOne(.*)TestTwo(.*)TestThree(.*)BenchmarkOne(.*)BenchmarkTwo(.*)BenchmarkThree'
37
38 go test -v -bench=. -shuffle=off foo_test.go
39 ! stdout '-test.shuffle '
40 stdout '(?s)TestOne(.*)TestTwo(.*)TestThree(.*)BenchmarkOne(.*)BenchmarkTwo(.*)BenchmarkThree'
41
42 go test -v -bench=. -shuffle=42 foo_test.go
43 stdout '^-test.shuffle 42'
44 stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo'
45
46 go test -v -bench=. -shuffle=0 foo_test.go
47 stdout '^-test.shuffle 0'
48 stdout '(?s)TestTwo(.*)TestOne(.*)TestThree(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo'
49
50 go test -v -bench=. -shuffle -1 foo_test.go
51 stdout '^-test.shuffle -1'
52 stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)BenchmarkTwo'
53
54 go test -v -bench=. -shuffle=on foo_test.go
55 stdout '^-test.shuffle '
56 stdout '(?s)=== RUN TestOne(.*)--- PASS: TestOne'
57 stdout '(?s)=== RUN TestTwo(.*)--- PASS: TestTwo'
58 stdout '(?s)=== RUN TestThree(.*)--- PASS: TestThree'
59 stdout -count=2 'BenchmarkOne'
60 stdout -count=2 'BenchmarkTwo'
61 stdout -count=2 'BenchmarkThree'
62
63
64 # When running go test -count=N, each of the N runs distinct runs should maintain the same
65 # shuffled order of these tests.
66 go test -v -shuffle=43 -count=4 foo_test.go
67 stdout '^-test.shuffle 43'
68 stdout '(?s)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne'
69
70 go test -v -bench=. -shuffle=44 -count=2 foo_test.go
71 stdout '^-test.shuffle 44'
72 stdout '(?s)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)BenchmarkTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)'
73
74
75 # The feature should work with test binaries as well
76 go test -c
77 exec ./m.test -test.shuffle=off
78 ! stdout '^-test.shuffle '
79
80 exec ./m.test -test.shuffle=on
81 stdout '^-test.shuffle '
82
83 exec ./m.test -test.v -test.bench=. -test.shuffle=0 foo_test.go
84 stdout '^-test.shuffle 0'
85 stdout '(?s)TestTwo(.*)TestOne(.*)TestThree(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo'
86
87 exec ./m.test -test.v -test.bench=. -test.shuffle=123 foo_test.go
88 stdout '^-test.shuffle 123'
89 stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkThree(.*)BenchmarkTwo(.*)BenchmarkOne'
90
91 exec ./m.test -test.v -test.bench=. -test.shuffle=-1 foo_test.go
92 stdout '^-test.shuffle -1'
93 stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)BenchmarkTwo'
94
95 exec ./m.test -test.v -test.bench=. -test.shuffle=44 -test.count=2 foo_test.go
96 stdout '^-test.shuffle 44'
97 stdout '(?s)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)BenchmarkTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)'
98
99
100 # Negative testcases for invalid input
101 ! go test -shuffle -count=2
102 stderr 'invalid value "-count=2" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "-count=2": invalid syntax'
103
104 ! go test -shuffle=
105 stderr '(?s)invalid value "" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "": invalid syntax'
106
107 ! go test -shuffle=' '
108 stderr '(?s)invalid value " " for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing " ": invalid syntax'
109
110 ! go test -shuffle=true
111 stderr 'invalid value "true" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "true": invalid syntax'
112
113 ! go test -shuffle='abc'
114 stderr 'invalid value "abc" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "abc": invalid syntax'
115
116 -- go.mod --
117 module m
118
119 go 1.16
120 -- foo_test.go --
121 package foo
122
123 import "testing"
124
125 func TestOne(t *testing.T) {}
126 func TestTwo(t *testing.T) {}
127 func TestThree(t *testing.T) {}
128
129 func BenchmarkOne(b *testing.B) {}
130 func BenchmarkTwo(b *testing.B) {}
131 func BenchmarkThree(b *testing.B) {}
132
133 -- foo.go --
134 package foo
135
View as plain text