# Shuffle order of tests and benchmarks

[short] skip 'builds and repeatedly runs a test binary'

# Run tests
go test -v foo_test.go
! stdout '-test.shuffle '
stdout '(?s)TestOne(.*)TestTwo(.*)TestThree'

go test -v -shuffle=off foo_test.go
! stdout '-test.shuffle '
stdout '(?s)TestOne(.*)TestTwo(.*)TestThree'

go test -v -shuffle=42 foo_test.go
stdout '^-test.shuffle 42'
stdout '(?s)TestThree(.*)TestOne(.*)TestTwo'

go test -v -shuffle=0 foo_test.go
stdout '^-test.shuffle 0'
stdout '(?s)TestTwo(.*)TestOne(.*)TestThree'

go test -v -shuffle -1 foo_test.go
stdout '^-test.shuffle -1'
stdout '(?s)TestThree(.*)TestOne(.*)TestTwo'

go test -v -shuffle=on foo_test.go
stdout '^-test.shuffle '
stdout '(?s)=== RUN   TestOne(.*)--- PASS: TestOne'
stdout '(?s)=== RUN   TestTwo(.*)--- PASS: TestTwo'
stdout '(?s)=== RUN   TestThree(.*)--- PASS: TestThree'


# Run tests and benchmarks
go test -v -bench=. foo_test.go
! stdout '-test.shuffle '
stdout '(?s)TestOne(.*)TestTwo(.*)TestThree(.*)BenchmarkOne(.*)BenchmarkTwo(.*)BenchmarkThree'

go test -v -bench=. -shuffle=off foo_test.go
! stdout '-test.shuffle '
stdout '(?s)TestOne(.*)TestTwo(.*)TestThree(.*)BenchmarkOne(.*)BenchmarkTwo(.*)BenchmarkThree'

go test -v -bench=. -shuffle=42 foo_test.go
stdout '^-test.shuffle 42'
stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo'

go test -v -bench=. -shuffle=0 foo_test.go
stdout '^-test.shuffle 0'
stdout '(?s)TestTwo(.*)TestOne(.*)TestThree(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo'

go test -v -bench=. -shuffle -1 foo_test.go
stdout '^-test.shuffle -1'
stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)BenchmarkTwo'

go test -v -bench=. -shuffle=on foo_test.go
stdout '^-test.shuffle '
stdout '(?s)=== RUN   TestOne(.*)--- PASS: TestOne'
stdout '(?s)=== RUN   TestTwo(.*)--- PASS: TestTwo'
stdout '(?s)=== RUN   TestThree(.*)--- PASS: TestThree'
stdout -count=2 'BenchmarkOne'
stdout -count=2 'BenchmarkTwo'
stdout -count=2 'BenchmarkThree'


# When running go test -count=N, each of the N runs distinct runs should maintain the same
# shuffled order of these tests.
go test -v -shuffle=43 -count=4 foo_test.go
stdout '^-test.shuffle 43'
stdout '(?s)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne'

go test -v -bench=. -shuffle=44 -count=2 foo_test.go
stdout '^-test.shuffle 44'
stdout '(?s)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)BenchmarkTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)'


# The feature should work with test binaries as well
go test -c
exec ./m.test -test.shuffle=off
! stdout '^-test.shuffle '

exec ./m.test -test.shuffle=on
stdout '^-test.shuffle '

exec ./m.test -test.v -test.bench=. -test.shuffle=0 foo_test.go
stdout '^-test.shuffle 0'
stdout '(?s)TestTwo(.*)TestOne(.*)TestThree(.*)BenchmarkThree(.*)BenchmarkOne(.*)BenchmarkTwo'

exec ./m.test -test.v -test.bench=. -test.shuffle=123 foo_test.go
stdout '^-test.shuffle 123'
stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkThree(.*)BenchmarkTwo(.*)BenchmarkOne'

exec ./m.test -test.v -test.bench=. -test.shuffle=-1 foo_test.go
stdout '^-test.shuffle -1'
stdout '(?s)TestThree(.*)TestOne(.*)TestTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)BenchmarkTwo'

exec ./m.test -test.v -test.bench=. -test.shuffle=44 -test.count=2 foo_test.go
stdout '^-test.shuffle 44'
stdout '(?s)TestOne(.*)TestThree(.*)TestTwo(.*)TestOne(.*)TestThree(.*)TestTwo(.*)BenchmarkTwo(.*)BenchmarkOne(.*)BenchmarkThree(.*)'


# Negative testcases for invalid input
! go test -shuffle -count=2
stderr 'invalid value "-count=2" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "-count=2": invalid syntax'

! go test -shuffle=
stderr '(?s)invalid value "" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "": invalid syntax'

! go test -shuffle=' '
stderr '(?s)invalid value " " for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing " ": invalid syntax'

! go test -shuffle=true
stderr 'invalid value "true" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "true": invalid syntax'

! go test -shuffle='abc'
stderr 'invalid value "abc" for flag -shuffle: -shuffle argument must be "on", "off", or an int64: strconv.ParseInt: parsing "abc": invalid syntax'

-- go.mod --
module m

go 1.16
-- foo_test.go --
package foo

import "testing"

func TestOne(t *testing.T)   {}
func TestTwo(t *testing.T)   {}
func TestThree(t *testing.T) {}

func BenchmarkOne(b *testing.B)   {}
func BenchmarkTwo(b *testing.B)   {}
func BenchmarkThree(b *testing.B) {}

-- foo.go --
package foo