cmd/go, testing: add go test -skip flag

For proposal #41583, add a new 'go test -skip' flag to make it easy
to disable specific tests, benchmarks, examples, or fuzz targets.

Fixes #41583.

Change-Id: Id12a6575f505dafdce4a149aedc454a002e93afa
Reviewed-on: https://go-review.googlesource.com/c/go/+/421439
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: Bryan Mills <bcmills@google.com>

diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go
index f8800eef732e9dd47d711340d2acaa516cbec775..f8cc52343a41a236c8b0a6ede30be2ae34a6a992 100644 (file)
--- a/src/cmd/go/alldocs.go
+++ b/src/cmd/go/alldocs.go
@@ -2983,6 +2983,7 @@
 //         run too, so that -run=X/Y matches and runs and reports the result
 //         of all tests matching X, even those without sub-tests matching Y,
 //         because it must run them to look for those sub-tests.
+//         See also -skip.
 //
 //     -short
 //         Tell long-running tests to shorten their run time.
@@ -2997,6 +2998,14 @@
 //         integer N, then N will be used as the seed value. In both cases,
 //         the seed will be reported for reproducibility.
 //
+//     -skip regexp
+//         Run only those tests, examples, fuzz tests, and benchmarks that
+//         do not match the regular expression. Like for -run and -bench,
+//         for tests and benchmarks, the regular expression is split by unbracketed
+//         slash (/) characters into a sequence of regular expressions, and each
+//         part of a test's identifier must match the corresponding element in
+//         the sequence, if any.
+//
 //     -timeout d
 //         If a test binary runs longer than duration d, panic.
 //         If d is 0, the timeout is disabled.

diff --git a/src/cmd/go/internal/test/flagdefs.go b/src/cmd/go/internal/test/flagdefs.go
index 1b79314eff365889fa9b4b1cb9b221e91abfbd49..b91204ee93758d7fc65d4d42334cf5f8ec62e16b 100644 (file)
--- a/src/cmd/go/internal/test/flagdefs.go
+++ b/src/cmd/go/internal/test/flagdefs.go
@@ -32,6 +32,7 @@ var passFlagToTest = map[string]bool{
        "run":                  true,
        "short":                true,
        "shuffle":              true,
+       "skip":                 true,
        "timeout":              true,
        "trace":                true,
        "v":                    true,

diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go
index 7e6747055e04acfbc4407dd237bc0c01f6813e58..7248445796e522a35e5c80b4c53582dcde31ae63 100644 (file)
--- a/src/cmd/go/internal/test/test.go
+++ b/src/cmd/go/internal/test/test.go
@@ -306,6 +306,7 @@ control the execution of any test:
            run too, so that -run=X/Y matches and runs and reports the result
            of all tests matching X, even those without sub-tests matching Y,
            because it must run them to look for those sub-tests.
+           See also -skip.
 
        -short
            Tell long-running tests to shorten their run time.
@@ -320,6 +321,14 @@ control the execution of any test:
            integer N, then N will be used as the seed value. In both cases,
            the seed will be reported for reproducibility.
 
+       -skip regexp
+           Run only those tests, examples, fuzz tests, and benchmarks that
+           do not match the regular expression. Like for -run and -bench,
+           for tests and benchmarks, the regular expression is split by unbracketed
+           slash (/) characters into a sequence of regular expressions, and each
+           part of a test's identifier must match the corresponding element in
+           the sequence, if any.
+
        -timeout d
            If a test binary runs longer than duration d, panic.
            If d is 0, the timeout is disabled.

diff --git a/src/cmd/go/internal/test/testflag.go b/src/cmd/go/internal/test/testflag.go
index f3cd0b1392f00a733328b5f8330f78257fa34f01..8f5ab38d9de7785b11ee9d7c02c5a02d13ef55ec 100644 (file)
--- a/src/cmd/go/internal/test/testflag.go
+++ b/src/cmd/go/internal/test/testflag.go
@@ -66,6 +66,7 @@ func init() {
        cf.Int("parallel", 0, "")
        cf.String("run", "", "")
        cf.Bool("short", false, "")
+       cf.String("skip", "", "")
        cf.DurationVar(&testTimeout, "timeout", 10*time.Minute, "")
        cf.String("fuzztime", "", "")
        cf.String("fuzzminimizetime", "", "")

diff --git a/src/cmd/go/testdata/script/test_skip.txt b/src/cmd/go/testdata/script/test_skip.txt
new file mode 100644 (file)
index 0000000..94d20b9
--- /dev/null
+++ b/src/cmd/go/testdata/script/test_skip.txt
@@ -0,0 +1,34 @@
+go test -v -run Test -skip T skip_test.go
+! stdout RUN
+stdout '^ok.*\[no tests to run\]'
+
+go test -v -skip T skip_test.go
+! stdout RUN
+
+go test -v -skip 1 skip_test.go
+! stdout Test1
+stdout RUN.*Test2
+stdout RUN.*Test2/3
+
+go test -v -skip 2/3 skip_test.go
+stdout RUN.*Test1
+stdout RUN.*Test2
+! stdout Test2/3
+
+go test -v -skip 2/4 skip_test.go
+stdout RUN.*Test1
+stdout RUN.*Test2
+stdout RUN.*Test2/3
+
+
+-- skip_test.go --
+package skip_test
+
+import "testing"
+
+func Test1(t *testing.T) {
+}
+
+func Test2(t *testing.T) {
+       t.Run("3", func(t *testing.T) {})
+}

diff --git a/src/testing/benchmark.go b/src/testing/benchmark.go
index 2f7936611f7df6b79e67b885bd2522a4c714b4b8..7ee517604ba3fe7239a03745718393df5274c5e7 100644 (file)
--- a/src/testing/benchmark.go
+++ b/src/testing/benchmark.go
@@ -536,7 +536,7 @@ func runBenchmarks(importPath string, matchString func(pat, str string) (bool, e
                }
        }
        ctx := &benchContext{
-               match:  newMatcher(matchString, *matchBenchmarks, "-test.bench"),
+               match:  newMatcher(matchString, *matchBenchmarks, "-test.bench", *skip),
                extLen: len(benchmarkName("", maxprocs)),
        }
        var bs []InternalBenchmark

diff --git a/src/testing/fuzz.go b/src/testing/fuzz.go
index 87b60fc1bbe228cf481f3ee9b0880b06d9126c18..d885f44b32dfb96a792585a237316bf260fe48f7 100644 (file)
--- a/src/testing/fuzz.go
+++ b/src/testing/fuzz.go
@@ -471,12 +471,12 @@ func runFuzzTests(deps testDeps, fuzzTests []InternalFuzzTarget, deadline time.T
        if len(fuzzTests) == 0 || *isFuzzWorker {
                return ran, ok
        }
-       m := newMatcher(deps.MatchString, *match, "-test.run")
+       m := newMatcher(deps.MatchString, *match, "-test.run", *skip)
        tctx := newTestContext(*parallel, m)
        tctx.deadline = deadline
        var mFuzz *matcher
        if *matchFuzz != "" {
-               mFuzz = newMatcher(deps.MatchString, *matchFuzz, "-test.fuzz")
+               mFuzz = newMatcher(deps.MatchString, *matchFuzz, "-test.fuzz", *skip)
        }
        fctx := &fuzzContext{deps: deps, mode: seedCorpusOnly}
        root := common{w: os.Stdout} // gather output in one place
@@ -532,7 +532,7 @@ func runFuzzing(deps testDeps, fuzzTests []InternalFuzzTarget) (ok bool) {
        if len(fuzzTests) == 0 || *matchFuzz == "" {
                return true
        }
-       m := newMatcher(deps.MatchString, *matchFuzz, "-test.fuzz")
+       m := newMatcher(deps.MatchString, *matchFuzz, "-test.fuzz", *skip)
        tctx := newTestContext(1, m)
        tctx.isFuzzing = true
        fctx := &fuzzContext{

diff --git a/src/testing/helper_test.go b/src/testing/helper_test.go
index fa1d2b608252b4dc0813aba53c5abe991b16d975..6e8986a2ab1bf6b1ee3c439077f03995e47e8067 100644 (file)
--- a/src/testing/helper_test.go
+++ b/src/testing/helper_test.go
@@ -11,7 +11,7 @@ import (
 
 func TestTBHelper(t *T) {
        var buf strings.Builder
-       ctx := newTestContext(1, newMatcher(regexp.MatchString, "", ""))
+       ctx := newTestContext(1, allMatcher())
        t1 := &T{
                common: common{
                        signal: make(chan bool),
@@ -55,7 +55,7 @@ helperfuncs_test.go:67: 10
 
 func TestTBHelperParallel(t *T) {
        var buf strings.Builder
-       ctx := newTestContext(1, newMatcher(regexp.MatchString, "", ""))
+       ctx := newTestContext(1, newMatcher(regexp.MatchString, "", "", ""))
        t1 := &T{
                common: common{
                        signal: make(chan bool),
@@ -81,7 +81,7 @@ func (nw *noopWriter) Write(b []byte) (int, error) { return len(b), nil }
 
 func BenchmarkTBHelper(b *B) {
        w := noopWriter(0)
-       ctx := newTestContext(1, newMatcher(regexp.MatchString, "", ""))
+       ctx := newTestContext(1, allMatcher())
        t1 := &T{
                common: common{
                        signal: make(chan bool),

diff --git a/src/testing/match.go b/src/testing/match.go
index d530f70c2676c61c2b43ec56aff204f67e20fcea..92b7dc622de2dc88c42cd5374bb19cffa6da4be3 100644 (file)
--- a/src/testing/match.go
+++ b/src/testing/match.go
@@ -15,6 +15,7 @@ import (
 // matcher sanitizes, uniques, and filters names of subtests and subbenchmarks.
 type matcher struct {
        filter    filterMatch
+       skip      filterMatch
        matchFunc func(pat, str string) (bool, error)
 
        mu sync.Mutex
@@ -47,17 +48,33 @@ type alternationMatch []filterMatch
 // eliminate this Mutex.
 var matchMutex sync.Mutex
 
-func newMatcher(matchString func(pat, str string) (bool, error), patterns, name string) *matcher {
-       var impl filterMatch
-       if patterns != "" {
-               impl = splitRegexp(patterns)
-               if err := impl.verify(name, matchString); err != nil {
+func allMatcher() *matcher {
+       return newMatcher(nil, "", "", "")
+}
+
+func newMatcher(matchString func(pat, str string) (bool, error), patterns, name, skips string) *matcher {
+       var filter, skip filterMatch
+       if patterns == "" {
+               filter = simpleMatch{} // always partial true
+       } else {
+               filter = splitRegexp(patterns)
+               if err := filter.verify(name, matchString); err != nil {
                        fmt.Fprintf(os.Stderr, "testing: invalid regexp for %s\n", err)
                        os.Exit(1)
                }
        }
+       if skips == "" {
+               skip = alternationMatch{} // always false
+       } else {
+               skip = splitRegexp(skips)
+               if err := skip.verify("-test.skip", matchString); err != nil {
+                       fmt.Fprintf(os.Stderr, "testing: invalid regexp for %v\n", err)
+                       os.Exit(1)
+               }
+       }
        return &matcher{
-               filter:    impl,
+               filter:    filter,
+               skip:      skip,
                matchFunc: matchString,
                subNames:  map[string]int32{},
        }
@@ -76,14 +93,23 @@ func (m *matcher) fullName(c *common, subname string) (name string, ok, partial
        matchMutex.Lock()
        defer matchMutex.Unlock()
 
-       if m.filter == nil {
-               return name, true, false
-       }
-
-       // We check the full array of paths each time to allow for the case that
-       // a pattern contains a '/'.
+       // We check the full array of paths each time to allow for the case that a pattern contains a '/'.
        elem := strings.Split(name, "/")
+
+       // filter must match.
+       // accept partial match that may produce full match later.
        ok, partial = m.filter.matches(elem, m.matchFunc)
+       if !ok {
+               return name, false, false
+       }
+
+       // skip must not match.
+       // ignore partial match so we can get to more precise match later.
+       skip, partialSkip := m.skip.matches(elem, m.matchFunc)
+       if skip && !partialSkip {
+               return name, false, false
+       }
+
        return name, ok, partial
 }
 

diff --git a/src/testing/match_test.go b/src/testing/match_test.go
index 206ac0b651f4082c3b45380468894341873a0125..d31efbc95e6803742aaca39e3a825936791e6b5a 100644 (file)
--- a/src/testing/match_test.go
+++ b/src/testing/match_test.go
@@ -12,6 +12,10 @@ import (
        "unicode"
 )
 
+func init() {
+       testingTesting = true
+}
+
 // Verify that our IsSpace agrees with unicode.IsSpace.
 func TestIsSpace(t *T) {
        n := 0
@@ -89,54 +93,75 @@ func TestSplitRegexp(t *T) {
 func TestMatcher(t *T) {
        testCases := []struct {
                pattern     string
+               skip        string
                parent, sub string
                ok          bool
                partial     bool
        }{
                // Behavior without subtests.
-               {"", "", "TestFoo", true, false},
-               {"TestFoo", "", "TestFoo", true, false},
-               {"TestFoo/", "", "TestFoo", true, true},
-               {"TestFoo/bar/baz", "", "TestFoo", true, true},
-               {"TestFoo", "", "TestBar", false, false},
-               {"TestFoo/", "", "TestBar", false, false},
-               {"TestFoo/bar/baz", "", "TestBar/bar/baz", false, false},
+               {"", "", "", "TestFoo", true, false},
+               {"TestFoo", "", "", "TestFoo", true, false},
+               {"TestFoo/", "", "", "TestFoo", true, true},
+               {"TestFoo/bar/baz", "", "", "TestFoo", true, true},
+               {"TestFoo", "", "", "TestBar", false, false},
+               {"TestFoo/", "", "", "TestBar", false, false},
+               {"TestFoo/bar/baz", "", "", "TestBar/bar/baz", false, false},
+               {"", "TestBar", "", "TestFoo", true, false},
+               {"", "TestBar", "", "TestBar", false, false},
+
+               // Skipping a non-existent test doesn't change anything.
+               {"", "TestFoo/skipped", "", "TestFoo", true, false},
+               {"TestFoo", "TestFoo/skipped", "", "TestFoo", true, false},
+               {"TestFoo/", "TestFoo/skipped", "", "TestFoo", true, true},
+               {"TestFoo/bar/baz", "TestFoo/skipped", "", "TestFoo", true, true},
+               {"TestFoo", "TestFoo/skipped", "", "TestBar", false, false},
+               {"TestFoo/", "TestFoo/skipped", "", "TestBar", false, false},
+               {"TestFoo/bar/baz", "TestFoo/skipped", "", "TestBar/bar/baz", false, false},
 
                // with subtests
-               {"", "TestFoo", "x", true, false},
-               {"TestFoo", "TestFoo", "x", true, false},
-               {"TestFoo/", "TestFoo", "x", true, false},
-               {"TestFoo/bar/baz", "TestFoo", "bar", true, true},
+               {"", "", "TestFoo", "x", true, false},
+               {"TestFoo", "", "TestFoo", "x", true, false},
+               {"TestFoo/", "", "TestFoo", "x", true, false},
+               {"TestFoo/bar/baz", "", "TestFoo", "bar", true, true},
+
+               {"", "TestFoo/skipped", "TestFoo", "x", true, false},
+               {"TestFoo", "TestFoo/skipped", "TestFoo", "x", true, false},
+               {"TestFoo", "TestFoo/skipped", "TestFoo", "skipped", false, false},
+               {"TestFoo/", "TestFoo/skipped", "TestFoo", "x", true, false},
+               {"TestFoo/bar/baz", "TestFoo/skipped", "TestFoo", "bar", true, true},
+
                // Subtest with a '/' in its name still allows for copy and pasted names
                // to match.
-               {"TestFoo/bar/baz", "TestFoo", "bar/baz", true, false},
-               {"TestFoo/bar/baz", "TestFoo/bar", "baz", true, false},
-               {"TestFoo/bar/baz", "TestFoo", "x", false, false},
-               {"TestFoo", "TestBar", "x", false, false},
-               {"TestFoo/", "TestBar", "x", false, false},
-               {"TestFoo/bar/baz", "TestBar", "x/bar/baz", false, false},
+               {"TestFoo/bar/baz", "", "TestFoo", "bar/baz", true, false},
+               {"TestFoo/bar/baz", "TestFoo/bar/baz", "TestFoo", "bar/baz", false, false},
+               {"TestFoo/bar/baz", "TestFoo/bar/baz/skip", "TestFoo", "bar/baz", true, false},
+               {"TestFoo/bar/baz", "", "TestFoo/bar", "baz", true, false},
+               {"TestFoo/bar/baz", "", "TestFoo", "x", false, false},
+               {"TestFoo", "", "TestBar", "x", false, false},
+               {"TestFoo/", "", "TestBar", "x", false, false},
+               {"TestFoo/bar/baz", "", "TestBar", "x/bar/baz", false, false},
 
-               {"A/B|C/D", "TestA", "B", true, false},
-               {"A/B|C/D", "TestC", "D", true, false},
-               {"A/B|C/D", "TestA", "C", false, false},
+               {"A/B|C/D", "", "TestA", "B", true, false},
+               {"A/B|C/D", "", "TestC", "D", true, false},
+               {"A/B|C/D", "", "TestA", "C", false, false},
 
                // subtests only
-               {"", "TestFoo", "x", true, false},
-               {"/", "TestFoo", "x", true, false},
-               {"./", "TestFoo", "x", true, false},
-               {"./.", "TestFoo", "x", true, false},
-               {"/bar/baz", "TestFoo", "bar", true, true},
-               {"/bar/baz", "TestFoo", "bar/baz", true, false},
-               {"//baz", "TestFoo", "bar/baz", true, false},
-               {"//", "TestFoo", "bar/baz", true, false},
-               {"/bar/baz", "TestFoo/bar", "baz", true, false},
-               {"//foo", "TestFoo", "bar/baz", false, false},
-               {"/bar/baz", "TestFoo", "x", false, false},
-               {"/bar/baz", "TestBar", "x/bar/baz", false, false},
+               {"", "", "TestFoo", "x", true, false},
+               {"/", "", "TestFoo", "x", true, false},
+               {"./", "", "TestFoo", "x", true, false},
+               {"./.", "", "TestFoo", "x", true, false},
+               {"/bar/baz", "", "TestFoo", "bar", true, true},
+               {"/bar/baz", "", "TestFoo", "bar/baz", true, false},
+               {"//baz", "", "TestFoo", "bar/baz", true, false},
+               {"//", "", "TestFoo", "bar/baz", true, false},
+               {"/bar/baz", "", "TestFoo/bar", "baz", true, false},
+               {"//foo", "", "TestFoo", "bar/baz", false, false},
+               {"/bar/baz", "", "TestFoo", "x", false, false},
+               {"/bar/baz", "", "TestBar", "x/bar/baz", false, false},
        }
 
        for _, tc := range testCases {
-               m := newMatcher(regexp.MatchString, tc.pattern, "-test.run")
+               m := newMatcher(regexp.MatchString, tc.pattern, "-test.run", tc.skip)
 
                parent := &common{name: tc.parent}
                if tc.parent != "" {
@@ -184,7 +209,7 @@ var namingTestCases = []struct{ name, want string }{
 }
 
 func TestNaming(t *T) {
-       m := newMatcher(regexp.MatchString, "", "")
+       m := newMatcher(regexp.MatchString, "", "", "")
        parent := &common{name: "x", level: 1} // top-level test.
 
        for i, tc := range namingTestCases {
@@ -202,7 +227,7 @@ func FuzzNaming(f *F) {
        var m *matcher
        var seen map[string]string
        reset := func() {
-               m = newMatcher(regexp.MatchString, "", "")
+               m = allMatcher()
                seen = make(map[string]string)
        }
        reset()

diff --git a/src/testing/sub_test.go b/src/testing/sub_test.go
index 6d8badfbf8194a68578443b7d87982aa0fe6dec5..7d6b97b66e465db9fe687de7709cf8c6d2d46392 100644 (file)
--- a/src/testing/sub_test.go
+++ b/src/testing/sub_test.go
@@ -476,7 +476,7 @@ func TestTRun(t *T) {
        }}
        for _, tc := range testCases {
                t.Run(tc.desc, func(t *T) {
-                       ctx := newTestContext(tc.maxPar, newMatcher(regexp.MatchString, "", ""))
+                       ctx := newTestContext(tc.maxPar, allMatcher())
                        buf := &strings.Builder{}
                        root := &T{
                                common: common{
@@ -775,7 +775,7 @@ func TestRacyOutput(t *T) {
        var wg sync.WaitGroup
        root := &T{
                common:  common{w: &funcWriter{raceDetector}},
-               context: newTestContext(1, newMatcher(regexp.MatchString, "", "")),
+               context: newTestContext(1, allMatcher()),
        }
        root.chatty = newChattyPrinter(root.w)
        root.Run("", func(t *T) {
@@ -798,7 +798,7 @@ func TestRacyOutput(t *T) {
 
 // The late log message did not include the test name.  Issue 29388.
 func TestLogAfterComplete(t *T) {
-       ctx := newTestContext(1, newMatcher(regexp.MatchString, "", ""))
+       ctx := newTestContext(1, allMatcher())
        var buf bytes.Buffer
        t1 := &T{
                common: common{

diff --git a/src/testing/testing.go b/src/testing/testing.go
index 0228d2904b2c585193876b01c5db36ffaa9c10cd..e3460e049da28193e81df3410eb1ca7525be0b85 100644 (file)
--- a/src/testing/testing.go
+++ b/src/testing/testing.go
@@ -422,6 +422,7 @@ func Init() {
        coverProfile = flag.String("test.coverprofile", "", "write a coverage profile to `file`")
        matchList = flag.String("test.list", "", "list tests, examples, and benchmarks matching `regexp` then exit")
        match = flag.String("test.run", "", "run only tests and examples matching `regexp`")
+       skip = flag.String("test.skip", "", "do not list or run tests matching `regexp`")
        memProfile = flag.String("test.memprofile", "", "write an allocation profile to `file`")
        memProfileRate = flag.Int("test.memprofilerate", 0, "set memory allocation profiling `rate` (see runtime.MemProfileRate)")
        cpuProfile = flag.String("test.cpuprofile", "", "write a cpu profile to `file`")
@@ -451,6 +452,7 @@ var (
        coverProfile         *string
        matchList            *string
        match                *string
+       skip                 *string
        memProfile           *string
        memProfileRate       *int
        cpuProfile           *string
@@ -1690,6 +1692,8 @@ func MainStart(deps testDeps, tests []InternalTest, benchmarks []InternalBenchma
        }
 }
 
+var testingTesting bool
+
 // Run runs the tests. It returns an exit code to pass to os.Exit.
 func (m *M) Run() (code int) {
        defer func() {
@@ -1720,7 +1724,7 @@ func (m *M) Run() (code int) {
                return
        }
 
-       if len(*matchList) != 0 {
+       if *matchList != "" {
                listTests(m.deps.MatchString, m.tests, m.benchmarks, m.fuzzTargets, m.examples)
                m.exitCode = 0
                return
@@ -1762,6 +1766,15 @@ func (m *M) Run() (code int) {
                m.stopAlarm()
                if !testRan && !exampleRan && !fuzzTargetsRan && *matchBenchmarks == "" && *matchFuzz == "" {
                        fmt.Fprintln(os.Stderr, "testing: warning: no tests to run")
+                       if testingTesting {
+                               // If this happens during testing of package testing it could be that
+                               // package testing's own logic for when to run a test is broken,
+                               // in which case every test will run nothing and succeed,
+                               // with no obvious way to detect this problem (since no tests are running).
+                               // So make 'no tests to run' a hard failure when testing package testing itself.
+                               fmt.Println("FAIL: package testing must run tests")
+                               testOk = false
+                       }
                }
                if !testOk || !exampleOk || !fuzzTargetsOk || !runBenchmarks(m.deps.ImportPath(), m.deps.MatchString, m.benchmarks) || race.Errors() > 0 {
                        fmt.Println("FAIL")
@@ -1861,7 +1874,7 @@ func runTests(matchString func(pat, str string) (bool, error), tests []InternalT
                                // to keep trying.
                                break
                        }
-                       ctx := newTestContext(*parallel, newMatcher(matchString, *match, "-test.run"))
+                       ctx := newTestContext(*parallel, newMatcher(matchString, *match, "-test.run", *skip))
                        ctx.deadline = deadline
                        t := &T{
                                common: common{