1 // Copyright 2020 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
20 func initFuzzFlags() {
21 matchFuzz = flag.String("test.fuzz", "", "run the fuzz test matching `regexp`")
22 flag.Var(&fuzzDuration, "test.fuzztime", "time to spend fuzzing; default is to run indefinitely")
23 flag.Var(&minimizeDuration, "test.fuzzminimizetime", "time to spend minimizing a value after finding a failing input")
25 fuzzCacheDir = flag.String("test.fuzzcachedir", "", "directory where interesting fuzzing inputs are stored (for use only by cmd/go)")
26 isFuzzWorker = flag.Bool("test.fuzzworker", false, "coordinate with the parent process to fuzz random values (for use only by cmd/go)")
31 fuzzDuration durationOrCountFlag
32 minimizeDuration = durationOrCountFlag{d: 60 * time.Second, allowZero: true}
36 // corpusDir is the parent directory of the fuzz test's seed corpus within
38 corpusDir = "testdata/fuzz"
41 // fuzzWorkerExitCode is used as an exit code by fuzz worker processes after an
42 // internal error. This distinguishes internal errors from uncontrolled panics
43 // and other failures. Keep in sync with internal/fuzz.workerExitCode.
44 const fuzzWorkerExitCode = 70
46 // InternalFuzzTarget is an internal type but exported because it is
47 // cross-package; it is part of the implementation of the "go test" command.
48 type InternalFuzzTarget struct {
53 // F is a type passed to fuzz tests.
55 // Fuzz tests run generated inputs against a provided fuzz target, which can
56 // find and report potential bugs in the code being tested.
58 // A fuzz test runs the seed corpus by default, which includes entries provided
59 // by (*F).Add and entries in the testdata/fuzz/<FuzzTestName> directory. After
60 // any necessary setup and calls to (*F).Add, the fuzz test must then call
61 // (*F).Fuzz to provide the fuzz target. See the testing package documentation
62 // for an example, and see the F.Fuzz and F.Add method documentation for
65 // *F methods can only be called before (*F).Fuzz. Once the test is
66 // executing the fuzz target, only (*T) methods can be used. The only *F methods
67 // that are allowed in the (*F).Fuzz function are (*F).Failed and (*F).Name.
70 fuzzContext *fuzzContext
71 testContext *testContext
73 // inFuzzFn is true when the fuzz function is running. Most F methods cannot
74 // be called when inFuzzFn is true.
77 // corpus is a set of seed corpus entries, added with F.Add and loaded
87 // corpusEntry is an alias to the same type as internal/fuzz.CorpusEntry.
88 // We use a type alias because we don't want to export this type, and we can't
89 // import internal/fuzz from testing.
90 type corpusEntry = struct {
99 // Helper marks the calling function as a test helper function.
100 // When printing file and line information, that function will be skipped.
101 // Helper may be called simultaneously from multiple goroutines.
102 func (f *F) Helper() {
104 panic("testing: f.Helper was called inside the fuzz target, use t.Helper instead")
107 // common.Helper is inlined here.
108 // If we called it, it would mark F.Helper as the helper
109 // instead of the caller.
112 if f.helperPCs == nil {
113 f.helperPCs = make(map[uintptr]struct{})
115 // repeating code from callerName here to save walking a stack frame
117 n := runtime.Callers(2, pc[:]) // skip runtime.Callers + Helper
119 panic("testing: zero callers found")
121 if _, found := f.helperPCs[pc[0]]; !found {
122 f.helperPCs[pc[0]] = struct{}{}
123 f.helperNames = nil // map will be recreated next time it is needed
127 // Fail marks the function as having failed but continues execution.
129 // (*F).Fail may be called by (*T).Fail, which we should allow. However, we
130 // shouldn't allow direct (*F).Fail calls from inside the (*F).Fuzz function.
132 panic("testing: f.Fail was called inside the fuzz target, use t.Fail instead")
138 // Skipped reports whether the test was skipped.
139 func (f *F) Skipped() bool {
140 // (*F).Skipped may be called by tRunner, which we should allow. However, we
141 // shouldn't allow direct (*F).Skipped calls from inside the (*F).Fuzz function.
143 panic("testing: f.Skipped was called inside the fuzz target, use t.Skipped instead")
146 return f.common.Skipped()
149 // Add will add the arguments to the seed corpus for the fuzz test. This will be
150 // a no-op if called after or within the fuzz target, and args must match the
151 // arguments for the fuzz target.
152 func (f *F) Add(args ...any) {
154 for i := range args {
155 if t := reflect.TypeOf(args[i]); !supportedTypes[t] {
156 panic(fmt.Sprintf("testing: unsupported type to Add %v", t))
158 values = append(values, args[i])
160 f.corpus = append(f.corpus, corpusEntry{Values: values, IsSeed: true, Path: fmt.Sprintf("seed#%d", len(f.corpus))})
163 // supportedTypes represents all of the supported types which can be fuzzed.
164 var supportedTypes = map[reflect.Type]bool{
165 reflect.TypeOf(([]byte)("")): true,
166 reflect.TypeOf((string)("")): true,
167 reflect.TypeOf((bool)(false)): true,
168 reflect.TypeOf((byte)(0)): true,
169 reflect.TypeOf((rune)(0)): true,
170 reflect.TypeOf((float32)(0)): true,
171 reflect.TypeOf((float64)(0)): true,
172 reflect.TypeOf((int)(0)): true,
173 reflect.TypeOf((int8)(0)): true,
174 reflect.TypeOf((int16)(0)): true,
175 reflect.TypeOf((int32)(0)): true,
176 reflect.TypeOf((int64)(0)): true,
177 reflect.TypeOf((uint)(0)): true,
178 reflect.TypeOf((uint8)(0)): true,
179 reflect.TypeOf((uint16)(0)): true,
180 reflect.TypeOf((uint32)(0)): true,
181 reflect.TypeOf((uint64)(0)): true,
184 // Fuzz runs the fuzz function, ff, for fuzz testing. If ff fails for a set of
185 // arguments, those arguments will be added to the seed corpus.
187 // ff must be a function with no return value whose first argument is *T and
188 // whose remaining arguments are the types to be fuzzed.
191 // f.Fuzz(func(t *testing.T, b []byte, i int) { ... })
193 // The following types are allowed: []byte, string, bool, byte, rune, float32,
194 // float64, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64.
195 // More types may be supported in the future.
197 // ff must not call any *F methods, e.g. (*F).Log, (*F).Error, (*F).Skip. Use
198 // the corresponding *T method instead. The only *F methods that are allowed in
199 // the (*F).Fuzz function are (*F).Failed and (*F).Name.
201 // This function should be fast and deterministic, and its behavior should not
202 // depend on shared state. No mutatable input arguments, or pointers to them,
203 // should be retained between executions of the fuzz function, as the memory
204 // backing them may be mutated during a subsequent invocation. ff must not
205 // modify the underlying data of the arguments provided by the fuzzing engine.
207 // When fuzzing, F.Fuzz does not return until a problem is found, time runs out
208 // (set with -fuzztime), or the test process is interrupted by a signal. F.Fuzz
209 // should be called exactly once, unless F.Skip or F.Fail is called beforehand.
210 func (f *F) Fuzz(ff any) {
212 panic("testing: F.Fuzz called more than once")
220 // ff should be in the form func(*testing.T, ...interface{})
221 fn := reflect.ValueOf(ff)
223 if fnType.Kind() != reflect.Func {
224 panic("testing: F.Fuzz must receive a function")
226 if fnType.NumIn() < 2 || fnType.In(0) != reflect.TypeOf((*T)(nil)) {
227 panic("testing: fuzz target must receive at least two arguments, where the first argument is a *T")
229 if fnType.NumOut() != 0 {
230 panic("testing: fuzz target must not return a value")
233 // Save the types of the function to compare against the corpus.
234 var types []reflect.Type
235 for i := 1; i < fnType.NumIn(); i++ {
237 if !supportedTypes[t] {
238 panic(fmt.Sprintf("testing: unsupported type for fuzzing %v", t))
240 types = append(types, t)
243 // Load the testdata seed corpus. Check types of entries in the testdata
244 // corpus and entries declared with F.Add.
246 // Don't load the seed corpus if this is a worker process; we won't use it.
247 if f.fuzzContext.mode != fuzzWorker {
248 for _, c := range f.corpus {
249 if err := f.fuzzContext.deps.CheckCorpus(c.Values, types); err != nil {
250 // TODO(#48302): Report the source location of the F.Add call.
256 c, err := f.fuzzContext.deps.ReadCorpus(filepath.Join(corpusDir, f.name), types)
261 c[i].IsSeed = true // these are all seed corpus values
262 if f.fuzzContext.mode == fuzzCoordinator {
263 // If this is the coordinator process, zero the values, since we don't need
264 // to hold onto them.
269 f.corpus = append(f.corpus, c...)
272 // run calls fn on a given input, as a subtest with its own T.
273 // run is analogous to T.Run. The test filtering and cleanup works similarly.
274 // fn is called in its own goroutine.
275 run := func(captureOut io.Writer, e corpusEntry) (ok bool) {
277 // The corpusEntry must have non-nil Values in order to run the
278 // test. If Values is nil, it is a bug in our code.
279 panic(fmt.Sprintf("corpus file %q was not unmarshaled", e.Path))
281 if shouldFailFast() {
286 testName = fmt.Sprintf("%s/%s", testName, filepath.Base(e.Path))
288 if f.testContext.isFuzzing {
289 // Don't preserve subtest names while fuzzing. If fn calls T.Run,
290 // there will be a very large number of subtests with duplicate names,
291 // which will use a large amount of memory. The subtest names aren't
292 // useful since there's no way to re-run them deterministically.
293 f.testContext.match.clearSubNames()
296 // Record the stack trace at the point of this call so that if the subtest
297 // function - which runs in a separate stack - is marked as a helper, we can
298 // continue walking the stack into the parent test.
299 var pc [maxStackLen]uintptr
300 n := runtime.Callers(2, pc[:])
303 barrier: make(chan bool),
304 signal: make(chan bool),
311 context: f.testContext,
313 if captureOut != nil {
314 // t.parent aliases f.common.
315 t.parent.w = captureOut
317 t.w = indenter{&t.common}
319 t.chatty.Updatef(t.name, "=== RUN %s\n", t.name)
321 f.common.inFuzzFn, f.inFuzzFn = true, true
322 go tRunner(t, func(t *T) {
323 args := []reflect.Value{reflect.ValueOf(t)}
324 for _, v := range e.Values {
325 args = append(args, reflect.ValueOf(v))
327 // Before resetting the current coverage, defer the snapshot so that
328 // we make sure it is called right before the tRunner function
329 // exits, regardless of whether it was executed cleanly, panicked,
330 // or if the fuzzFn called t.Fatal.
331 if f.testContext.isFuzzing {
332 defer f.fuzzContext.deps.SnapshotCoverage()
333 f.fuzzContext.deps.ResetCoverage()
338 if t.chatty != nil && t.chatty.json {
339 t.chatty.Updatef(t.parent.name, "=== NAME %s\n", t.parent.name)
341 f.common.inFuzzFn, f.inFuzzFn = false, false
345 switch f.fuzzContext.mode {
346 case fuzzCoordinator:
347 // Fuzzing is enabled, and this is the test process started by 'go test'.
348 // Act as the coordinator process, and coordinate workers to perform the
350 corpusTargetDir := filepath.Join(corpusDir, f.name)
351 cacheTargetDir := filepath.Join(*fuzzCacheDir, f.name)
352 err := f.fuzzContext.deps.CoordinateFuzzing(
354 int64(fuzzDuration.n),
356 int64(minimizeDuration.n),
363 f.result = fuzzResult{Error: err}
365 fmt.Fprintf(f.w, "%v\n", err)
366 if crashErr, ok := err.(fuzzCrashError); ok {
367 crashPath := crashErr.CrashPath()
368 fmt.Fprintf(f.w, "Failing input written to %s\n", crashPath)
369 testName := filepath.Base(crashPath)
370 fmt.Fprintf(f.w, "To re-run:\ngo test -run=%s/%s\n", f.name, testName)
373 // TODO(jayconrod,katiehockman): Aggregate statistics across workers
374 // and add to FuzzResult (ie. time taken, num iterations)
377 // Fuzzing is enabled, and this is a worker process. Follow instructions
378 // from the coordinator.
379 if err := f.fuzzContext.deps.RunFuzzWorker(func(e corpusEntry) error {
380 // Don't write to f.w (which points to Stdout) if running from a
381 // fuzz worker. This would become very verbose, particularly during
382 // minimization. Return the error instead, and let the caller deal
384 var buf strings.Builder
385 if ok := run(&buf, e); !ok {
386 return errors.New(buf.String())
390 // Internal errors are marked with f.Fail; user code may call this too, before F.Fuzz.
391 // The worker will exit with fuzzWorkerExitCode, indicating this is a failure
392 // (and 'go test' should exit non-zero) but a failing input should not be recorded.
393 f.Errorf("communicating with fuzzing coordinator: %v", err)
397 // Fuzzing is not enabled, or will be done later. Only run the seed
399 for _, e := range f.corpus {
400 name := fmt.Sprintf("%s/%s", f.name, filepath.Base(e.Path))
401 if _, ok, _ := f.testContext.match.fullName(nil, name); ok {
408 func (f *F) report() {
409 if *isFuzzWorker || f.parent == nil {
412 dstr := fmtDuration(f.duration)
413 format := "--- %s: %s (%s)\n"
415 f.flushToParent(f.name, format, "FAIL", f.name, dstr)
416 } else if f.chatty != nil {
418 f.flushToParent(f.name, format, "SKIP", f.name, dstr)
420 f.flushToParent(f.name, format, "PASS", f.name, dstr)
425 // fuzzResult contains the results of a fuzz run.
426 type fuzzResult struct {
427 N int // The number of iterations.
428 T time.Duration // The total time taken.
429 Error error // Error is the error from the failing input
432 func (r fuzzResult) String() string {
436 return r.Error.Error()
439 // fuzzCrashError is satisfied by a failing input detected while fuzzing.
440 // These errors are written to the seed corpus and can be re-run with 'go test'.
441 // Errors within the fuzzing framework (like I/O errors between coordinator
442 // and worker processes) don't satisfy this interface.
443 type fuzzCrashError interface {
447 // CrashPath returns the path of the subtest that corresponds to the saved
448 // crash input file in the seed corpus. The test can be re-run with go test
449 // -run=$test/$name $test is the fuzz test name, and $name is the
450 // filepath.Base of the string returned here.
454 // fuzzContext holds fields common to all fuzz tests.
455 type fuzzContext struct {
463 seedCorpusOnly fuzzMode = iota
468 // runFuzzTests runs the fuzz tests matching the pattern for -run. This will
469 // only run the (*F).Fuzz function for each seed corpus without using the
470 // fuzzing engine to generate or mutate inputs.
471 func runFuzzTests(deps testDeps, fuzzTests []InternalFuzzTarget, deadline time.Time) (ran, ok bool) {
473 if len(fuzzTests) == 0 || *isFuzzWorker {
476 m := newMatcher(deps.MatchString, *match, "-test.run", *skip)
478 if *matchFuzz != "" {
479 mFuzz = newMatcher(deps.MatchString, *matchFuzz, "-test.fuzz", *skip)
482 for _, procs := range cpuList {
483 runtime.GOMAXPROCS(procs)
484 for i := uint(0); i < *count; i++ {
485 if shouldFailFast() {
489 tctx := newTestContext(*parallel, m)
490 tctx.deadline = deadline
491 fctx := &fuzzContext{deps: deps, mode: seedCorpusOnly}
492 root := common{w: os.Stdout} // gather output in one place
494 root.chatty = newChattyPrinter(root.w)
496 for _, ft := range fuzzTests {
497 if shouldFailFast() {
500 testName, matched, _ := tctx.match.fullName(nil, ft.Name)
505 if _, fuzzMatched, _ := mFuzz.fullName(nil, ft.Name); fuzzMatched {
506 // If this will be fuzzed, then don't run the seed corpus
507 // right now. That will happen later.
513 signal: make(chan bool),
514 barrier: make(chan bool),
517 level: root.level + 1,
523 f.w = indenter{&f.common}
525 f.chatty.Updatef(f.name, "=== RUN %s\n", f.name)
529 if f.chatty != nil && f.chatty.json {
530 f.chatty.Updatef(f.parent.name, "=== NAME %s\n", f.parent.name)
532 ok = ok && !f.Failed()
536 // There were no tests to run on this iteration.
537 // This won't change, so no reason to keep trying.
546 // runFuzzing runs the fuzz test matching the pattern for -fuzz. Only one such
547 // fuzz test must match. This will run the fuzzing engine to generate and
548 // mutate new inputs against the fuzz target.
550 // If fuzzing is disabled (-test.fuzz is not set), runFuzzing
551 // returns immediately.
552 func runFuzzing(deps testDeps, fuzzTests []InternalFuzzTarget) (ok bool) {
553 if len(fuzzTests) == 0 || *matchFuzz == "" {
556 m := newMatcher(deps.MatchString, *matchFuzz, "-test.fuzz", *skip)
557 tctx := newTestContext(1, m)
558 tctx.isFuzzing = true
559 fctx := &fuzzContext{
562 root := common{w: os.Stdout}
565 fctx.mode = fuzzWorker
567 fctx.mode = fuzzCoordinator
569 if Verbose() && !*isFuzzWorker {
570 root.chatty = newChattyPrinter(root.w)
572 var fuzzTest *InternalFuzzTarget
575 for i := range fuzzTests {
576 name, ok, _ := tctx.match.fullName(nil, fuzzTests[i].Name)
580 matched = append(matched, name)
581 fuzzTest = &fuzzTests[i]
584 if len(matched) == 0 {
585 fmt.Fprintln(os.Stderr, "testing: warning: no fuzz tests to fuzz")
588 if len(matched) > 1 {
589 fmt.Fprintf(os.Stderr, "testing: will not fuzz, -fuzz matches more than one fuzz test: %v\n", matched)
595 signal: make(chan bool),
596 barrier: nil, // T.Parallel has no effect when fuzzing.
599 level: root.level + 1,
605 f.w = indenter{&f.common}
607 f.chatty.Updatef(f.name, "=== RUN %s\n", f.name)
609 go fRunner(f, fuzzTest.Fn)
612 f.chatty.Updatef(f.parent.name, "=== NAME %s\n", f.parent.name)
617 // fRunner wraps a call to a fuzz test and ensures that cleanup functions are
618 // called and status flags are set. fRunner should be called in its own
619 // goroutine. To wait for its completion, receive from f.signal.
621 // fRunner is analogous to tRunner, which wraps subtests started with T.Run.
622 // Unit tests and fuzz tests work a little differently, so for now, these
623 // functions aren't consolidated. In particular, because there are no F.Run and
624 // F.Parallel methods, i.e., no fuzz sub-tests or parallel fuzz tests, a few
625 // simplifications are made. We also require that F.Fuzz, F.Skip, or F.Fail is
627 func fRunner(f *F, fn func(*F)) {
628 // When this goroutine is done, either because runtime.Goexit was called, a
629 // panic started, or fn returned normally, record the duration and send
630 // t.signal, indicating the fuzz test is done.
632 // Detect whether the fuzz test panicked or called runtime.Goexit
633 // without calling F.Fuzz, F.Fail, or F.Skip. If it did, panic (possibly
634 // replacing a nil panic value). Nothing should recover after fRunner
635 // unwinds, so this should crash the process and print stack.
636 // Unfortunately, recovering here adds stack frames, but the location of
637 // the original panic should still be
645 fuzzNotCalled := !f.fuzzCalled && !f.skipped && !f.failed
646 if !f.finished && !f.skipped && !f.failed {
647 err = errNilPanicOrGoexit
650 if fuzzNotCalled && err == nil {
651 f.Error("returned without calling F.Fuzz, F.Fail, or F.Skip")
655 // Use a deferred call to ensure that we report that the test is
656 // complete even if a cleanup function calls F.FailNow. See issue 41355.
660 // Only report that the test is complete if it doesn't panic,
661 // as otherwise the test binary can exit before the panic is
662 // reported to the user. See issue 41479.
667 // If we recovered a panic or inappropriate runtime.Goexit, fail the test,
668 // flush the output log up to the root, then panic.
669 doPanic := func(err any) {
671 if r := f.runCleanup(recoverAndReturnPanic); r != nil {
672 f.Logf("cleanup panicked with %v", r)
674 for root := &f.common; root.parent != nil; root = root.parent {
676 root.duration += time.Since(root.start)
679 root.flushToParent(root.name, "--- FAIL: %s (%s)\n", root.name, fmtDuration(d))
688 // No panic or inappropriate Goexit.
689 f.duration += time.Since(f.start)
692 // Unblock inputs that called T.Parallel while running the seed corpus.
693 // This only affects fuzz tests run as normal tests.
694 // While fuzzing, T.Parallel has no effect, so f.sub is empty, and this
695 // branch is not taken. f.barrier is nil in that case.
696 f.testContext.release()
698 // Wait for the subtests to complete.
699 for _, sub := range f.sub {
702 cleanupStart := time.Now()
703 err := f.runCleanup(recoverAndReturnPanic)
704 f.duration += time.Since(cleanupStart)
710 // Report after all subtests have finished.
717 f.runCleanup(normalPanic)
724 // Code beyond this point will not be executed when FailNow or SkipNow