1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
29 saveTraces = flag.Bool("savetraces", false, "save traces collected by tests")
32 // TestEventBatch tests Flush calls that happen during Start
33 // don't produce corrupted traces.
34 func TestEventBatch(t *testing.T) {
36 t.Skip("skipping in race mode")
39 t.Skip("skipping because -test.trace is set")
42 t.Skip("skipping in short mode")
44 // During Start, bunch of records are written to reflect the current
45 // snapshot of the program, including state of each goroutines.
46 // And some string constants are written to the trace to aid trace
47 // parsing. This test checks Flush of the buffer occurred during
48 // this process doesn't cause corrupted traces.
49 // When a Flush is called during Start is complicated
50 // so we test with a range of number of goroutines hoping that one
51 // of them triggers Flush.
52 // This range was chosen to fill up a ~64KB buffer with traceEvGoCreate
53 // and traceEvGoWaiting events (12~13bytes per goroutine).
54 for g := 4950; g < 5050; g++ {
56 t.Run("G="+strconv.Itoa(n), func(t *testing.T) {
60 in := make(chan bool, 1000)
61 for i := 0; i < n; i++ {
67 buf := new(bytes.Buffer)
68 if err := Start(buf); err != nil {
69 t.Fatalf("failed to start tracing: %v", err)
72 for i := 0; i < n; i++ {
78 _, err := trace.Parse(buf, "")
79 if err == trace.ErrTimeOrder {
80 t.Skipf("skipping trace: %v", err)
84 t.Fatalf("failed to parse trace: %v", err)
90 func TestTraceStartStop(t *testing.T) {
92 t.Skip("skipping because -test.trace is set")
94 buf := new(bytes.Buffer)
95 if err := Start(buf); err != nil {
96 t.Fatalf("failed to start tracing: %v", err)
101 t.Fatalf("trace is empty")
103 time.Sleep(100 * time.Millisecond)
104 if size != buf.Len() {
105 t.Fatalf("trace writes after stop: %v -> %v", size, buf.Len())
107 saveTrace(t, buf, "TestTraceStartStop")
110 func TestTraceDoubleStart(t *testing.T) {
112 t.Skip("skipping because -test.trace is set")
115 buf := new(bytes.Buffer)
116 if err := Start(buf); err != nil {
117 t.Fatalf("failed to start tracing: %v", err)
119 if err := Start(buf); err == nil {
120 t.Fatalf("succeed to start tracing second time")
126 func TestTrace(t *testing.T) {
128 t.Skip("skipping because -test.trace is set")
130 buf := new(bytes.Buffer)
131 if err := Start(buf); err != nil {
132 t.Fatalf("failed to start tracing: %v", err)
135 saveTrace(t, buf, "TestTrace")
136 _, err := trace.Parse(buf, "")
137 if err == trace.ErrTimeOrder {
138 t.Skipf("skipping trace: %v", err)
141 t.Fatalf("failed to parse trace: %v", err)
145 func parseTrace(t *testing.T, r io.Reader) ([]*trace.Event, map[uint64]*trace.GDesc) {
146 res, err := trace.Parse(r, "")
147 if err == trace.ErrTimeOrder {
148 t.Skipf("skipping trace: %v", err)
151 t.Fatalf("failed to parse trace: %v", err)
153 gs := trace.GoroutineStats(res.Events)
154 for goid := range gs {
155 // We don't do any particular checks on the result at the moment.
156 // But still check that RelatedGoroutines does not crash, hang, etc.
157 _ = trace.RelatedGoroutines(res.Events, goid)
159 return res.Events, gs
162 func testBrokenTimestamps(t *testing.T, data []byte) {
163 // On some processors cputicks (used to generate trace timestamps)
164 // produce non-monotonic timestamps. It is important that the parser
165 // distinguishes logically inconsistent traces (e.g. missing, excessive
166 // or misordered events) from broken timestamps. The former is a bug
167 // in tracer, the latter is a machine issue.
168 // So now that we have a consistent trace, test that (1) parser does
169 // not return a logical error in case of broken timestamps
170 // and (2) broken timestamps are eventually detected and reported.
171 trace.BreakTimestampsForTesting = true
173 trace.BreakTimestampsForTesting = false
175 for i := 0; i < 1e4; i++ {
176 _, err := trace.Parse(bytes.NewReader(data), "")
177 if err == trace.ErrTimeOrder {
181 t.Fatalf("failed to parse trace: %v", err)
186 func TestTraceStress(t *testing.T) {
187 if runtime.GOOS == "js" {
188 t.Skip("no os.Pipe on js")
191 t.Skip("skipping because -test.trace is set")
194 t.Skip("skipping in -short mode")
197 var wg sync.WaitGroup
198 done := make(chan bool)
200 // Create a goroutine blocked before tracing.
207 // Create a goroutine blocked in syscall before tracing.
208 rp, wp, err := os.Pipe()
210 t.Fatalf("failed to create pipe: %v", err)
223 time.Sleep(time.Millisecond) // give the goroutine above time to block
225 buf := new(bytes.Buffer)
226 if err := Start(buf); err != nil {
227 t.Fatalf("failed to start tracing: %v", err)
230 procs := runtime.GOMAXPROCS(10)
231 time.Sleep(50 * time.Millisecond) // test proc stop/start events
234 runtime.LockOSThread()
246 // Trigger GC from malloc.
248 if isMemoryConstrained() {
249 // Reduce allocation to avoid running out of
250 // memory on the builder - see issue/12032.
253 for i := 0; i < n; i++ {
254 _ = make([]byte, 1<<20)
257 // Create a bunch of busy goroutines to load all Ps.
258 for p := 0; p < 10; p++ {
261 // Do something useful.
262 tmp := make([]byte, 1<<16)
282 timerDone := make(chan bool)
284 time.Sleep(time.Millisecond)
290 ln, err := net.Listen("tcp", "127.0.0.1:0")
292 t.Fatalf("listen failed: %v", err)
296 c, err := ln.Accept()
300 time.Sleep(time.Millisecond)
305 c, err := net.Dial("tcp", ln.Addr().String())
307 t.Fatalf("dial failed: %v", err)
318 // Unblock helper goroutines and wait them to finish.
324 runtime.GOMAXPROCS(procs)
327 saveTrace(t, buf, "TestTraceStress")
330 testBrokenTimestamps(t, trace)
333 // isMemoryConstrained reports whether the current machine is likely
334 // to be memory constrained.
335 // This was originally for the openbsd/arm builder (Issue 12032).
336 // TODO: move this to testenv? Make this look at memory? Look at GO_BUILDER_NAME?
337 func isMemoryConstrained() bool {
338 if runtime.GOOS == "plan9" {
341 switch runtime.GOARCH {
342 case "arm", "mips", "mipsle":
348 // Do a bunch of various stuff (timers, GC, network, etc) in a separate goroutine.
349 // And concurrently with all that start/stop trace 3 times.
350 func TestTraceStressStartStop(t *testing.T) {
351 if runtime.GOOS == "js" {
352 t.Skip("no os.Pipe on js")
355 t.Skip("skipping because -test.trace is set")
357 defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(8))
358 outerDone := make(chan bool)
365 var wg sync.WaitGroup
366 done := make(chan bool)
374 rp, wp, err := os.Pipe()
376 t.Errorf("failed to create pipe: %v", err)
390 time.Sleep(time.Millisecond)
393 runtime.LockOSThread()
405 // Trigger GC from malloc.
407 if isMemoryConstrained() {
408 // Reduce allocation to avoid running out of
409 // memory on the builder.
412 for i := 0; i < n; i++ {
413 _ = make([]byte, 1<<20)
416 // Create a bunch of busy goroutines to load all Ps.
417 for p := 0; p < 10; p++ {
420 // Do something useful.
421 tmp := make([]byte, 1<<16)
440 runtime.GOMAXPROCS(runtime.GOMAXPROCS(1))
443 timerDone := make(chan bool)
445 time.Sleep(time.Millisecond)
451 ln, err := net.Listen("tcp", "127.0.0.1:0")
453 t.Errorf("listen failed: %v", err)
458 c, err := ln.Accept()
462 time.Sleep(time.Millisecond)
467 c, err := net.Dial("tcp", ln.Addr().String())
469 t.Errorf("dial failed: %v", err)
481 // Unblock helper goroutines and wait them to finish.
488 for i := 0; i < 3; i++ {
489 buf := new(bytes.Buffer)
490 if err := Start(buf); err != nil {
491 t.Fatalf("failed to start tracing: %v", err)
493 time.Sleep(time.Millisecond)
495 saveTrace(t, buf, "TestTraceStressStartStop")
498 testBrokenTimestamps(t, trace)
503 func TestTraceFutileWakeup(t *testing.T) {
505 t.Skip("skipping because -test.trace is set")
507 buf := new(bytes.Buffer)
508 if err := Start(buf); err != nil {
509 t.Fatalf("failed to start tracing: %v", err)
512 defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(8))
513 c0 := make(chan int, 1)
514 c1 := make(chan int, 1)
515 c2 := make(chan int, 1)
517 var done sync.WaitGroup
519 for p := 0; p < procs; p++ {
522 for i := 0; i < iters; i++ {
529 for i := 0; i < iters; i++ {
536 for i := 0; i < iters; i++ {
546 for i := 0; i < iters; i++ {
559 saveTrace(t, buf, "TestTraceFutileWakeup")
560 events, _ := parseTrace(t, buf)
561 // Check that (1) trace does not contain EvFutileWakeup events and
562 // (2) there are no consecutive EvGoBlock/EvGCStart/EvGoBlock events
563 // (we call runtime.Gosched between all operations, so these would be futile wakeups).
564 gs := make(map[uint64]int)
565 for _, ev := range events {
567 case trace.EvFutileWakeup:
568 t.Fatalf("found EvFutileWakeup event")
569 case trace.EvGoBlockSend, trace.EvGoBlockRecv, trace.EvGoBlockSelect:
571 t.Fatalf("goroutine %v blocked on %v at %v right after start",
572 ev.G, trace.EventDescriptions[ev.Type].Name, ev.Ts)
575 t.Fatalf("goroutine %v blocked on %v at %v while blocked",
576 ev.G, trace.EventDescriptions[ev.Type].Name, ev.Ts)
579 case trace.EvGoStart:
589 func TestTraceCPUProfile(t *testing.T) {
591 t.Skip("skipping because -test.trace is set")
594 cpuBuf := new(bytes.Buffer)
595 if err := pprof.StartCPUProfile(cpuBuf); err != nil {
596 t.Skipf("failed to start CPU profile: %v", err)
599 buf := new(bytes.Buffer)
600 if err := Start(buf); err != nil {
601 t.Fatalf("failed to start tracing: %v", err)
604 dur := 100 * time.Millisecond
606 // Create a region in the execution trace. Set and clear goroutine
607 // labels fully within that region, so we know that any CPU profile
608 // sample with the label must also be eligible for inclusion in the
610 ctx := context.Background()
611 defer StartRegion(ctx, "cpuHogger").End()
612 pprof.Do(ctx, pprof.Labels("tracing", "on"), func(ctx context.Context) {
613 cpuHogger(cpuHog1, &salt1, dur)
615 // Be sure the execution trace's view, when filtered to this goroutine
616 // via the explicit goroutine ID in each event, gets many more samples
617 // than the CPU profiler when filtered to this goroutine via labels.
618 cpuHogger(cpuHog1, &salt1, dur)
622 pprof.StopCPUProfile()
623 saveTrace(t, buf, "TestTraceCPUProfile")
625 prof, err := profile.Parse(cpuBuf)
627 t.Fatalf("failed to parse CPU profile: %v", err)
629 // Examine the CPU profiler's view. Filter it to only include samples from
630 // the single test goroutine. Use labels to execute that filter: they should
631 // apply to all work done while that goroutine is getg().m.curg, and they
632 // should apply to no other goroutines.
634 pprofStacks := make(map[string]int)
635 for _, s := range prof.Sample {
636 if s.Label["tracing"] != nil {
637 samples := int(s.Value[0])
638 pprofSamples += samples
640 for _, loc := range s.Location {
641 for _, line := range loc.Line {
642 fns = append(fns, fmt.Sprintf("%s:%d", line.Function.Name, line.Line))
645 stack := strings.Join(fns, " ")
646 pprofStacks[stack] += samples
649 if pprofSamples == 0 {
650 t.Skipf("CPU profile did not include any samples while tracing was active\n%s", prof)
653 // Examine the execution tracer's view of the CPU profile samples. Filter it
654 // to only include samples from the single test goroutine. Use the goroutine
655 // ID that was recorded in the events: that should reflect getg().m.curg,
656 // same as the profiler's labels (even when the M is using its g0 stack).
657 totalTraceSamples := 0
659 traceStacks := make(map[string]int)
660 events, _ := parseTrace(t, buf)
661 var hogRegion *trace.Event
662 for _, ev := range events {
663 if ev.Type == trace.EvUserRegion && ev.Args[1] == 0 && ev.SArgs[0] == "cpuHogger" {
664 // mode "0" indicates region start
668 if hogRegion == nil {
669 t.Fatalf("execution trace did not identify cpuHogger goroutine")
670 } else if hogRegion.Link == nil {
671 t.Fatalf("execution trace did not close cpuHogger region")
673 for _, ev := range events {
674 if ev.Type == trace.EvCPUSample {
676 if ev.G == hogRegion.G {
679 for _, frame := range ev.Stk {
680 if frame.Fn != "runtime.goexit" {
681 fns = append(fns, fmt.Sprintf("%s:%d", frame.Fn, frame.Line))
684 stack := strings.Join(fns, " ")
690 // The execution trace may drop CPU profile samples if the profiling buffer
691 // overflows. Based on the size of profBufWordCount, that takes a bit over
692 // 1900 CPU samples or 19 thread-seconds at a 100 Hz sample rate. If we've
693 // hit that case, then we definitely have at least one full buffer's worth
694 // of CPU samples, so we'll call that success.
695 overflowed := totalTraceSamples >= 1900
696 if traceSamples < pprofSamples {
697 t.Logf("exectution trace did not include all CPU profile samples; %d in profile, %d in trace", pprofSamples, traceSamples)
703 for stack, traceSamples := range traceStacks {
704 pprofSamples := pprofStacks[stack]
705 delete(pprofStacks, stack)
706 if traceSamples < pprofSamples {
707 t.Logf("execution trace did not include all CPU profile samples for stack %q; %d in profile, %d in trace",
708 stack, pprofSamples, traceSamples)
714 for stack, pprofSamples := range pprofStacks {
715 t.Logf("CPU profile included %d samples at stack %q not present in execution trace", pprofSamples, stack)
722 t.Logf("execution trace CPU samples:")
723 for stack, samples := range traceStacks {
724 t.Logf("%d: %q", samples, stack)
726 t.Logf("CPU profile:\n%v", prof)
730 func cpuHogger(f func(x int) int, y *int, dur time.Duration) {
731 // We only need to get one 100 Hz clock tick, so we've got
732 // a large safety buffer.
733 // But do at least 500 iterations (which should take about 100ms),
734 // otherwise TestCPUProfileMultithreaded can fail if only one
735 // thread is scheduled during the testing period.
738 for i := 0; i < 500 || time.Since(t0) < dur; i++ {
748 // The actual CPU hogging function.
749 // Must not call other functions nor access heap/globals in the loop,
750 // otherwise under race detector the samples will be in the race runtime.
751 func cpuHog1(x int) int {
752 return cpuHog0(x, 1e5)
755 func cpuHog0(x, n int) int {
757 for i := 0; i < n; i++ {
759 // Spend time in mcall, stored as gp.m.curg, with g0 running
771 func saveTrace(t *testing.T, buf *bytes.Buffer, name string) {
775 if err := os.WriteFile(name+".trace", buf.Bytes(), 0600); err != nil {
776 t.Errorf("failed to write trace file: %s", err)