testing: stop rounding b.N

The original goal of rounding to readable b.N
was to make it easier to eyeball times.
However, proper analysis requires tooling
(such as benchstat) anyway.

Instead, take b.N as it comes.
This will reduce the impact of external noise
such as GC on benchmarks.

This requires reworking our iteration estimates.
We used to calculate the estimated ns/op
and then divide our target ns by that estimate.
However, this order of operations was destructive
when the ns/op was very small; rounding could
hide almost an order of magnitude of variation.
Instead, multiply first, then divide.
Also, make n an int64 to avoid overflow.

Prior to this change, we attempted to cap b.N at 1e9.
Due to rounding up, it was possible to get b.N as high as 2e9.
This change consistently enforces the 1e9 cap.

This change also reduces the wall time required to run benchmarks.

Here's the impact of this change on the wall time to run
all benchmarks once with benchtime=1s on some std packages:

name           old time/op       new time/op       delta
bytes                 306s ± 1%         238s ± 1%  -22.24%  (p=0.000 n=10+10)
encoding/json         112s ± 8%          99s ± 7%  -11.64%  (p=0.000 n=10+10)
net/http             54.7s ± 7%        44.9s ± 4%  -17.94%  (p=0.000 n=10+9)
runtime               957s ± 1%         714s ± 0%  -25.38%  (p=0.000 n=10+9)
strings               262s ± 1%         201s ± 1%  -23.27%  (p=0.000 n=10+10)
[Geo mean]            216s              172s       -20.23%

Updates #24735

Change-Id: I7e38efb8e23c804046bf4fc065b3f5f3991d0a15
Reviewed-on: https://go-review.googlesource.com/c/go/+/112155
Reviewed-by: Austin Clements <austin@google.com>

diff --git a/src/cmd/go/go_test.go b/src/cmd/go/go_test.go
index 240ba594f5821e7e6479a5f5440e800f454b767d..1ee50ac983e80307824d2838da7422dfffaefd07 100644 (file)
--- a/src/cmd/go/go_test.go
+++ b/src/cmd/go/go_test.go
@@ -4947,14 +4947,14 @@ func TestTestRegexps(t *testing.T) {
     x_test.go:15: LOG: Y running N=10000
     x_test.go:15: LOG: Y running N=1000000
     x_test.go:15: LOG: Y running N=100000000
-    x_test.go:15: LOG: Y running N=2000000000
+    x_test.go:15: LOG: Y running N=1000000000
 --- BENCH: BenchmarkX/Y
     x_test.go:15: LOG: Y running N=1
     x_test.go:15: LOG: Y running N=100
     x_test.go:15: LOG: Y running N=10000
     x_test.go:15: LOG: Y running N=1000000
     x_test.go:15: LOG: Y running N=100000000
-    x_test.go:15: LOG: Y running N=2000000000
+    x_test.go:15: LOG: Y running N=1000000000
 --- BENCH: BenchmarkX
     x_test.go:13: LOG: X running N=1
 --- BENCH: BenchmarkXX

diff --git a/src/testing/benchmark.go b/src/testing/benchmark.go
index 73951767bd75d91d55fa51cee0a5f6b4aa09f14c..407e371c667d2595709d0563d3d0984f035b3515 100644 (file)
--- a/src/testing/benchmark.go
+++ b/src/testing/benchmark.go
@@ -170,13 +170,6 @@ func (b *B) ReportAllocs() {
        b.showAllocResult = true
 }
 
-func (b *B) nsPerOp() int64 {
-       if b.N <= 0 {
-               return 0
-       }
-       return b.duration.Nanoseconds() / int64(b.N)
-}
-
 // runN runs a single benchmark for the specified number of iterations.
 func (b *B) runN(n int) {
        benchmarkLock.Lock()
@@ -199,53 +192,20 @@ func (b *B) runN(n int) {
        }
 }
 
-func min(x, y int) int {
+func min(x, y int64) int64 {
        if x > y {
                return y
        }
        return x
 }
 
-func max(x, y int) int {
+func max(x, y int64) int64 {
        if x < y {
                return y
        }
        return x
 }
 
-// roundDown10 rounds a number down to the nearest power of 10.
-func roundDown10(n int) int {
-       var tens = 0
-       // tens = floor(log_10(n))
-       for n >= 10 {
-               n = n / 10
-               tens++
-       }
-       // result = 10^tens
-       result := 1
-       for i := 0; i < tens; i++ {
-               result *= 10
-       }
-       return result
-}
-
-// roundUp rounds x up to a number of the form [1eX, 2eX, 3eX, 5eX].
-func roundUp(n int) int {
-       base := roundDown10(n)
-       switch {
-       case n <= base:
-               return base
-       case n <= (2 * base):
-               return 2 * base
-       case n <= (3 * base):
-               return 3 * base
-       case n <= (5 * base):
-               return 5 * base
-       default:
-               return 10 * base
-       }
-}
-
 // run1 runs the first iteration of benchFunc. It reports whether more
 // iterations of this benchmarks should be run.
 func (b *B) run1() bool {
@@ -328,20 +288,31 @@ func (b *B) launch() {
                b.runN(b.benchTime.n)
        } else {
                d := b.benchTime.d
-               for n := 1; !b.failed && b.duration < d && n < 1e9; {
+               for n := int64(1); !b.failed && b.duration < d && n < 1e9; {
                        last := n
                        // Predict required iterations.
-                       n = int(d.Nanoseconds())
-                       if nsop := b.nsPerOp(); nsop != 0 {
-                               n /= int(nsop)
+                       goalns := d.Nanoseconds()
+                       prevIters := int64(b.N)
+                       prevns := b.duration.Nanoseconds()
+                       if prevns <= 0 {
+                               // Round up, to avoid div by zero.
+                               prevns = 1
                        }
+                       // Order of operations matters.
+                       // For very fast benchmarks, prevIters ~= prevns.
+                       // If you divide first, you get 0 or 1,
+                       // which can hide an order of magnitude in execution time.
+                       // So multiply first, then divide.
+                       n = goalns * prevIters / prevns
                        // Run more iterations than we think we'll need (1.2x).
+                       n += n / 5
                        // Don't grow too fast in case we had timing errors previously.
+                       n = min(n, 100*last)
                        // Be sure to run at least one more than last time.
-                       n = max(min(n+n/5, 100*last), last+1)
-                       // Round up to something easy to read.
-                       n = roundUp(n)
-                       b.runN(n)
+                       n = max(n, last+1)
+                       // Don't run more than 1e9 times. (This also keeps n in int range on 32 bit platforms.)
+                       n = min(n, 1e9)
+                       b.runN(int(n))
                }
        }
        b.result = BenchmarkResult{b.N, b.duration, b.bytes, b.netAllocs, b.netBytes, b.extra}

diff --git a/src/testing/benchmark_test.go b/src/testing/benchmark_test.go
index 9e87f137f186745ba31bd9bde89fc37c8ce1be89..7d28fb632a426dcc59100e6fd2c1448706c7d019 100644 (file)
--- a/src/testing/benchmark_test.go
+++ b/src/testing/benchmark_test.go
@@ -14,57 +14,6 @@ import (
        "text/template"
 )
 
-var roundDownTests = []struct {
-       v, expected int
-}{
-       {1, 1},
-       {9, 1},
-       {10, 10},
-       {11, 10},
-       {100, 100},
-       {101, 100},
-       {999, 100},
-       {1000, 1000},
-       {1001, 1000},
-}
-
-func TestRoundDown10(t *testing.T) {
-       for _, tt := range roundDownTests {
-               actual := testing.RoundDown10(tt.v)
-               if tt.expected != actual {
-                       t.Errorf("roundDown10(%d): expected %d, actual %d", tt.v, tt.expected, actual)
-               }
-       }
-}
-
-var roundUpTests = []struct {
-       v, expected int
-}{
-       {0, 1},
-       {1, 1},
-       {2, 2},
-       {3, 3},
-       {5, 5},
-       {9, 10},
-       {999, 1000},
-       {1000, 1000},
-       {1400, 2000},
-       {1700, 2000},
-       {2700, 3000},
-       {4999, 5000},
-       {5000, 5000},
-       {5001, 10000},
-}
-
-func TestRoundUp(t *testing.T) {
-       for _, tt := range roundUpTests {
-               actual := testing.RoundUp(tt.v)
-               if tt.expected != actual {
-                       t.Errorf("roundUp(%d): expected %d, actual %d", tt.v, tt.expected, actual)
-               }
-       }
-}
-
 var prettyPrintTests = []struct {
        v        float64
        expected string

diff --git a/src/testing/export_test.go b/src/testing/export_test.go
index 65e5c3dbb8207a7ff16e5e60ea7a15df1deb6ef5..0022491ecd9e479b96ebaea62f3707dfd409fee2 100644 (file)
--- a/src/testing/export_test.go
+++ b/src/testing/export_test.go
@@ -4,8 +4,4 @@
 
 package testing
 
-var (
-       RoundDown10 = roundDown10
-       RoundUp     = roundUp
-       PrettyPrint = prettyPrint
-)
+var PrettyPrint = prettyPrint