src/cmd/dist/test.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "flag"
  10         "fmt"
  11         "log"
  12         "os"
  13         "os/exec"
  14         "path/filepath"
  15         "reflect"
  16         "regexp"
  17         "runtime"
  18         "strconv"
  19         "strings"
  20         "sync"
  21         "time"
  22 )
  23
  24 func cmdtest() {
  25         gogcflags = os.Getenv("GO_GCFLAGS")
  26         setNoOpt()
  27
  28         var t tester
  29
  30         var noRebuild bool
  31         flag.BoolVar(&t.listMode, "list", false, "list available tests")
  32         flag.BoolVar(&t.rebuild, "rebuild", false, "rebuild everything first")
  33         flag.BoolVar(&noRebuild, "no-rebuild", false, "overrides -rebuild (historical dreg)")
  34         flag.BoolVar(&t.keepGoing, "k", false, "keep going even when error occurred")
  35         flag.BoolVar(&t.race, "race", false, "run in race builder mode (different set of tests)")
  36         flag.BoolVar(&t.compileOnly, "compile-only", false, "compile tests, but don't run them. This is for some builders. Not all dist tests respect this flag, but most do.")
  37         flag.StringVar(&t.banner, "banner", "##### ", "banner prefix; blank means no section banners")
  38         flag.StringVar(&t.runRxStr, "run", os.Getenv("GOTESTONLY"),
  39                 "run only those tests matching the regular expression; empty means to run all. "+
  40                         "Special exception: if the string begins with '!', the match is inverted.")
  41         flag.BoolVar(&t.msan, "msan", false, "run in memory sanitizer builder mode")
  42         flag.BoolVar(&t.asan, "asan", false, "run in address sanitizer builder mode")
  43
  44         xflagparse(-1) // any number of args
  45         if noRebuild {
  46                 t.rebuild = false
  47         }
  48
  49         t.run()
  50 }
  51
  52 // tester executes cmdtest.
  53 type tester struct {
  54         race        bool
  55         msan        bool
  56         asan        bool
  57         listMode    bool
  58         rebuild     bool
  59         failed      bool
  60         keepGoing   bool
  61         compileOnly bool // just try to compile all tests, but no need to run
  62         runRxStr    string
  63         runRx       *regexp.Regexp
  64         runRxWant   bool     // want runRx to match (true) or not match (false)
  65         runNames    []string // tests to run, exclusive with runRx; empty means all
  66         banner      string   // prefix, or "" for none
  67         lastHeading string   // last dir heading printed
  68
  69         cgoEnabled bool
  70         partial    bool
  71         haveTime   bool // the 'time' binary is available
  72
  73         tests        []distTest
  74         timeoutScale int
  75
  76         worklist []*work
  77 }
  78
  79 type work struct {
  80         dt    *distTest
  81         cmd   *exec.Cmd
  82         start chan bool
  83         out   []byte
  84         err   error
  85         end   chan bool
  86 }
  87
  88 // A distTest is a test run by dist test.
  89 // Each test has a unique name and belongs to a group (heading)
  90 type distTest struct {
  91         name    string // unique test name; may be filtered with -run flag
  92         heading string // group section; this header is printed before the test is run.
  93         fn      func(*distTest) error
  94 }
  95
  96 func (t *tester) run() {
  97         timelog("start", "dist test")
  98
  99         os.Setenv("PATH", fmt.Sprintf("%s%c%s", gorootBin, os.PathListSeparator, os.Getenv("PATH")))
 100
 101         cmd := exec.Command(gorootBinGo, "env", "CGO_ENABLED")
 102         cmd.Stderr = new(bytes.Buffer)
 103         slurp, err := cmd.Output()
 104         if err != nil {
 105                 fatalf("Error running go env CGO_ENABLED: %v\n%s", err, cmd.Stderr)
 106         }
 107         t.cgoEnabled, _ = strconv.ParseBool(strings.TrimSpace(string(slurp)))
 108         if flag.NArg() > 0 && t.runRxStr != "" {
 109                 fatalf("the -run regular expression flag is mutually exclusive with test name arguments")
 110         }
 111
 112         t.runNames = flag.Args()
 113
 114         if t.hasBash() {
 115                 if _, err := exec.LookPath("time"); err == nil {
 116                         t.haveTime = true
 117                 }
 118         }
 119
 120         // Set GOTRACEBACK to system if the user didn't set a level explicitly.
 121         // Since we're running tests for Go, we want as much detail as possible
 122         // if something goes wrong.
 123         //
 124         // Set it before running any commands just in case something goes wrong.
 125         if ok := isEnvSet("GOTRACEBACK"); !ok {
 126                 if err := os.Setenv("GOTRACEBACK", "system"); err != nil {
 127                         if t.keepGoing {
 128                                 log.Printf("Failed to set GOTRACEBACK: %v", err)
 129                         } else {
 130                                 fatalf("Failed to set GOTRACEBACK: %v", err)
 131                         }
 132                 }
 133         }
 134
 135         if t.rebuild {
 136                 t.out("Building packages and commands.")
 137                 // Force rebuild the whole toolchain.
 138                 goInstall("go", append([]string{"-a"}, toolchain...)...)
 139         }
 140
 141         if !t.listMode {
 142                 if os.Getenv("GO_BUILDER_NAME") == "" {
 143                         // Complete rebuild bootstrap, even with -no-rebuild.
 144                         // If everything is up-to-date, this is a no-op.
 145                         // If everything is not up-to-date, the first checkNotStale
 146                         // during the test process will kill the tests, so we might
 147                         // as well install the world.
 148                         // Now that for example "go install cmd/compile" does not
 149                         // also install runtime (you need "go install -i cmd/compile"
 150                         // for that), it's easy for previous workflows like
 151                         // "rebuild the compiler and then run run.bash"
 152                         // to break if we don't automatically refresh things here.
 153                         // Rebuilding is a shortened bootstrap.
 154                         // See cmdbootstrap for a description of the overall process.
 155                         goInstall("go", toolchain...)
 156                         goInstall("go", toolchain...)
 157                         goInstall("go", "std", "cmd")
 158                 } else {
 159                         // The Go builder infrastructure should always begin running tests from a
 160                         // clean, non-stale state, so there is no need to rebuild the world.
 161                         // Instead, we can just check that it is not stale, which may be less
 162                         // expensive (and is also more likely to catch bugs in the builder
 163                         // implementation).
 164                         checkNotStale("go", "std", "cmd")
 165                 }
 166         }
 167
 168         t.timeoutScale = 1
 169         switch goarch {
 170         case "arm":
 171                 t.timeoutScale = 2
 172         case "mips", "mipsle", "mips64", "mips64le":
 173                 t.timeoutScale = 4
 174         }
 175         if s := os.Getenv("GO_TEST_TIMEOUT_SCALE"); s != "" {
 176                 t.timeoutScale, err = strconv.Atoi(s)
 177                 if err != nil {
 178                         fatalf("failed to parse $GO_TEST_TIMEOUT_SCALE = %q as integer: %v", s, err)
 179                 }
 180         }
 181
 182         if t.runRxStr != "" {
 183                 if t.runRxStr[0] == '!' {
 184                         t.runRxWant = false
 185                         t.runRxStr = t.runRxStr[1:]
 186                 } else {
 187                         t.runRxWant = true
 188                 }
 189                 t.runRx = regexp.MustCompile(t.runRxStr)
 190         }
 191
 192         t.registerTests()
 193         if t.listMode {
 194                 for _, tt := range t.tests {
 195                         fmt.Println(tt.name)
 196                 }
 197                 return
 198         }
 199
 200         for _, name := range t.runNames {
 201                 if !t.isRegisteredTestName(name) {
 202                         fatalf("unknown test %q", name)
 203                 }
 204         }
 205
 206         // On a few builders, make GOROOT unwritable to catch tests writing to it.
 207         if strings.HasPrefix(os.Getenv("GO_BUILDER_NAME"), "linux-") {
 208                 if os.Getuid() == 0 {
 209                         // Don't bother making GOROOT unwritable:
 210                         // we're running as root, so permissions would have no effect.
 211                 } else {
 212                         xatexit(t.makeGOROOTUnwritable())
 213                 }
 214         }
 215
 216         if err := t.maybeLogMetadata(); err != nil {
 217                 t.failed = true
 218                 if t.keepGoing {
 219                         log.Printf("Failed logging metadata: %v", err)
 220                 } else {
 221                         fatalf("Failed logging metadata: %v", err)
 222                 }
 223         }
 224
 225         for _, dt := range t.tests {
 226                 if !t.shouldRunTest(dt.name) {
 227                         t.partial = true
 228                         continue
 229                 }
 230                 dt := dt // dt used in background after this iteration
 231                 if err := dt.fn(&dt); err != nil {
 232                         t.runPending(&dt) // in case that hasn't been done yet
 233                         t.failed = true
 234                         if t.keepGoing {
 235                                 log.Printf("Failed: %v", err)
 236                         } else {
 237                                 fatalf("Failed: %v", err)
 238                         }
 239                 }
 240         }
 241         t.runPending(nil)
 242         timelog("end", "dist test")
 243
 244         if t.failed {
 245                 fmt.Println("\nFAILED")
 246                 xexit(1)
 247         } else if incomplete[goos+"/"+goarch] {
 248                 // The test succeeded, but consider it as failed so we don't
 249                 // forget to remove the port from the incomplete map once the
 250                 // port is complete.
 251                 fmt.Println("\nFAILED (incomplete port)")
 252                 xexit(1)
 253         } else if t.partial {
 254                 fmt.Println("\nALL TESTS PASSED (some were excluded)")
 255         } else {
 256                 fmt.Println("\nALL TESTS PASSED")
 257         }
 258 }
 259
 260 func (t *tester) shouldRunTest(name string) bool {
 261         if t.runRx != nil {
 262                 return t.runRx.MatchString(name) == t.runRxWant
 263         }
 264         if len(t.runNames) == 0 {
 265                 return true
 266         }
 267         for _, runName := range t.runNames {
 268                 if runName == name {
 269                         return true
 270                 }
 271         }
 272         return false
 273 }
 274
 275 func (t *tester) maybeLogMetadata() error {
 276         if t.compileOnly {
 277                 // We need to run a subprocess to log metadata. Don't do that
 278                 // on compile-only runs.
 279                 return nil
 280         }
 281         t.out("Test execution environment.")
 282         // Helper binary to print system metadata (CPU model, etc). This is a
 283         // separate binary from dist so it need not build with the bootstrap
 284         // toolchain.
 285         //
 286         // TODO(prattmic): If we split dist bootstrap and dist test then this
 287         // could be simplified to directly use internal/sysinfo here.
 288         return t.dirCmd(filepath.Join(goroot, "src/cmd/internal/metadata"), "go", []string{"run", "main.go"}).Run()
 289 }
 290
 291 // short returns a -short flag value to use with 'go test'
 292 // or a test binary for tests intended to run in short mode.
 293 // It returns "true", unless the environment variable
 294 // GO_TEST_SHORT is set to a non-empty, false-ish string.
 295 //
 296 // This environment variable is meant to be an internal
 297 // detail between the Go build system and cmd/dist for
 298 // the purpose of longtest builders, and is not intended
 299 // for use by users. See golang.org/issue/12508.
 300 func short() string {
 301         if v := os.Getenv("GO_TEST_SHORT"); v != "" {
 302                 short, err := strconv.ParseBool(v)
 303                 if err != nil {
 304                         fatalf("invalid GO_TEST_SHORT %q: %v", v, err)
 305                 }
 306                 if !short {
 307                         return "false"
 308                 }
 309         }
 310         return "true"
 311 }
 312
 313 // goTest returns the beginning of the go test command line.
 314 // Callers should use goTest and then pass flags overriding these
 315 // defaults as later arguments in the command line.
 316 func (t *tester) goTest() []string {
 317         return []string{
 318                 "go", "test", "-short=" + short(), "-count=1", t.tags(), t.runFlag(""),
 319         }
 320 }
 321
 322 func (t *tester) tags() string {
 323         ios := t.iOS()
 324         switch {
 325         case ios && noOpt:
 326                 return "-tags=lldb,noopt"
 327         case ios:
 328                 return "-tags=lldb"
 329         case noOpt:
 330                 return "-tags=noopt"
 331         default:
 332                 return "-tags="
 333         }
 334 }
 335
 336 // timeoutDuration converts the provided number of seconds into a
 337 // time.Duration, scaled by the t.timeoutScale factor.
 338 func (t *tester) timeoutDuration(sec int) time.Duration {
 339         return time.Duration(sec) * time.Second * time.Duration(t.timeoutScale)
 340 }
 341
 342 // timeout returns the "-timeout=" string argument to "go test" given
 343 // the number of seconds of timeout. It scales it by the
 344 // t.timeoutScale factor.
 345 func (t *tester) timeout(sec int) string {
 346         return "-timeout=" + t.timeoutDuration(sec).String()
 347 }
 348
 349 // ranGoTest and stdMatches are state closed over by the stdlib
 350 // testing func in registerStdTest below. The tests are run
 351 // sequentially, so there's no need for locks.
 352 //
 353 // ranGoBench and benchMatches are the same, but are only used
 354 // in -race mode.
 355 var (
 356         ranGoTest  bool
 357         stdMatches []string
 358
 359         ranGoBench   bool
 360         benchMatches []string
 361 )
 362
 363 func (t *tester) registerStdTest(pkg string) {
 364         heading := "Testing packages."
 365         testPrefix := "go_test:"
 366         gcflags := gogcflags
 367
 368         testName := testPrefix + pkg
 369         if t.runRx == nil || t.runRx.MatchString(testName) == t.runRxWant {
 370                 stdMatches = append(stdMatches, pkg)
 371         }
 372
 373         t.tests = append(t.tests, distTest{
 374                 name:    testName,
 375                 heading: heading,
 376                 fn: func(dt *distTest) error {
 377                         if ranGoTest {
 378                                 return nil
 379                         }
 380                         t.runPending(dt)
 381                         timelog("start", dt.name)
 382                         defer timelog("end", dt.name)
 383                         ranGoTest = true
 384
 385                         timeoutSec := 180
 386                         for _, pkg := range stdMatches {
 387                                 if pkg == "cmd/go" {
 388                                         timeoutSec *= 3
 389                                         break
 390                                 }
 391                         }
 392                         args := []string{
 393                                 "test",
 394                                 "-short=" + short(),
 395                                 t.tags(),
 396                                 t.timeout(timeoutSec),
 397                         }
 398                         if gcflags != "" {
 399                                 args = append(args, "-gcflags=all="+gcflags)
 400                         }
 401                         if t.race {
 402                                 args = append(args, "-race")
 403                         }
 404                         if t.msan {
 405                                 args = append(args, "-msan")
 406                         }
 407                         if t.asan {
 408                                 args = append(args, "-asan")
 409                         }
 410                         if t.compileOnly {
 411                                 args = append(args, "-run=^$")
 412                         }
 413                         args = append(args, stdMatches...)
 414                         cmd := exec.Command(gorootBinGo, args...)
 415                         cmd.Stdout = os.Stdout
 416                         cmd.Stderr = os.Stderr
 417                         return cmd.Run()
 418                 },
 419         })
 420 }
 421
 422 func (t *tester) registerRaceBenchTest(pkg string) {
 423         testName := "go_test_bench:" + pkg
 424         if t.runRx == nil || t.runRx.MatchString(testName) == t.runRxWant {
 425                 benchMatches = append(benchMatches, pkg)
 426         }
 427         t.tests = append(t.tests, distTest{
 428                 name:    testName,
 429                 heading: "Running benchmarks briefly.",
 430                 fn: func(dt *distTest) error {
 431                         if ranGoBench {
 432                                 return nil
 433                         }
 434                         t.runPending(dt)
 435                         timelog("start", dt.name)
 436                         defer timelog("end", dt.name)
 437                         ranGoBench = true
 438                         args := []string{
 439                                 "test",
 440                                 "-short=" + short(),
 441                                 "-race",
 442                                 t.timeout(1200), // longer timeout for race with benchmarks
 443                                 "-run=^$",       // nothing. only benchmarks.
 444                                 "-benchtime=.1s",
 445                                 "-cpu=4",
 446                         }
 447                         if !t.compileOnly {
 448                                 args = append(args, "-bench=.*")
 449                         }
 450                         args = append(args, benchMatches...)
 451                         cmd := exec.Command(gorootBinGo, args...)
 452                         cmd.Stdout = os.Stdout
 453                         cmd.Stderr = os.Stderr
 454                         return cmd.Run()
 455                 },
 456         })
 457 }
 458
 459 // stdOutErrAreTerminals is defined in test_linux.go, to report
 460 // whether stdout & stderr are terminals.
 461 var stdOutErrAreTerminals func() bool
 462
 463 func (t *tester) registerTests() {
 464         // Fast path to avoid the ~1 second of `go list std cmd` when
 465         // the caller lists specific tests to run. (as the continuous
 466         // build coordinator does).
 467         if len(t.runNames) > 0 {
 468                 for _, name := range t.runNames {
 469                         if strings.HasPrefix(name, "go_test:") {
 470                                 t.registerStdTest(strings.TrimPrefix(name, "go_test:"))
 471                         }
 472                         if strings.HasPrefix(name, "go_test_bench:") {
 473                                 t.registerRaceBenchTest(strings.TrimPrefix(name, "go_test_bench:"))
 474                         }
 475                 }
 476         } else {
 477                 // Use a format string to only list packages and commands that have tests.
 478                 const format = "{{if (or .TestGoFiles .XTestGoFiles)}}{{.ImportPath}}{{end}}"
 479                 cmd := exec.Command(gorootBinGo, "list", "-f", format)
 480                 if t.race {
 481                         cmd.Args = append(cmd.Args, "-tags=race")
 482                 }
 483                 cmd.Args = append(cmd.Args, "std", "cmd")
 484                 cmd.Stderr = new(bytes.Buffer)
 485                 all, err := cmd.Output()
 486                 if err != nil {
 487                         fatalf("Error running go list std cmd: %v:\n%s", err, cmd.Stderr)
 488                 }
 489                 pkgs := strings.Fields(string(all))
 490                 for _, pkg := range pkgs {
 491                         t.registerStdTest(pkg)
 492                 }
 493                 if t.race {
 494                         for _, pkg := range pkgs {
 495                                 if t.packageHasBenchmarks(pkg) {
 496                                         t.registerRaceBenchTest(pkg)
 497                                 }
 498                         }
 499                 }
 500         }
 501
 502         // Test the os/user package in the pure-Go mode too.
 503         if !t.compileOnly {
 504                 t.tests = append(t.tests, distTest{
 505                         name:    "osusergo",
 506                         heading: "os/user with tag osusergo",
 507                         fn: func(dt *distTest) error {
 508                                 t.addCmd(dt, "src", t.goTest(), t.timeout(300), "-tags=osusergo", "os/user")
 509                                 return nil
 510                         },
 511                 })
 512         }
 513
 514         // Test ios/amd64 for the iOS simulator.
 515         if goos == "darwin" && goarch == "amd64" && t.cgoEnabled {
 516                 t.tests = append(t.tests, distTest{
 517                         name:    "amd64ios",
 518                         heading: "GOOS=ios on darwin/amd64",
 519                         fn: func(dt *distTest) error {
 520                                 cmd := t.addCmd(dt, "src", t.goTest(), t.timeout(300), "-run=SystemRoots", "crypto/x509")
 521                                 setEnv(cmd, "GOOS", "ios")
 522                                 setEnv(cmd, "CGO_ENABLED", "1")
 523                                 return nil
 524                         },
 525                 })
 526         }
 527
 528         if t.race {
 529                 return
 530         }
 531
 532         // Runtime CPU tests.
 533         if !t.compileOnly && goos != "js" { // js can't handle -cpu != 1
 534                 testName := "runtime:cpu124"
 535                 t.tests = append(t.tests, distTest{
 536                         name:    testName,
 537                         heading: "GOMAXPROCS=2 runtime -cpu=1,2,4 -quick",
 538                         fn: func(dt *distTest) error {
 539                                 cmd := t.addCmd(dt, "src", t.goTest(), "-short=true", t.timeout(300), "runtime", "-cpu=1,2,4", "-quick")
 540                                 // We set GOMAXPROCS=2 in addition to -cpu=1,2,4 in order to test runtime bootstrap code,
 541                                 // creation of first goroutines and first garbage collections in the parallel setting.
 542                                 setEnv(cmd, "GOMAXPROCS", "2")
 543                                 return nil
 544                         },
 545                 })
 546         }
 547
 548         // morestack tests. We only run these on in long-test mode
 549         // (with GO_TEST_SHORT=false) because the runtime test is
 550         // already quite long and mayMoreStackMove makes it about
 551         // twice as slow.
 552         if !t.compileOnly && short() == "false" {
 553                 // hooks is the set of maymorestack hooks to test with.
 554                 hooks := []string{"mayMoreStackPreempt", "mayMoreStackMove"}
 555                 // pkgs is the set of test packages to run.
 556                 pkgs := []string{"runtime", "reflect", "sync"}
 557                 // hookPkgs is the set of package patterns to apply
 558                 // the maymorestack hook to.
 559                 hookPkgs := []string{"runtime/...", "reflect", "sync"}
 560                 // unhookPkgs is the set of package patterns to
 561                 // exclude from hookPkgs.
 562                 unhookPkgs := []string{"runtime/testdata/..."}
 563                 for _, hook := range hooks {
 564                         // Construct the build flags to use the
 565                         // maymorestack hook in the compiler and
 566                         // assembler. We pass this via the GOFLAGS
 567                         // environment variable so that it applies to
 568                         // both the test itself and to binaries built
 569                         // by the test.
 570                         goFlagsList := []string{}
 571                         for _, flag := range []string{"-gcflags", "-asmflags"} {
 572                                 for _, hookPkg := range hookPkgs {
 573                                         goFlagsList = append(goFlagsList, flag+"="+hookPkg+"=-d=maymorestack=runtime."+hook)
 574                                 }
 575                                 for _, unhookPkg := range unhookPkgs {
 576                                         goFlagsList = append(goFlagsList, flag+"="+unhookPkg+"=")
 577                                 }
 578                         }
 579                         goFlags := strings.Join(goFlagsList, " ")
 580
 581                         for _, pkg := range pkgs {
 582                                 pkg := pkg
 583                                 testName := hook + ":" + pkg
 584                                 t.tests = append(t.tests, distTest{
 585                                         name:    testName,
 586                                         heading: "maymorestack=" + hook,
 587                                         fn: func(dt *distTest) error {
 588                                                 cmd := t.addCmd(dt, "src", t.goTest(), t.timeout(600), pkg, "-short")
 589                                                 setEnv(cmd, "GOFLAGS", goFlags)
 590                                                 return nil
 591                                         },
 592                                 })
 593                         }
 594                 }
 595         }
 596
 597         // This test needs its stdout/stderr to be terminals, so we don't run it from cmd/go's tests.
 598         // See issue 18153.
 599         if goos == "linux" {
 600                 t.tests = append(t.tests, distTest{
 601                         name:    "cmd_go_test_terminal",
 602                         heading: "cmd/go terminal test",
 603                         fn: func(dt *distTest) error {
 604                                 t.runPending(dt)
 605                                 timelog("start", dt.name)
 606                                 defer timelog("end", dt.name)
 607                                 if !stdOutErrAreTerminals() {
 608                                         fmt.Println("skipping terminal test; stdout/stderr not terminals")
 609                                         return nil
 610                                 }
 611                                 cmd := exec.Command(gorootBinGo, "test")
 612                                 setDir(cmd, filepath.Join(os.Getenv("GOROOT"), "src/cmd/go/testdata/testterminal18153"))
 613                                 cmd.Stdout = os.Stdout
 614                                 cmd.Stderr = os.Stderr
 615                                 return cmd.Run()
 616                         },
 617                 })
 618         }
 619
 620         // On the builders only, test that a moved GOROOT still works.
 621         // Fails on iOS because CC_FOR_TARGET refers to clangwrap.sh
 622         // in the unmoved GOROOT.
 623         // Fails on Android and js/wasm with an exec format error.
 624         // Fails on plan9 with "cannot find GOROOT" (issue #21016).
 625         if os.Getenv("GO_BUILDER_NAME") != "" && goos != "android" && !t.iOS() && goos != "plan9" && goos != "js" {
 626                 t.tests = append(t.tests, distTest{
 627                         name:    "moved_goroot",
 628                         heading: "moved GOROOT",
 629                         fn: func(dt *distTest) error {
 630                                 t.runPending(dt)
 631                                 timelog("start", dt.name)
 632                                 defer timelog("end", dt.name)
 633                                 moved := goroot + "-moved"
 634                                 if err := os.Rename(goroot, moved); err != nil {
 635                                         if goos == "windows" {
 636                                                 // Fails on Windows (with "Access is denied") if a process
 637                                                 // or binary is in this directory. For instance, using all.bat
 638                                                 // when run from c:\workdir\go\src fails here
 639                                                 // if GO_BUILDER_NAME is set. Our builders invoke tests
 640                                                 // a different way which happens to work when sharding
 641                                                 // tests, but we should be tolerant of the non-sharded
 642                                                 // all.bat case.
 643                                                 log.Printf("skipping test on Windows")
 644                                                 return nil
 645                                         }
 646                                         return err
 647                                 }
 648
 649                                 // Run `go test fmt` in the moved GOROOT, without explicitly setting
 650                                 // GOROOT in the environment. The 'go' command should find itself.
 651                                 cmd := exec.Command(filepath.Join(moved, "bin", "go"), "test", "fmt")
 652                                 cmd.Stdout = os.Stdout
 653                                 cmd.Stderr = os.Stderr
 654                                 unsetEnv(cmd, "GOROOT")
 655                                 unsetEnv(cmd, "GOCACHE") // TODO(bcmills): ...why‽
 656                                 err := cmd.Run()
 657
 658                                 if rerr := os.Rename(moved, goroot); rerr != nil {
 659                                         fatalf("failed to restore GOROOT: %v", rerr)
 660                                 }
 661                                 return err
 662                         },
 663                 })
 664         }
 665
 666         // Test that internal linking of standard packages does not
 667         // require libgcc. This ensures that we can install a Go
 668         // release on a system that does not have a C compiler
 669         // installed and still build Go programs (that don't use cgo).
 670         for _, pkg := range cgoPackages {
 671                 if !t.internalLink() {
 672                         break
 673                 }
 674
 675                 // ARM libgcc may be Thumb, which internal linking does not support.
 676                 if goarch == "arm" {
 677                         break
 678                 }
 679
 680                 pkg := pkg
 681                 var run string
 682                 if pkg == "net" {
 683                         run = "TestTCPStress"
 684                 }
 685                 t.tests = append(t.tests, distTest{
 686                         name:    "nolibgcc:" + pkg,
 687                         heading: "Testing without libgcc.",
 688                         fn: func(dt *distTest) error {
 689                                 // What matters is that the tests build and start up.
 690                                 // Skip expensive tests, especially x509 TestSystemRoots.
 691                                 t.addCmd(dt, "src", t.goTest(), "-ldflags=-linkmode=internal -libgcc=none", "-run=^Test[^CS]", pkg, t.runFlag(run))
 692                                 return nil
 693                         },
 694                 })
 695         }
 696
 697         // Stub out following test on alpine until 54354 resolved.
 698         builderName := os.Getenv("GO_BUILDER_NAME")
 699         disablePIE := strings.HasSuffix(builderName, "-alpine")
 700
 701         // Test internal linking of PIE binaries where it is supported.
 702         if t.internalLinkPIE() && !disablePIE {
 703                 t.tests = append(t.tests, distTest{
 704                         name:    "pie_internal",
 705                         heading: "internal linking of -buildmode=pie",
 706                         fn: func(dt *distTest) error {
 707                                 cmd := t.addCmd(dt, "src", t.goTest(), "reflect", "-buildmode=pie", "-ldflags=-linkmode=internal", t.timeout(60))
 708                                 setEnv(cmd, "CGO_ENABLED", "0")
 709                                 return nil
 710                         },
 711                 })
 712                 // Also test a cgo package.
 713                 if t.cgoEnabled && t.internalLink() && !disablePIE {
 714                         t.tests = append(t.tests, distTest{
 715                                 name:    "pie_internal_cgo",
 716                                 heading: "internal linking of -buildmode=pie",
 717                                 fn: func(dt *distTest) error {
 718                                         t.addCmd(dt, "src", t.goTest(), "os/user", "-buildmode=pie", "-ldflags=-linkmode=internal", t.timeout(60))
 719                                         return nil
 720                                 },
 721                         })
 722                 }
 723         }
 724
 725         // sync tests
 726         if goos != "js" { // js doesn't support -cpu=10
 727                 t.tests = append(t.tests, distTest{
 728                         name:    "sync_cpu",
 729                         heading: "sync -cpu=10",
 730                         fn: func(dt *distTest) error {
 731                                 t.addCmd(dt, "src", t.goTest(), "sync", t.timeout(120), "-cpu=10", t.runFlag(""))
 732                                 return nil
 733                         },
 734                 })
 735         }
 736
 737         if t.raceDetectorSupported() {
 738                 t.tests = append(t.tests, distTest{
 739                         name:    "race",
 740                         heading: "Testing race detector",
 741                         fn:      t.raceTest,
 742                 })
 743         }
 744
 745         if t.cgoEnabled && !t.iOS() {
 746                 // Disabled on iOS. golang.org/issue/15919
 747                 t.registerHostTest("cgo_stdio", "../misc/cgo/stdio", "misc/cgo/stdio", ".")
 748                 t.registerHostTest("cgo_life", "../misc/cgo/life", "misc/cgo/life", ".")
 749                 fortran := os.Getenv("FC")
 750                 if fortran == "" {
 751                         fortran, _ = exec.LookPath("gfortran")
 752                 }
 753                 if t.hasBash() && goos != "android" && fortran != "" {
 754                         t.tests = append(t.tests, distTest{
 755                                 name:    "cgo_fortran",
 756                                 heading: "../misc/cgo/fortran",
 757                                 fn: func(dt *distTest) error {
 758                                         t.addCmd(dt, "misc/cgo/fortran", "./test.bash", fortran)
 759                                         return nil
 760                                 },
 761                         })
 762                 }
 763                 if t.hasSwig() && goos != "android" {
 764                         t.tests = append(t.tests, distTest{
 765                                 name:    "swig_stdio",
 766                                 heading: "../misc/swig/stdio",
 767                                 fn: func(dt *distTest) error {
 768                                         t.addCmd(dt, "misc/swig/stdio", t.goTest(), ".")
 769                                         return nil
 770                                 },
 771                         })
 772                         if t.hasCxx() {
 773                                 t.tests = append(t.tests,
 774                                         distTest{
 775                                                 name:    "swig_callback",
 776                                                 heading: "../misc/swig/callback",
 777                                                 fn: func(dt *distTest) error {
 778                                                         t.addCmd(dt, "misc/swig/callback", t.goTest(), ".")
 779                                                         return nil
 780                                                 },
 781                                         },
 782                                         distTest{
 783                                                 name:    "swig_callback_lto",
 784                                                 heading: "../misc/swig/callback",
 785                                                 fn: func(dt *distTest) error {
 786                                                         cmd := t.addCmd(dt, "misc/swig/callback", t.goTest(), ".")
 787                                                         setEnv(cmd, "CGO_CFLAGS", "-flto -Wno-lto-type-mismatch -Wno-unknown-warning-option")
 788                                                         setEnv(cmd, "CGO_CXXFLAGS", "-flto -Wno-lto-type-mismatch -Wno-unknown-warning-option")
 789                                                         setEnv(cmd, "CGO_LDFLAGS", "-flto -Wno-lto-type-mismatch -Wno-unknown-warning-option")
 790                                                         return nil
 791                                                 },
 792                                         },
 793                                 )
 794                         }
 795                 }
 796         }
 797         if t.cgoEnabled {
 798                 t.tests = append(t.tests, distTest{
 799                         name:    "cgo_test",
 800                         heading: "../misc/cgo/test",
 801                         fn:      t.cgoTest,
 802                 })
 803         }
 804
 805         // Don't run these tests with $GO_GCFLAGS because most of them
 806         // assume that they can run "go install" with no -gcflags and not
 807         // recompile the entire standard library. If make.bash ran with
 808         // special -gcflags, that's not true.
 809         if t.cgoEnabled && gogcflags == "" {
 810                 t.registerHostTest("testgodefs", "../misc/cgo/testgodefs", "misc/cgo/testgodefs", ".")
 811
 812                 t.registerTest("testso", "../misc/cgo/testso", t.goTest(), t.timeout(600), ".")
 813                 t.registerTest("testsovar", "../misc/cgo/testsovar", t.goTest(), t.timeout(600), ".")
 814                 if t.supportedBuildmode("c-archive") {
 815                         t.registerHostTest("testcarchive", "../misc/cgo/testcarchive", "misc/cgo/testcarchive", ".")
 816                 }
 817                 if t.supportedBuildmode("c-shared") {
 818                         t.registerHostTest("testcshared", "../misc/cgo/testcshared", "misc/cgo/testcshared", ".")
 819                 }
 820                 if t.supportedBuildmode("shared") {
 821                         t.registerTest("testshared", "../misc/cgo/testshared", t.goTest(), t.timeout(600), ".")
 822                 }
 823                 if t.supportedBuildmode("plugin") {
 824                         t.registerTest("testplugin", "../misc/cgo/testplugin", t.goTest(), t.timeout(600), ".")
 825                 }
 826                 if goos == "linux" || (goos == "freebsd" && goarch == "amd64") {
 827                         // because Pdeathsig of syscall.SysProcAttr struct used in misc/cgo/testsanitizers is only
 828                         // supported on Linux and FreeBSD.
 829                         t.registerHostTest("testsanitizers", "../misc/cgo/testsanitizers", "misc/cgo/testsanitizers", ".")
 830                 }
 831                 if t.hasBash() && goos != "android" && !t.iOS() && gohostos != "windows" {
 832                         t.registerHostTest("cgo_errors", "../misc/cgo/errors", "misc/cgo/errors", ".")
 833                 }
 834         }
 835
 836         if goos != "android" && !t.iOS() {
 837                 // There are no tests in this directory, only benchmarks.
 838                 // Check that the test binary builds but don't bother running it.
 839                 // (It has init-time work to set up for the benchmarks that is not worth doing unnecessarily.)
 840                 t.registerTest("bench_go1", "../test/bench/go1", t.goTest(), "-c", "-o="+os.DevNull)
 841         }
 842         if goos != "android" && !t.iOS() {
 843                 // Only start multiple test dir shards on builders,
 844                 // where they get distributed to multiple machines.
 845                 // See issues 20141 and 31834.
 846                 nShards := 1
 847                 if os.Getenv("GO_BUILDER_NAME") != "" {
 848                         nShards = 10
 849                 }
 850                 if n, err := strconv.Atoi(os.Getenv("GO_TEST_SHARDS")); err == nil {
 851                         nShards = n
 852                 }
 853                 for shard := 0; shard < nShards; shard++ {
 854                         shard := shard
 855                         t.tests = append(t.tests, distTest{
 856                                 name:    fmt.Sprintf("test:%d_%d", shard, nShards),
 857                                 heading: "../test",
 858                                 fn:      func(dt *distTest) error { return t.testDirTest(dt, shard, nShards) },
 859                         })
 860                 }
 861         }
 862         // Only run the API check on fast development platforms.
 863         // Every platform checks the API on every GOOS/GOARCH/CGO_ENABLED combination anyway,
 864         // so we really only need to run this check once anywhere to get adequate coverage.
 865         // To help developers avoid trybot-only failures, we try to run on typical developer machines
 866         // which is darwin/linux/windows and amd64/arm64.
 867         if (goos == "darwin" || goos == "linux" || goos == "windows") && (goarch == "amd64" || goarch == "arm64") {
 868                 t.tests = append(t.tests, distTest{
 869                         name:    "api",
 870                         heading: "API check",
 871                         fn: func(dt *distTest) error {
 872                                 if t.compileOnly {
 873                                         t.addCmd(dt, "src", "go", "build", "-o", os.DevNull, filepath.Join(goroot, "src/cmd/api/run.go"))
 874                                         return nil
 875                                 }
 876                                 t.addCmd(dt, "src", "go", "run", filepath.Join(goroot, "src/cmd/api/run.go"))
 877                                 return nil
 878                         },
 879                 })
 880         }
 881
 882         // Ensure that the toolchain can bootstrap itself.
 883         // This test adds another ~45s to all.bash if run sequentially, so run it only on the builders.
 884         if os.Getenv("GO_BUILDER_NAME") != "" && goos != "android" && !t.iOS() {
 885                 t.registerHostTest("reboot", "../misc/reboot", "misc/reboot", ".")
 886         }
 887 }
 888
 889 // isRegisteredTestName reports whether a test named testName has already
 890 // been registered.
 891 func (t *tester) isRegisteredTestName(testName string) bool {
 892         for _, tt := range t.tests {
 893                 if tt.name == testName {
 894                         return true
 895                 }
 896         }
 897         return false
 898 }
 899
 900 func (t *tester) registerTest1(seq bool, name, dirBanner string, cmdline ...interface{}) {
 901         bin, args := flattenCmdline(cmdline)
 902         if bin == "time" && !t.haveTime {
 903                 bin, args = args[0], args[1:]
 904         }
 905         if t.isRegisteredTestName(name) {
 906                 panic("duplicate registered test name " + name)
 907         }
 908         t.tests = append(t.tests, distTest{
 909                 name:    name,
 910                 heading: dirBanner,
 911                 fn: func(dt *distTest) error {
 912                         if seq {
 913                                 t.runPending(dt)
 914                                 timelog("start", name)
 915                                 defer timelog("end", name)
 916                                 return t.dirCmd(filepath.Join(goroot, "src", dirBanner), bin, args).Run()
 917                         }
 918                         t.addCmd(dt, filepath.Join(goroot, "src", dirBanner), bin, args)
 919                         return nil
 920                 },
 921         })
 922 }
 923
 924 func (t *tester) registerTest(name, dirBanner string, cmdline ...interface{}) {
 925         t.registerTest1(false, name, dirBanner, cmdline...)
 926 }
 927
 928 func (t *tester) registerSeqTest(name, dirBanner string, cmdline ...interface{}) {
 929         t.registerTest1(true, name, dirBanner, cmdline...)
 930 }
 931
 932 func (t *tester) bgDirCmd(dir, bin string, args ...string) *exec.Cmd {
 933         cmd := exec.Command(bin, args...)
 934         if filepath.IsAbs(dir) {
 935                 setDir(cmd, dir)
 936         } else {
 937                 setDir(cmd, filepath.Join(goroot, dir))
 938         }
 939         return cmd
 940 }
 941
 942 func (t *tester) dirCmd(dir string, cmdline ...interface{}) *exec.Cmd {
 943         bin, args := flattenCmdline(cmdline)
 944         cmd := t.bgDirCmd(dir, bin, args...)
 945         cmd.Stdout = os.Stdout
 946         cmd.Stderr = os.Stderr
 947         if vflag > 1 {
 948                 errprintf("%s\n", strings.Join(cmd.Args, " "))
 949         }
 950         return cmd
 951 }
 952
 953 // flattenCmdline flattens a mixture of string and []string as single list
 954 // and then interprets it as a command line: first element is binary, then args.
 955 func flattenCmdline(cmdline []interface{}) (bin string, args []string) {
 956         var list []string
 957         for _, x := range cmdline {
 958                 switch x := x.(type) {
 959                 case string:
 960                         list = append(list, x)
 961                 case []string:
 962                         list = append(list, x...)
 963                 default:
 964                         panic("invalid addCmd argument type: " + reflect.TypeOf(x).String())
 965                 }
 966         }
 967
 968         // The go command is too picky about duplicated flags.
 969         // Drop all but the last of the allowed duplicated flags.
 970         drop := make([]bool, len(list))
 971         have := map[string]int{}
 972         for i := 1; i < len(list); i++ {
 973                 j := strings.Index(list[i], "=")
 974                 if j < 0 {
 975                         continue
 976                 }
 977                 flag := list[i][:j]
 978                 switch flag {
 979                 case "-run", "-tags":
 980                         if have[flag] != 0 {
 981                                 drop[have[flag]] = true
 982                         }
 983                         have[flag] = i
 984                 }
 985         }
 986         out := list[:0]
 987         for i, x := range list {
 988                 if !drop[i] {
 989                         out = append(out, x)
 990                 }
 991         }
 992         list = out
 993
 994         bin = list[0]
 995         if bin == "go" {
 996                 bin = gorootBinGo
 997         }
 998         return bin, list[1:]
 999 }
1000
1001 func (t *tester) addCmd(dt *distTest, dir string, cmdline ...interface{}) *exec.Cmd {
1002         bin, args := flattenCmdline(cmdline)
1003         w := &work{
1004                 dt:  dt,
1005                 cmd: t.bgDirCmd(dir, bin, args...),
1006         }
1007         t.worklist = append(t.worklist, w)
1008         return w.cmd
1009 }
1010
1011 func (t *tester) iOS() bool {
1012         return goos == "ios"
1013 }
1014
1015 func (t *tester) out(v string) {
1016         if t.banner == "" {
1017                 return
1018         }
1019         fmt.Println("\n" + t.banner + v)
1020 }
1021
1022 func (t *tester) extLink() bool {
1023         pair := gohostos + "-" + goarch
1024         switch pair {
1025         case "aix-ppc64",
1026                 "android-arm", "android-arm64",
1027                 "darwin-amd64", "darwin-arm64",
1028                 "dragonfly-amd64",
1029                 "freebsd-386", "freebsd-amd64", "freebsd-arm", "freebsd-riscv64",
1030                 "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-loong64", "linux-ppc64le", "linux-mips64", "linux-mips64le", "linux-mips", "linux-mipsle", "linux-riscv64", "linux-s390x",
1031                 "netbsd-386", "netbsd-amd64",
1032                 "openbsd-386", "openbsd-amd64",
1033                 "windows-386", "windows-amd64":
1034                 return true
1035         }
1036         return false
1037 }
1038
1039 func (t *tester) internalLink() bool {
1040         if gohostos == "dragonfly" {
1041                 // linkmode=internal fails on dragonfly since errno is a TLS relocation.
1042                 return false
1043         }
1044         if goos == "android" {
1045                 return false
1046         }
1047         if goos == "ios" {
1048                 return false
1049         }
1050         if goos == "windows" && goarch == "arm64" {
1051                 return false
1052         }
1053         // Internally linking cgo is incomplete on some architectures.
1054         // https://golang.org/issue/10373
1055         // https://golang.org/issue/14449
1056         if goarch == "loong64" || goarch == "mips64" || goarch == "mips64le" || goarch == "mips" || goarch == "mipsle" || goarch == "riscv64" {
1057                 return false
1058         }
1059         if goos == "aix" {
1060                 // linkmode=internal isn't supported.
1061                 return false
1062         }
1063         return true
1064 }
1065
1066 func (t *tester) internalLinkPIE() bool {
1067         switch goos + "-" + goarch {
1068         case "darwin-amd64", "darwin-arm64",
1069                 "linux-amd64", "linux-arm64", "linux-ppc64le",
1070                 "android-arm64",
1071                 "windows-amd64", "windows-386", "windows-arm":
1072                 return true
1073         }
1074         return false
1075 }
1076
1077 func (t *tester) supportedBuildmode(mode string) bool {
1078         pair := goos + "-" + goarch
1079         switch mode {
1080         case "c-archive":
1081                 if !t.extLink() {
1082                         return false
1083                 }
1084                 switch pair {
1085                 case "aix-ppc64",
1086                         "darwin-amd64", "darwin-arm64", "ios-arm64",
1087                         "linux-amd64", "linux-386", "linux-ppc64le", "linux-riscv64", "linux-s390x",
1088                         "freebsd-amd64",
1089                         "windows-amd64", "windows-386":
1090                         return true
1091                 }
1092                 return false
1093         case "c-shared":
1094                 switch pair {
1095                 case "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-riscv64", "linux-s390x",
1096                         "darwin-amd64", "darwin-arm64",
1097                         "freebsd-amd64",
1098                         "android-arm", "android-arm64", "android-386",
1099                         "windows-amd64", "windows-386", "windows-arm64":
1100                         return true
1101                 }
1102                 return false
1103         case "shared":
1104                 switch pair {
1105                 case "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-s390x":
1106                         return true
1107                 }
1108                 return false
1109         case "plugin":
1110                 switch pair {
1111                 case "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-s390x", "linux-ppc64le":
1112                         return true
1113                 case "darwin-amd64", "darwin-arm64":
1114                         return true
1115                 case "freebsd-amd64":
1116                         return true
1117                 }
1118                 return false
1119         case "pie":
1120                 switch pair {
1121                 case "aix/ppc64",
1122                         "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-riscv64", "linux-s390x",
1123                         "android-amd64", "android-arm", "android-arm64", "android-386":
1124                         return true
1125                 case "darwin-amd64", "darwin-arm64":
1126                         return true
1127                 case "windows-amd64", "windows-386", "windows-arm":
1128                         return true
1129                 }
1130                 return false
1131
1132         default:
1133                 fatalf("internal error: unknown buildmode %s", mode)
1134                 return false
1135         }
1136 }
1137
1138 func (t *tester) registerHostTest(name, heading, dir, pkg string) {
1139         t.tests = append(t.tests, distTest{
1140                 name:    name,
1141                 heading: heading,
1142                 fn: func(dt *distTest) error {
1143                         t.runPending(dt)
1144                         timelog("start", name)
1145                         defer timelog("end", name)
1146                         return t.runHostTest(dir, pkg)
1147                 },
1148         })
1149 }
1150
1151 func (t *tester) runHostTest(dir, pkg string) error {
1152         out, err := exec.Command(gorootBinGo, "env", "GOEXE", "GOTMPDIR").Output()
1153         if err != nil {
1154                 return err
1155         }
1156
1157         parts := strings.Split(string(out), "\n")
1158         if len(parts) < 2 {
1159                 return fmt.Errorf("'go env GOEXE GOTMPDIR' output contains <2 lines")
1160         }
1161         GOEXE := strings.TrimSpace(parts[0])
1162         GOTMPDIR := strings.TrimSpace(parts[1])
1163
1164         f, err := os.CreateTemp(GOTMPDIR, "test.test-*"+GOEXE)
1165         if err != nil {
1166                 return err
1167         }
1168         f.Close()
1169         defer os.Remove(f.Name())
1170
1171         cmd := t.dirCmd(dir, t.goTest(), "-c", "-o", f.Name(), pkg)
1172         setEnv(cmd, "GOARCH", gohostarch)
1173         setEnv(cmd, "GOOS", gohostos)
1174         if err := cmd.Run(); err != nil {
1175                 return err
1176         }
1177         return t.dirCmd(dir, f.Name(), "-test.short="+short(), "-test.timeout="+t.timeoutDuration(300).String()).Run()
1178 }
1179
1180 func (t *tester) cgoTest(dt *distTest) error {
1181         cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), ".")
1182         setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=auto")
1183
1184         // Stub out various buildmode=pie tests  on alpine until 54354 resolved.
1185         builderName := os.Getenv("GO_BUILDER_NAME")
1186         disablePIE := strings.HasSuffix(builderName, "-alpine")
1187
1188         if t.internalLink() {
1189                 cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=internal", ".")
1190                 setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=internal")
1191         }
1192
1193         pair := gohostos + "-" + goarch
1194         switch pair {
1195         case "darwin-amd64", "darwin-arm64",
1196                 "windows-386", "windows-amd64":
1197                 // test linkmode=external, but __thread not supported, so skip testtls.
1198                 if !t.extLink() {
1199                         break
1200                 }
1201                 cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), ".")
1202                 setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=external")
1203
1204                 t.addCmd(dt, "misc/cgo/test", t.goTest(), "-ldflags", "-linkmode=external -s", ".")
1205
1206                 if t.supportedBuildmode("pie") && !disablePIE {
1207
1208                         t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", ".")
1209                         if t.internalLink() && t.internalLinkPIE() {
1210                                 t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", "-ldflags=-linkmode=internal", "-tags=internal,internal_pie", ".")
1211                         }
1212                 }
1213
1214         case "aix-ppc64",
1215                 "android-arm", "android-arm64",
1216                 "dragonfly-amd64",
1217                 "freebsd-386", "freebsd-amd64", "freebsd-arm", "freebsd-riscv64",
1218                 "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-riscv64", "linux-s390x",
1219                 "netbsd-386", "netbsd-amd64",
1220                 "openbsd-386", "openbsd-amd64", "openbsd-arm", "openbsd-arm64", "openbsd-mips64":
1221
1222                 cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), ".")
1223                 setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=external")
1224                 // cgo should be able to cope with both -g arguments and colored
1225                 // diagnostics.
1226                 setEnv(cmd, "CGO_CFLAGS", "-g0 -fdiagnostics-color")
1227
1228                 t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-ldflags", "-linkmode=auto", ".")
1229                 t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-ldflags", "-linkmode=external", ".")
1230
1231                 switch pair {
1232                 case "aix-ppc64", "netbsd-386", "netbsd-amd64":
1233                         // no static linking
1234                 case "freebsd-arm":
1235                         // -fPIC compiled tls code will use __tls_get_addr instead
1236                         // of __aeabi_read_tp, however, on FreeBSD/ARM, __tls_get_addr
1237                         // is implemented in rtld-elf, so -fPIC isn't compatible with
1238                         // static linking on FreeBSD/ARM with clang. (cgo depends on
1239                         // -fPIC fundamentally.)
1240                 default:
1241                         cmd := t.dirCmd("misc/cgo/test",
1242                                 compilerEnvLookup(defaultcc, goos, goarch), "-xc", "-o", "/dev/null", "-static", "-")
1243                         cmd.Stdin = strings.NewReader("int main() {}")
1244                         if err := cmd.Run(); err != nil {
1245                                 fmt.Println("No support for static linking found (lacks libc.a?), skip cgo static linking test.")
1246                         } else {
1247                                 if goos != "android" {
1248                                         t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-ldflags", `-linkmode=external -extldflags "-static -pthread"`, ".")
1249                                 }
1250                                 t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), ".")
1251                                 t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), "-ldflags", `-linkmode=external`, ".")
1252                                 if goos != "android" {
1253                                         t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), "-ldflags", `-linkmode=external -extldflags "-static -pthread"`, ".")
1254                                         t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=static", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`, ".")
1255                                         // -static in CGO_LDFLAGS triggers a different code path
1256                                         // than -static in -extldflags, so test both.
1257                                         // See issue #16651.
1258                                         cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=static", ".")
1259                                         setEnv(cmd, "CGO_LDFLAGS", "-static -pthread")
1260                                 }
1261                         }
1262
1263                         if t.supportedBuildmode("pie") && !disablePIE {
1264                                 t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", ".")
1265                                 if t.internalLink() && t.internalLinkPIE() {
1266                                         t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", "-ldflags=-linkmode=internal", "-tags=internal,internal_pie", ".")
1267                                 }
1268                                 t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-buildmode=pie", ".")
1269                                 t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), "-buildmode=pie", ".")
1270                         }
1271                 }
1272         }
1273
1274         return nil
1275 }
1276
1277 // run pending test commands, in parallel, emitting headers as appropriate.
1278 // When finished, emit header for nextTest, which is going to run after the
1279 // pending commands are done (and runPending returns).
1280 // A test should call runPending if it wants to make sure that it is not
1281 // running in parallel with earlier tests, or if it has some other reason
1282 // for needing the earlier tests to be done.
1283 func (t *tester) runPending(nextTest *distTest) {
1284         checkNotStale("go", "std")
1285         worklist := t.worklist
1286         t.worklist = nil
1287         for _, w := range worklist {
1288                 w.start = make(chan bool)
1289                 w.end = make(chan bool)
1290                 go func(w *work) {
1291                         if !<-w.start {
1292                                 timelog("skip", w.dt.name)
1293                                 w.out = []byte(fmt.Sprintf("skipped due to earlier error\n"))
1294                         } else {
1295                                 timelog("start", w.dt.name)
1296                                 w.out, w.err = w.cmd.CombinedOutput()
1297                                 if w.err != nil {
1298                                         if isUnsupportedVMASize(w) {
1299                                                 timelog("skip", w.dt.name)
1300                                                 w.out = []byte(fmt.Sprintf("skipped due to unsupported VMA\n"))
1301                                                 w.err = nil
1302                                         }
1303                                 }
1304                         }
1305                         timelog("end", w.dt.name)
1306                         w.end <- true
1307                 }(w)
1308         }
1309
1310         started := 0
1311         ended := 0
1312         var last *distTest
1313         for ended < len(worklist) {
1314                 for started < len(worklist) && started-ended < maxbg {
1315                         w := worklist[started]
1316                         started++
1317                         w.start <- !t.failed || t.keepGoing
1318                 }
1319                 w := worklist[ended]
1320                 dt := w.dt
1321                 if dt.heading != "" && t.lastHeading != dt.heading {
1322                         t.lastHeading = dt.heading
1323                         t.out(dt.heading)
1324                 }
1325                 if dt != last {
1326                         // Assumes all the entries for a single dt are in one worklist.
1327                         last = w.dt
1328                         if vflag > 0 {
1329                                 fmt.Printf("# go tool dist test -run=^%s$\n", dt.name)
1330                         }
1331                 }
1332                 if vflag > 1 {
1333                         errprintf("%s\n", strings.Join(w.cmd.Args, " "))
1334                 }
1335                 ended++
1336                 <-w.end
1337                 os.Stdout.Write(w.out)
1338                 if w.err != nil {
1339                         log.Printf("Failed: %v", w.err)
1340                         t.failed = true
1341                 }
1342                 checkNotStale("go", "std")
1343         }
1344         if t.failed && !t.keepGoing {
1345                 fatalf("FAILED")
1346         }
1347
1348         if dt := nextTest; dt != nil {
1349                 if dt.heading != "" && t.lastHeading != dt.heading {
1350                         t.lastHeading = dt.heading
1351                         t.out(dt.heading)
1352                 }
1353                 if vflag > 0 {
1354                         fmt.Printf("# go tool dist test -run=^%s$\n", dt.name)
1355                 }
1356         }
1357 }
1358
1359 func (t *tester) hasBash() bool {
1360         switch gohostos {
1361         case "windows", "plan9":
1362                 return false
1363         }
1364         return true
1365 }
1366
1367 func (t *tester) hasCxx() bool {
1368         cxx, _ := exec.LookPath(compilerEnvLookup(defaultcxx, goos, goarch))
1369         return cxx != ""
1370 }
1371
1372 func (t *tester) hasSwig() bool {
1373         swig, err := exec.LookPath("swig")
1374         if err != nil {
1375                 return false
1376         }
1377
1378         // Check that swig was installed with Go support by checking
1379         // that a go directory exists inside the swiglib directory.
1380         // See https://golang.org/issue/23469.
1381         output, err := exec.Command(swig, "-go", "-swiglib").Output()
1382         if err != nil {
1383                 return false
1384         }
1385         swigDir := strings.TrimSpace(string(output))
1386
1387         _, err = os.Stat(filepath.Join(swigDir, "go"))
1388         if err != nil {
1389                 return false
1390         }
1391
1392         // Check that swig has a new enough version.
1393         // See https://golang.org/issue/22858.
1394         out, err := exec.Command(swig, "-version").CombinedOutput()
1395         if err != nil {
1396                 return false
1397         }
1398
1399         re := regexp.MustCompile(`[vV]ersion +(\d+)([.]\d+)?([.]\d+)?`)
1400         matches := re.FindSubmatch(out)
1401         if matches == nil {
1402                 // Can't find version number; hope for the best.
1403                 return true
1404         }
1405
1406         major, err := strconv.Atoi(string(matches[1]))
1407         if err != nil {
1408                 // Can't find version number; hope for the best.
1409                 return true
1410         }
1411         if major < 3 {
1412                 return false
1413         }
1414         if major > 3 {
1415                 // 4.0 or later
1416                 return true
1417         }
1418
1419         // We have SWIG version 3.x.
1420         if len(matches[2]) > 0 {
1421                 minor, err := strconv.Atoi(string(matches[2][1:]))
1422                 if err != nil {
1423                         return true
1424                 }
1425                 if minor > 0 {
1426                         // 3.1 or later
1427                         return true
1428                 }
1429         }
1430
1431         // We have SWIG version 3.0.x.
1432         if len(matches[3]) > 0 {
1433                 patch, err := strconv.Atoi(string(matches[3][1:]))
1434                 if err != nil {
1435                         return true
1436                 }
1437                 if patch < 6 {
1438                         // Before 3.0.6.
1439                         return false
1440                 }
1441         }
1442
1443         return true
1444 }
1445
1446 func (t *tester) raceDetectorSupported() bool {
1447         if gohostos != goos {
1448                 return false
1449         }
1450         if !t.cgoEnabled {
1451                 return false
1452         }
1453         if !raceDetectorSupported(goos, goarch) {
1454                 return false
1455         }
1456         // The race detector doesn't work on Alpine Linux:
1457         // golang.org/issue/14481
1458         if isAlpineLinux() {
1459                 return false
1460         }
1461         // NetBSD support is unfinished.
1462         // golang.org/issue/26403
1463         if goos == "netbsd" {
1464                 return false
1465         }
1466         return true
1467 }
1468
1469 func isAlpineLinux() bool {
1470         if runtime.GOOS != "linux" {
1471                 return false
1472         }
1473         fi, err := os.Lstat("/etc/alpine-release")
1474         return err == nil && fi.Mode().IsRegular()
1475 }
1476
1477 func (t *tester) runFlag(rx string) string {
1478         if t.compileOnly {
1479                 return "-run=^$"
1480         }
1481         return "-run=" + rx
1482 }
1483
1484 func (t *tester) raceTest(dt *distTest) error {
1485         t.addCmd(dt, "src", t.goTest(), "-race", t.runFlag("Output"), "runtime/race")
1486         t.addCmd(dt, "src", t.goTest(), "-race", t.runFlag("TestParse|TestEcho|TestStdinCloseRace|TestClosedPipeRace|TestTypeRace|TestFdRace|TestFdReadRace|TestFileCloseRace"), "flag", "net", "os", "os/exec", "encoding/gob")
1487         // We don't want the following line, because it
1488         // slows down all.bash (by 10 seconds on my laptop).
1489         // The race builder should catch any error here, but doesn't.
1490         // TODO(iant): Figure out how to catch this.
1491         // t.addCmd(dt, "src", t.goTest(),  "-race", "-run=TestParallelTest", "cmd/go")
1492         if t.cgoEnabled {
1493                 // Building misc/cgo/test takes a long time.
1494                 // There are already cgo-enabled packages being tested with the race detector.
1495                 // We shouldn't need to redo all of misc/cgo/test too.
1496                 // The race buildler will take care of this.
1497                 // cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), "-race")
1498                 // setEnv(cmd, "GOTRACEBACK", "2")
1499         }
1500         if t.extLink() {
1501                 // Test with external linking; see issue 9133.
1502                 t.addCmd(dt, "src", t.goTest(), "-race", "-ldflags=-linkmode=external", t.runFlag("TestParse|TestEcho|TestStdinCloseRace"), "flag", "os/exec")
1503         }
1504         return nil
1505 }
1506
1507 var runtest struct {
1508         sync.Once
1509         exe string
1510         err error
1511 }
1512
1513 func (t *tester) testDirTest(dt *distTest, shard, shards int) error {
1514         runtest.Do(func() {
1515                 f, err := os.CreateTemp("", "runtest-*.exe") // named exe for Windows, but harmless elsewhere
1516                 if err != nil {
1517                         runtest.err = err
1518                         return
1519                 }
1520                 f.Close()
1521
1522                 runtest.exe = f.Name()
1523                 xatexit(func() {
1524                         os.Remove(runtest.exe)
1525                 })
1526
1527                 cmd := t.dirCmd("test", "go", "build", "-o", runtest.exe, "run.go")
1528                 setEnv(cmd, "GOOS", gohostos)
1529                 setEnv(cmd, "GOARCH", gohostarch)
1530                 runtest.err = cmd.Run()
1531         })
1532         if runtest.err != nil {
1533                 return runtest.err
1534         }
1535         if t.compileOnly {
1536                 return nil
1537         }
1538         t.addCmd(dt, "test", runtest.exe,
1539                 fmt.Sprintf("--shard=%d", shard),
1540                 fmt.Sprintf("--shards=%d", shards),
1541         )
1542         return nil
1543 }
1544
1545 // cgoPackages is the standard packages that use cgo.
1546 var cgoPackages = []string{
1547         "net",
1548         "os/user",
1549 }
1550
1551 var funcBenchmark = []byte("\nfunc Benchmark")
1552
1553 // packageHasBenchmarks reports whether pkg has benchmarks.
1554 // On any error, it conservatively returns true.
1555 //
1556 // This exists just to eliminate work on the builders, since compiling
1557 // a test in race mode just to discover it has no benchmarks costs a
1558 // second or two per package, and this function returns false for
1559 // about 100 packages.
1560 func (t *tester) packageHasBenchmarks(pkg string) bool {
1561         pkgDir := filepath.Join(goroot, "src", pkg)
1562         d, err := os.Open(pkgDir)
1563         if err != nil {
1564                 return true // conservatively
1565         }
1566         defer d.Close()
1567         names, err := d.Readdirnames(-1)
1568         if err != nil {
1569                 return true // conservatively
1570         }
1571         for _, name := range names {
1572                 if !strings.HasSuffix(name, "_test.go") {
1573                         continue
1574                 }
1575                 slurp, err := os.ReadFile(filepath.Join(pkgDir, name))
1576                 if err != nil {
1577                         return true // conservatively
1578                 }
1579                 if bytes.Contains(slurp, funcBenchmark) {
1580                         return true
1581                 }
1582         }
1583         return false
1584 }
1585
1586 // makeGOROOTUnwritable makes all $GOROOT files & directories non-writable to
1587 // check that no tests accidentally write to $GOROOT.
1588 func (t *tester) makeGOROOTUnwritable() (undo func()) {
1589         dir := os.Getenv("GOROOT")
1590         if dir == "" {
1591                 panic("GOROOT not set")
1592         }
1593
1594         type pathMode struct {
1595                 path string
1596                 mode os.FileMode
1597         }
1598         var dirs []pathMode // in lexical order
1599
1600         undo = func() {
1601                 for i := range dirs {
1602                         os.Chmod(dirs[i].path, dirs[i].mode) // best effort
1603                 }
1604         }
1605
1606         gocache := os.Getenv("GOCACHE")
1607         if gocache == "" {
1608                 panic("GOCACHE not set")
1609         }
1610         gocacheSubdir, _ := filepath.Rel(dir, gocache)
1611
1612         // Note: Can't use WalkDir here, because this has to compile with Go 1.4.
1613         filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
1614                 if suffix := strings.TrimPrefix(path, dir+string(filepath.Separator)); suffix != "" {
1615                         if suffix == gocacheSubdir {
1616                                 // Leave GOCACHE writable: we may need to write test binaries into it.
1617                                 return filepath.SkipDir
1618                         }
1619                         if suffix == ".git" {
1620                                 // Leave Git metadata in whatever state it was in. It may contain a lot
1621                                 // of files, and it is highly unlikely that a test will try to modify
1622                                 // anything within that directory.
1623                                 return filepath.SkipDir
1624                         }
1625                 }
1626                 if err == nil {
1627                         mode := info.Mode()
1628                         if mode&0222 != 0 && (mode.IsDir() || mode.IsRegular()) {
1629                                 dirs = append(dirs, pathMode{path, mode})
1630                         }
1631                 }
1632                 return nil
1633         })
1634
1635         // Run over list backward to chmod children before parents.
1636         for i := len(dirs) - 1; i >= 0; i-- {
1637                 err := os.Chmod(dirs[i].path, dirs[i].mode&^0222)
1638                 if err != nil {
1639                         dirs = dirs[i:] // Only undo what we did so far.
1640                         undo()
1641                         fatalf("failed to make GOROOT read-only: %v", err)
1642                 }
1643         }
1644
1645         return undo
1646 }
1647
1648 // raceDetectorSupported is a copy of the function
1649 // internal/platform.RaceDetectorSupported, which can't be used here
1650 // because cmd/dist has to be buildable by Go 1.4.
1651 // The race detector only supports 48-bit VMA on arm64. But we don't have
1652 // a good solution to check VMA size(See https://golang.org/issue/29948)
1653 // raceDetectorSupported will always return true for arm64. But race
1654 // detector tests may abort on non 48-bit VMA configuration, the tests
1655 // will be marked as "skipped" in this case.
1656 func raceDetectorSupported(goos, goarch string) bool {
1657         switch goos {
1658         case "linux":
1659                 return goarch == "amd64" || goarch == "ppc64le" || goarch == "arm64" || goarch == "s390x"
1660         case "darwin":
1661                 return goarch == "amd64" || goarch == "arm64"
1662         case "freebsd", "netbsd", "openbsd", "windows":
1663                 return goarch == "amd64"
1664         default:
1665                 return false
1666         }
1667 }
1668
1669 // isUnsupportedVMASize reports whether the failure is caused by an unsupported
1670 // VMA for the race detector (for example, running the race detector on an
1671 // arm64 machine configured with 39-bit VMA)
1672 func isUnsupportedVMASize(w *work) bool {
1673         unsupportedVMA := []byte("unsupported VMA range")
1674         return w.dt.name == "race" && bytes.Contains(w.out, unsupportedVMA)
1675 }
1676
1677 // isEnvSet reports whether the environment variable evar is
1678 // set in the environment.
1679 func isEnvSet(evar string) bool {
1680         evarEq := evar + "="
1681         for _, e := range os.Environ() {
1682                 if strings.HasPrefix(e, evarEq) {
1683                         return true
1684                 }
1685         }
1686         return false
1687 }