src/cmd/dist/test.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "flag"
  10         "fmt"
  11         "log"
  12         "os"
  13         "os/exec"
  14         "path/filepath"
  15         "reflect"
  16         "regexp"
  17         "runtime"
  18         "strconv"
  19         "strings"
  20         "sync"
  21         "time"
  22 )
  23
  24 func cmdtest() {
  25         gogcflags = os.Getenv("GO_GCFLAGS")
  26         setNoOpt()
  27
  28         var t tester
  29
  30         var noRebuild bool
  31         flag.BoolVar(&t.listMode, "list", false, "list available tests")
  32         flag.BoolVar(&t.rebuild, "rebuild", false, "rebuild everything first")
  33         flag.BoolVar(&noRebuild, "no-rebuild", false, "overrides -rebuild (historical dreg)")
  34         flag.BoolVar(&t.keepGoing, "k", false, "keep going even when error occurred")
  35         flag.BoolVar(&t.race, "race", false, "run in race builder mode (different set of tests)")
  36         flag.BoolVar(&t.compileOnly, "compile-only", false, "compile tests, but don't run them. This is for some builders. Not all dist tests respect this flag, but most do.")
  37         flag.StringVar(&t.banner, "banner", "##### ", "banner prefix; blank means no section banners")
  38         flag.StringVar(&t.runRxStr, "run", os.Getenv("GOTESTONLY"),
  39                 "run only those tests matching the regular expression; empty means to run all. "+
  40                         "Special exception: if the string begins with '!', the match is inverted.")
  41         flag.BoolVar(&t.msan, "msan", false, "run in memory sanitizer builder mode")
  42         flag.BoolVar(&t.asan, "asan", false, "run in address sanitizer builder mode")
  43
  44         xflagparse(-1) // any number of args
  45         if noRebuild {
  46                 t.rebuild = false
  47         }
  48
  49         t.run()
  50 }
  51
  52 // tester executes cmdtest.
  53 type tester struct {
  54         race        bool
  55         msan        bool
  56         asan        bool
  57         listMode    bool
  58         rebuild     bool
  59         failed      bool
  60         keepGoing   bool
  61         compileOnly bool // just try to compile all tests, but no need to run
  62         runRxStr    string
  63         runRx       *regexp.Regexp
  64         runRxWant   bool     // want runRx to match (true) or not match (false)
  65         runNames    []string // tests to run, exclusive with runRx; empty means all
  66         banner      string   // prefix, or "" for none
  67         lastHeading string   // last dir heading printed
  68
  69         cgoEnabled bool
  70         partial    bool
  71         haveTime   bool // the 'time' binary is available
  72
  73         tests        []distTest
  74         timeoutScale int
  75
  76         worklist []*work
  77 }
  78
  79 type work struct {
  80         dt    *distTest
  81         cmd   *exec.Cmd
  82         start chan bool
  83         out   []byte
  84         err   error
  85         end   chan bool
  86 }
  87
  88 // A distTest is a test run by dist test.
  89 // Each test has a unique name and belongs to a group (heading)
  90 type distTest struct {
  91         name    string // unique test name; may be filtered with -run flag
  92         heading string // group section; this header is printed before the test is run.
  93         fn      func(*distTest) error
  94 }
  95
  96 func (t *tester) run() {
  97         timelog("start", "dist test")
  98
  99         os.Setenv("PATH", fmt.Sprintf("%s%c%s", gorootBin, os.PathListSeparator, os.Getenv("PATH")))
 100
 101         cmd := exec.Command(gorootBinGo, "env", "CGO_ENABLED")
 102         cmd.Stderr = new(bytes.Buffer)
 103         slurp, err := cmd.Output()
 104         if err != nil {
 105                 fatalf("Error running go env CGO_ENABLED: %v\n%s", err, cmd.Stderr)
 106         }
 107         t.cgoEnabled, _ = strconv.ParseBool(strings.TrimSpace(string(slurp)))
 108         if flag.NArg() > 0 && t.runRxStr != "" {
 109                 fatalf("the -run regular expression flag is mutually exclusive with test name arguments")
 110         }
 111
 112         t.runNames = flag.Args()
 113
 114         if t.hasBash() {
 115                 if _, err := exec.LookPath("time"); err == nil {
 116                         t.haveTime = true
 117                 }
 118         }
 119
 120         // Set GOTRACEBACK to system if the user didn't set a level explicitly.
 121         // Since we're running tests for Go, we want as much detail as possible
 122         // if something goes wrong.
 123         //
 124         // Set it before running any commands just in case something goes wrong.
 125         if ok := isEnvSet("GOTRACEBACK"); !ok {
 126                 if err := os.Setenv("GOTRACEBACK", "system"); err != nil {
 127                         if t.keepGoing {
 128                                 log.Printf("Failed to set GOTRACEBACK: %v", err)
 129                         } else {
 130                                 fatalf("Failed to set GOTRACEBACK: %v", err)
 131                         }
 132                 }
 133         }
 134
 135         if t.rebuild {
 136                 t.out("Building packages and commands.")
 137                 // Force rebuild the whole toolchain.
 138                 goInstall("go", append([]string{"-a"}, toolchain...)...)
 139         }
 140
 141         if !t.listMode {
 142                 if os.Getenv("GO_BUILDER_NAME") == "" {
 143                         // Complete rebuild bootstrap, even with -no-rebuild.
 144                         // If everything is up-to-date, this is a no-op.
 145                         // If everything is not up-to-date, the first checkNotStale
 146                         // during the test process will kill the tests, so we might
 147                         // as well install the world.
 148                         // Now that for example "go install cmd/compile" does not
 149                         // also install runtime (you need "go install -i cmd/compile"
 150                         // for that), it's easy for previous workflows like
 151                         // "rebuild the compiler and then run run.bash"
 152                         // to break if we don't automatically refresh things here.
 153                         // Rebuilding is a shortened bootstrap.
 154                         // See cmdbootstrap for a description of the overall process.
 155                         goInstall("go", toolchain...)
 156                         goInstall("go", toolchain...)
 157                         goInstall("go", "std", "cmd")
 158                 } else {
 159                         // The Go builder infrastructure should always begin running tests from a
 160                         // clean, non-stale state, so there is no need to rebuild the world.
 161                         // Instead, we can just check that it is not stale, which may be less
 162                         // expensive (and is also more likely to catch bugs in the builder
 163                         // implementation).
 164                         checkNotStale("go", "std", "cmd")
 165                 }
 166         }
 167
 168         t.timeoutScale = 1
 169         switch goarch {
 170         case "arm":
 171                 t.timeoutScale = 2
 172         case "mips", "mipsle", "mips64", "mips64le":
 173                 t.timeoutScale = 4
 174         }
 175         if s := os.Getenv("GO_TEST_TIMEOUT_SCALE"); s != "" {
 176                 t.timeoutScale, err = strconv.Atoi(s)
 177                 if err != nil {
 178                         fatalf("failed to parse $GO_TEST_TIMEOUT_SCALE = %q as integer: %v", s, err)
 179                 }
 180         }
 181
 182         if t.runRxStr != "" {
 183                 if t.runRxStr[0] == '!' {
 184                         t.runRxWant = false
 185                         t.runRxStr = t.runRxStr[1:]
 186                 } else {
 187                         t.runRxWant = true
 188                 }
 189                 t.runRx = regexp.MustCompile(t.runRxStr)
 190         }
 191
 192         t.registerTests()
 193         if t.listMode {
 194                 for _, tt := range t.tests {
 195                         fmt.Println(tt.name)
 196                 }
 197                 return
 198         }
 199
 200         for _, name := range t.runNames {
 201                 if !t.isRegisteredTestName(name) {
 202                         fatalf("unknown test %q", name)
 203                 }
 204         }
 205
 206         // On a few builders, make GOROOT unwritable to catch tests writing to it.
 207         if strings.HasPrefix(os.Getenv("GO_BUILDER_NAME"), "linux-") {
 208                 if os.Getuid() == 0 {
 209                         // Don't bother making GOROOT unwritable:
 210                         // we're running as root, so permissions would have no effect.
 211                 } else {
 212                         xatexit(t.makeGOROOTUnwritable())
 213                 }
 214         }
 215
 216         if err := t.maybeLogMetadata(); err != nil {
 217                 t.failed = true
 218                 if t.keepGoing {
 219                         log.Printf("Failed logging metadata: %v", err)
 220                 } else {
 221                         fatalf("Failed logging metadata: %v", err)
 222                 }
 223         }
 224
 225         for _, dt := range t.tests {
 226                 if !t.shouldRunTest(dt.name) {
 227                         t.partial = true
 228                         continue
 229                 }
 230                 dt := dt // dt used in background after this iteration
 231                 if err := dt.fn(&dt); err != nil {
 232                         t.runPending(&dt) // in case that hasn't been done yet
 233                         t.failed = true
 234                         if t.keepGoing {
 235                                 log.Printf("Failed: %v", err)
 236                         } else {
 237                                 fatalf("Failed: %v", err)
 238                         }
 239                 }
 240         }
 241         t.runPending(nil)
 242         timelog("end", "dist test")
 243
 244         if t.failed {
 245                 fmt.Println("\nFAILED")
 246                 xexit(1)
 247         } else if incomplete[goos+"/"+goarch] {
 248                 // The test succeeded, but consider it as failed so we don't
 249                 // forget to remove the port from the incomplete map once the
 250                 // port is complete.
 251                 fmt.Println("\nFAILED (incomplete port)")
 252                 xexit(1)
 253         } else if t.partial {
 254                 fmt.Println("\nALL TESTS PASSED (some were excluded)")
 255         } else {
 256                 fmt.Println("\nALL TESTS PASSED")
 257         }
 258 }
 259
 260 func (t *tester) shouldRunTest(name string) bool {
 261         if t.runRx != nil {
 262                 return t.runRx.MatchString(name) == t.runRxWant
 263         }
 264         if len(t.runNames) == 0 {
 265                 return true
 266         }
 267         for _, runName := range t.runNames {
 268                 if runName == name {
 269                         return true
 270                 }
 271         }
 272         return false
 273 }
 274
 275 func (t *tester) maybeLogMetadata() error {
 276         if t.compileOnly {
 277                 // We need to run a subprocess to log metadata. Don't do that
 278                 // on compile-only runs.
 279                 return nil
 280         }
 281         t.out("Test execution environment.")
 282         // Helper binary to print system metadata (CPU model, etc). This is a
 283         // separate binary from dist so it need not build with the bootstrap
 284         // toolchain.
 285         //
 286         // TODO(prattmic): If we split dist bootstrap and dist test then this
 287         // could be simplified to directly use internal/sysinfo here.
 288         return t.dirCmd(filepath.Join(goroot, "src/cmd/internal/metadata"), "go", []string{"run", "main.go"}).Run()
 289 }
 290
 291 // short returns a -short flag value to use with 'go test'
 292 // or a test binary for tests intended to run in short mode.
 293 // It returns "true", unless the environment variable
 294 // GO_TEST_SHORT is set to a non-empty, false-ish string.
 295 //
 296 // This environment variable is meant to be an internal
 297 // detail between the Go build system and cmd/dist for
 298 // the purpose of longtest builders, and is not intended
 299 // for use by users. See golang.org/issue/12508.
 300 func short() string {
 301         if v := os.Getenv("GO_TEST_SHORT"); v != "" {
 302                 short, err := strconv.ParseBool(v)
 303                 if err != nil {
 304                         fatalf("invalid GO_TEST_SHORT %q: %v", v, err)
 305                 }
 306                 if !short {
 307                         return "false"
 308                 }
 309         }
 310         return "true"
 311 }
 312
 313 // goTest returns the beginning of the go test command line.
 314 // Callers should use goTest and then pass flags overriding these
 315 // defaults as later arguments in the command line.
 316 func (t *tester) goTest() []string {
 317         return []string{
 318                 "go", "test", "-short=" + short(), "-count=1", t.tags(), t.runFlag(""),
 319         }
 320 }
 321
 322 func (t *tester) tags() string {
 323         ios := t.iOS()
 324         switch {
 325         case ios && noOpt:
 326                 return "-tags=lldb,noopt"
 327         case ios:
 328                 return "-tags=lldb"
 329         case noOpt:
 330                 return "-tags=noopt"
 331         default:
 332                 return "-tags="
 333         }
 334 }
 335
 336 // timeoutDuration converts the provided number of seconds into a
 337 // time.Duration, scaled by the t.timeoutScale factor.
 338 func (t *tester) timeoutDuration(sec int) time.Duration {
 339         return time.Duration(sec) * time.Second * time.Duration(t.timeoutScale)
 340 }
 341
 342 // timeout returns the "-timeout=" string argument to "go test" given
 343 // the number of seconds of timeout. It scales it by the
 344 // t.timeoutScale factor.
 345 func (t *tester) timeout(sec int) string {
 346         return "-timeout=" + t.timeoutDuration(sec).String()
 347 }
 348
 349 // ranGoTest and stdMatches are state closed over by the stdlib
 350 // testing func in registerStdTest below. The tests are run
 351 // sequentially, so there's no need for locks.
 352 //
 353 // ranGoBench and benchMatches are the same, but are only used
 354 // in -race mode.
 355 var (
 356         ranGoTest  bool
 357         stdMatches []string
 358
 359         ranGoBench   bool
 360         benchMatches []string
 361 )
 362
 363 func (t *tester) registerStdTest(pkg string) {
 364         heading := "Testing packages."
 365         testPrefix := "go_test:"
 366         gcflags := gogcflags
 367
 368         testName := testPrefix + pkg
 369         if t.runRx == nil || t.runRx.MatchString(testName) == t.runRxWant {
 370                 stdMatches = append(stdMatches, pkg)
 371         }
 372
 373         t.tests = append(t.tests, distTest{
 374                 name:    testName,
 375                 heading: heading,
 376                 fn: func(dt *distTest) error {
 377                         if ranGoTest {
 378                                 return nil
 379                         }
 380                         t.runPending(dt)
 381                         timelog("start", dt.name)
 382                         defer timelog("end", dt.name)
 383                         ranGoTest = true
 384
 385                         timeoutSec := 180
 386                         for _, pkg := range stdMatches {
 387                                 if pkg == "cmd/go" {
 388                                         timeoutSec *= 3
 389                                         break
 390                                 }
 391                         }
 392                         args := []string{
 393                                 "test",
 394                                 "-short=" + short(),
 395                                 t.tags(),
 396                                 t.timeout(timeoutSec),
 397                         }
 398                         if gcflags != "" {
 399                                 args = append(args, "-gcflags=all="+gcflags)
 400                         }
 401                         if t.race {
 402                                 args = append(args, "-race")
 403                         }
 404                         if t.msan {
 405                                 args = append(args, "-msan")
 406                         }
 407                         if t.asan {
 408                                 args = append(args, "-asan")
 409                         }
 410                         if t.compileOnly {
 411                                 args = append(args, "-run=^$")
 412                         }
 413                         args = append(args, stdMatches...)
 414                         cmd := exec.Command(gorootBinGo, args...)
 415                         cmd.Stdout = os.Stdout
 416                         cmd.Stderr = os.Stderr
 417                         return cmd.Run()
 418                 },
 419         })
 420 }
 421
 422 func (t *tester) registerRaceBenchTest(pkg string) {
 423         testName := "go_test_bench:" + pkg
 424         if t.runRx == nil || t.runRx.MatchString(testName) == t.runRxWant {
 425                 benchMatches = append(benchMatches, pkg)
 426         }
 427         t.tests = append(t.tests, distTest{
 428                 name:    testName,
 429                 heading: "Running benchmarks briefly.",
 430                 fn: func(dt *distTest) error {
 431                         if ranGoBench {
 432                                 return nil
 433                         }
 434                         t.runPending(dt)
 435                         timelog("start", dt.name)
 436                         defer timelog("end", dt.name)
 437                         ranGoBench = true
 438                         args := []string{
 439                                 "test",
 440                                 "-short=" + short(),
 441                                 "-race",
 442                                 t.timeout(1200), // longer timeout for race with benchmarks
 443                                 "-run=^$",       // nothing. only benchmarks.
 444                                 "-benchtime=.1s",
 445                                 "-cpu=4",
 446                         }
 447                         if !t.compileOnly {
 448                                 args = append(args, "-bench=.*")
 449                         }
 450                         args = append(args, benchMatches...)
 451                         cmd := exec.Command(gorootBinGo, args...)
 452                         cmd.Stdout = os.Stdout
 453                         cmd.Stderr = os.Stderr
 454                         return cmd.Run()
 455                 },
 456         })
 457 }
 458
 459 // stdOutErrAreTerminals is defined in test_linux.go, to report
 460 // whether stdout & stderr are terminals.
 461 var stdOutErrAreTerminals func() bool
 462
 463 func (t *tester) registerTests() {
 464         // Fast path to avoid the ~1 second of `go list std cmd` when
 465         // the caller lists specific tests to run. (as the continuous
 466         // build coordinator does).
 467         if len(t.runNames) > 0 {
 468                 for _, name := range t.runNames {
 469                         if strings.HasPrefix(name, "go_test:") {
 470                                 t.registerStdTest(strings.TrimPrefix(name, "go_test:"))
 471                         }
 472                         if strings.HasPrefix(name, "go_test_bench:") {
 473                                 t.registerRaceBenchTest(strings.TrimPrefix(name, "go_test_bench:"))
 474                         }
 475                 }
 476         } else {
 477                 // Use a format string to only list packages and commands that have tests.
 478                 const format = "{{if (or .TestGoFiles .XTestGoFiles)}}{{.ImportPath}}{{end}}"
 479                 cmd := exec.Command(gorootBinGo, "list", "-f", format)
 480                 if t.race {
 481                         cmd.Args = append(cmd.Args, "-tags=race")
 482                 }
 483                 cmd.Args = append(cmd.Args, "std", "cmd")
 484                 cmd.Stderr = new(bytes.Buffer)
 485                 all, err := cmd.Output()
 486                 if err != nil {
 487                         fatalf("Error running go list std cmd: %v:\n%s", err, cmd.Stderr)
 488                 }
 489                 pkgs := strings.Fields(string(all))
 490                 for _, pkg := range pkgs {
 491                         t.registerStdTest(pkg)
 492                 }
 493                 if t.race {
 494                         for _, pkg := range pkgs {
 495                                 if t.packageHasBenchmarks(pkg) {
 496                                         t.registerRaceBenchTest(pkg)
 497                                 }
 498                         }
 499                 }
 500         }
 501
 502         // Test the os/user package in the pure-Go mode too.
 503         if !t.compileOnly {
 504                 t.tests = append(t.tests, distTest{
 505                         name:    "osusergo",
 506                         heading: "os/user with tag osusergo",
 507                         fn: func(dt *distTest) error {
 508                                 t.addCmd(dt, "src", t.goTest(), t.timeout(300), "-tags=osusergo", "os/user")
 509                                 return nil
 510                         },
 511                 })
 512         }
 513
 514         // Test ios/amd64 for the iOS simulator.
 515         if goos == "darwin" && goarch == "amd64" && t.cgoEnabled {
 516                 t.tests = append(t.tests, distTest{
 517                         name:    "amd64ios",
 518                         heading: "GOOS=ios on darwin/amd64",
 519                         fn: func(dt *distTest) error {
 520                                 cmd := t.addCmd(dt, "src", t.goTest(), t.timeout(300), "-run=SystemRoots", "crypto/x509")
 521                                 setEnv(cmd, "GOOS", "ios")
 522                                 setEnv(cmd, "CGO_ENABLED", "1")
 523                                 return nil
 524                         },
 525                 })
 526         }
 527
 528         if t.race {
 529                 return
 530         }
 531
 532         // Runtime CPU tests.
 533         if !t.compileOnly && goos != "js" { // js can't handle -cpu != 1
 534                 testName := "runtime:cpu124"
 535                 t.tests = append(t.tests, distTest{
 536                         name:    testName,
 537                         heading: "GOMAXPROCS=2 runtime -cpu=1,2,4 -quick",
 538                         fn: func(dt *distTest) error {
 539                                 cmd := t.addCmd(dt, "src", t.goTest(), "-short=true", t.timeout(300), "runtime", "-cpu=1,2,4", "-quick")
 540                                 // We set GOMAXPROCS=2 in addition to -cpu=1,2,4 in order to test runtime bootstrap code,
 541                                 // creation of first goroutines and first garbage collections in the parallel setting.
 542                                 setEnv(cmd, "GOMAXPROCS", "2")
 543                                 return nil
 544                         },
 545                 })
 546         }
 547
 548         // morestack tests. We only run these on in long-test mode
 549         // (with GO_TEST_SHORT=false) because the runtime test is
 550         // already quite long and mayMoreStackMove makes it about
 551         // twice as slow.
 552         if !t.compileOnly && short() == "false" {
 553                 // hooks is the set of maymorestack hooks to test with.
 554                 hooks := []string{"mayMoreStackPreempt", "mayMoreStackMove"}
 555                 // pkgs is the set of test packages to run.
 556                 pkgs := []string{"runtime", "reflect", "sync"}
 557                 // hookPkgs is the set of package patterns to apply
 558                 // the maymorestack hook to.
 559                 hookPkgs := []string{"runtime/...", "reflect", "sync"}
 560                 // unhookPkgs is the set of package patterns to
 561                 // exclude from hookPkgs.
 562                 unhookPkgs := []string{"runtime/testdata/..."}
 563                 for _, hook := range hooks {
 564                         // Construct the build flags to use the
 565                         // maymorestack hook in the compiler and
 566                         // assembler. We pass this via the GOFLAGS
 567                         // environment variable so that it applies to
 568                         // both the test itself and to binaries built
 569                         // by the test.
 570                         goFlagsList := []string{}
 571                         for _, flag := range []string{"-gcflags", "-asmflags"} {
 572                                 for _, hookPkg := range hookPkgs {
 573                                         goFlagsList = append(goFlagsList, flag+"="+hookPkg+"=-d=maymorestack=runtime."+hook)
 574                                 }
 575                                 for _, unhookPkg := range unhookPkgs {
 576                                         goFlagsList = append(goFlagsList, flag+"="+unhookPkg+"=")
 577                                 }
 578                         }
 579                         goFlags := strings.Join(goFlagsList, " ")
 580
 581                         for _, pkg := range pkgs {
 582                                 pkg := pkg
 583                                 testName := hook + ":" + pkg
 584                                 t.tests = append(t.tests, distTest{
 585                                         name:    testName,
 586                                         heading: "maymorestack=" + hook,
 587                                         fn: func(dt *distTest) error {
 588                                                 cmd := t.addCmd(dt, "src", t.goTest(), t.timeout(600), pkg, "-short")
 589                                                 setEnv(cmd, "GOFLAGS", goFlags)
 590                                                 return nil
 591                                         },
 592                                 })
 593                         }
 594                 }
 595         }
 596
 597         // This test needs its stdout/stderr to be terminals, so we don't run it from cmd/go's tests.
 598         // See issue 18153.
 599         if goos == "linux" {
 600                 t.tests = append(t.tests, distTest{
 601                         name:    "cmd_go_test_terminal",
 602                         heading: "cmd/go terminal test",
 603                         fn: func(dt *distTest) error {
 604                                 t.runPending(dt)
 605                                 timelog("start", dt.name)
 606                                 defer timelog("end", dt.name)
 607                                 if !stdOutErrAreTerminals() {
 608                                         fmt.Println("skipping terminal test; stdout/stderr not terminals")
 609                                         return nil
 610                                 }
 611                                 cmd := exec.Command(gorootBinGo, "test")
 612                                 setDir(cmd, filepath.Join(os.Getenv("GOROOT"), "src/cmd/go/testdata/testterminal18153"))
 613                                 cmd.Stdout = os.Stdout
 614                                 cmd.Stderr = os.Stderr
 615                                 return cmd.Run()
 616                         },
 617                 })
 618         }
 619
 620         // On the builders only, test that a moved GOROOT still works.
 621         // Fails on iOS because CC_FOR_TARGET refers to clangwrap.sh
 622         // in the unmoved GOROOT.
 623         // Fails on Android and js/wasm with an exec format error.
 624         // Fails on plan9 with "cannot find GOROOT" (issue #21016).
 625         if os.Getenv("GO_BUILDER_NAME") != "" && goos != "android" && !t.iOS() && goos != "plan9" && goos != "js" {
 626                 t.tests = append(t.tests, distTest{
 627                         name:    "moved_goroot",
 628                         heading: "moved GOROOT",
 629                         fn: func(dt *distTest) error {
 630                                 t.runPending(dt)
 631                                 timelog("start", dt.name)
 632                                 defer timelog("end", dt.name)
 633                                 moved := goroot + "-moved"
 634                                 if err := os.Rename(goroot, moved); err != nil {
 635                                         if goos == "windows" {
 636                                                 // Fails on Windows (with "Access is denied") if a process
 637                                                 // or binary is in this directory. For instance, using all.bat
 638                                                 // when run from c:\workdir\go\src fails here
 639                                                 // if GO_BUILDER_NAME is set. Our builders invoke tests
 640                                                 // a different way which happens to work when sharding
 641                                                 // tests, but we should be tolerant of the non-sharded
 642                                                 // all.bat case.
 643                                                 log.Printf("skipping test on Windows")
 644                                                 return nil
 645                                         }
 646                                         return err
 647                                 }
 648
 649                                 // Run `go test fmt` in the moved GOROOT, without explicitly setting
 650                                 // GOROOT in the environment. The 'go' command should find itself.
 651                                 cmd := exec.Command(filepath.Join(moved, "bin", "go"), "test", "fmt")
 652                                 cmd.Stdout = os.Stdout
 653                                 cmd.Stderr = os.Stderr
 654                                 unsetEnv(cmd, "GOROOT")
 655                                 unsetEnv(cmd, "GOCACHE") // TODO(bcmills): ...why‽
 656                                 err := cmd.Run()
 657
 658                                 if rerr := os.Rename(moved, goroot); rerr != nil {
 659                                         fatalf("failed to restore GOROOT: %v", rerr)
 660                                 }
 661                                 return err
 662                         },
 663                 })
 664         }
 665
 666         // Test that internal linking of standard packages does not
 667         // require libgcc. This ensures that we can install a Go
 668         // release on a system that does not have a C compiler
 669         // installed and still build Go programs (that don't use cgo).
 670         for _, pkg := range cgoPackages {
 671                 if !t.internalLink() {
 672                         break
 673                 }
 674
 675                 // ARM libgcc may be Thumb, which internal linking does not support.
 676                 if goarch == "arm" {
 677                         break
 678                 }
 679
 680                 pkg := pkg
 681                 var run string
 682                 if pkg == "net" {
 683                         run = "TestTCPStress"
 684                 }
 685                 t.tests = append(t.tests, distTest{
 686                         name:    "nolibgcc:" + pkg,
 687                         heading: "Testing without libgcc.",
 688                         fn: func(dt *distTest) error {
 689                                 // What matters is that the tests build and start up.
 690                                 // Skip expensive tests, especially x509 TestSystemRoots.
 691                                 t.addCmd(dt, "src", t.goTest(), "-ldflags=-linkmode=internal -libgcc=none", "-run=^Test[^CS]", pkg, t.runFlag(run))
 692                                 return nil
 693                         },
 694                 })
 695         }
 696
 697         // Stub out following test on alpine until 54354 resolved.
 698         builderName := os.Getenv("GO_BUILDER_NAME")
 699         disablePIE := strings.HasSuffix(builderName, "-alpine")
 700
 701         // Test internal linking of PIE binaries where it is supported.
 702         if t.internalLinkPIE() && !disablePIE {
 703                 t.tests = append(t.tests, distTest{
 704                         name:    "pie_internal",
 705                         heading: "internal linking of -buildmode=pie",
 706                         fn: func(dt *distTest) error {
 707                                 cmd := t.addCmd(dt, "src", t.goTest(), "reflect", "-buildmode=pie", "-ldflags=-linkmode=internal", t.timeout(60))
 708                                 setEnv(cmd, "CGO_ENABLED", "0")
 709                                 return nil
 710                         },
 711                 })
 712                 // Also test a cgo package.
 713                 if t.cgoEnabled && t.internalLink() && !disablePIE {
 714                         t.tests = append(t.tests, distTest{
 715                                 name:    "pie_internal_cgo",
 716                                 heading: "internal linking of -buildmode=pie",
 717                                 fn: func(dt *distTest) error {
 718                                         t.addCmd(dt, "src", t.goTest(), "os/user", "-buildmode=pie", "-ldflags=-linkmode=internal", t.timeout(60))
 719                                         return nil
 720                                 },
 721                         })
 722                 }
 723         }
 724
 725         // sync tests
 726         if goos != "js" { // js doesn't support -cpu=10
 727                 t.tests = append(t.tests, distTest{
 728                         name:    "sync_cpu",
 729                         heading: "sync -cpu=10",
 730                         fn: func(dt *distTest) error {
 731                                 t.addCmd(dt, "src", t.goTest(), "sync", t.timeout(120), "-cpu=10", t.runFlag(""))
 732                                 return nil
 733                         },
 734                 })
 735         }
 736
 737         if t.raceDetectorSupported() {
 738                 t.tests = append(t.tests, distTest{
 739                         name:    "race",
 740                         heading: "Testing race detector",
 741                         fn:      t.raceTest,
 742                 })
 743         }
 744
 745         if t.cgoEnabled && !t.iOS() {
 746                 // Disabled on iOS. golang.org/issue/15919
 747                 t.registerHostTest("cgo_stdio", "../misc/cgo/stdio", "misc/cgo/stdio", ".")
 748                 t.registerHostTest("cgo_life", "../misc/cgo/life", "misc/cgo/life", ".")
 749                 fortran := os.Getenv("FC")
 750                 if fortran == "" {
 751                         fortran, _ = exec.LookPath("gfortran")
 752                 }
 753                 if t.hasBash() && goos != "android" && fortran != "" {
 754                         t.tests = append(t.tests, distTest{
 755                                 name:    "cgo_fortran",
 756                                 heading: "../misc/cgo/fortran",
 757                                 fn: func(dt *distTest) error {
 758                                         t.addCmd(dt, "misc/cgo/fortran", "./test.bash", fortran)
 759                                         return nil
 760                                 },
 761                         })
 762                 }
 763                 if t.hasSwig() && goos != "android" {
 764                         t.tests = append(t.tests, distTest{
 765                                 name:    "swig_stdio",
 766                                 heading: "../misc/swig/stdio",
 767                                 fn: func(dt *distTest) error {
 768                                         t.addCmd(dt, "misc/swig/stdio", t.goTest(), ".")
 769                                         return nil
 770                                 },
 771                         })
 772                         if t.hasCxx() {
 773                                 t.tests = append(t.tests,
 774                                         distTest{
 775                                                 name:    "swig_callback",
 776                                                 heading: "../misc/swig/callback",
 777                                                 fn: func(dt *distTest) error {
 778                                                         t.addCmd(dt, "misc/swig/callback", t.goTest(), ".")
 779                                                         return nil
 780                                                 },
 781                                         },
 782                                         distTest{
 783                                                 name:    "swig_callback_lto",
 784                                                 heading: "../misc/swig/callback",
 785                                                 fn: func(dt *distTest) error {
 786                                                         cmd := t.addCmd(dt, "misc/swig/callback", t.goTest(), ".")
 787                                                         setEnv(cmd, "CGO_CFLAGS", "-flto -Wno-lto-type-mismatch -Wno-unknown-warning-option")
 788                                                         setEnv(cmd, "CGO_CXXFLAGS", "-flto -Wno-lto-type-mismatch -Wno-unknown-warning-option")
 789                                                         setEnv(cmd, "CGO_LDFLAGS", "-flto -Wno-lto-type-mismatch -Wno-unknown-warning-option")
 790                                                         return nil
 791                                                 },
 792                                         },
 793                                 )
 794                         }
 795                 }
 796         }
 797         if t.cgoEnabled {
 798                 t.tests = append(t.tests, distTest{
 799                         name:    "cgo_test",
 800                         heading: "../misc/cgo/test",
 801                         fn:      t.cgoTest,
 802                 })
 803         }
 804
 805         // Don't run these tests with $GO_GCFLAGS because most of them
 806         // assume that they can run "go install" with no -gcflags and not
 807         // recompile the entire standard library. If make.bash ran with
 808         // special -gcflags, that's not true.
 809         if t.cgoEnabled && gogcflags == "" {
 810                 t.registerHostTest("testgodefs", "../misc/cgo/testgodefs", "misc/cgo/testgodefs", ".")
 811
 812                 t.registerTest("testso", "../misc/cgo/testso", t.goTest(), t.timeout(600), ".")
 813                 t.registerTest("testsovar", "../misc/cgo/testsovar", t.goTest(), t.timeout(600), ".")
 814                 if t.supportedBuildmode("c-archive") {
 815                         t.registerHostTest("testcarchive", "../misc/cgo/testcarchive", "misc/cgo/testcarchive", ".")
 816                 }
 817                 if t.supportedBuildmode("c-shared") {
 818                         t.registerHostTest("testcshared", "../misc/cgo/testcshared", "misc/cgo/testcshared", ".")
 819                 }
 820                 if t.supportedBuildmode("shared") {
 821                         t.registerTest("testshared", "../misc/cgo/testshared", t.goTest(), t.timeout(600), ".")
 822                 }
 823                 if t.supportedBuildmode("plugin") {
 824                         t.registerTest("testplugin", "../misc/cgo/testplugin", t.goTest(), t.timeout(600), ".")
 825                 }
 826                 if goos == "linux" || (goos == "freebsd" && goarch == "amd64") {
 827                         // because Pdeathsig of syscall.SysProcAttr struct used in misc/cgo/testsanitizers is only
 828                         // supported on Linux and FreeBSD.
 829                         t.registerHostTest("testsanitizers", "../misc/cgo/testsanitizers", "misc/cgo/testsanitizers", ".")
 830                 }
 831                 if t.hasBash() && goos != "android" && !t.iOS() && gohostos != "windows" {
 832                         t.registerHostTest("cgo_errors", "../misc/cgo/errors", "misc/cgo/errors", ".")
 833                 }
 834                 if gohostos == "linux" && t.extLink() {
 835                         t.registerTest("testsigfwd", "../misc/cgo/testsigfwd", "go", "run", ".")
 836                 }
 837         }
 838
 839         if goos != "android" && !t.iOS() {
 840                 // There are no tests in this directory, only benchmarks.
 841                 // Check that the test binary builds but don't bother running it.
 842                 // (It has init-time work to set up for the benchmarks that is not worth doing unnecessarily.)
 843                 t.registerTest("bench_go1", "../test/bench/go1", t.goTest(), "-c", "-o="+os.DevNull)
 844         }
 845         if goos != "android" && !t.iOS() {
 846                 // Only start multiple test dir shards on builders,
 847                 // where they get distributed to multiple machines.
 848                 // See issues 20141 and 31834.
 849                 nShards := 1
 850                 if os.Getenv("GO_BUILDER_NAME") != "" {
 851                         nShards = 10
 852                 }
 853                 if n, err := strconv.Atoi(os.Getenv("GO_TEST_SHARDS")); err == nil {
 854                         nShards = n
 855                 }
 856                 for shard := 0; shard < nShards; shard++ {
 857                         shard := shard
 858                         t.tests = append(t.tests, distTest{
 859                                 name:    fmt.Sprintf("test:%d_%d", shard, nShards),
 860                                 heading: "../test",
 861                                 fn:      func(dt *distTest) error { return t.testDirTest(dt, shard, nShards) },
 862                         })
 863                 }
 864         }
 865         // Only run the API check on fast development platforms.
 866         // Every platform checks the API on every GOOS/GOARCH/CGO_ENABLED combination anyway,
 867         // so we really only need to run this check once anywhere to get adequate coverage.
 868         // To help developers avoid trybot-only failures, we try to run on typical developer machines
 869         // which is darwin/linux/windows and amd64/arm64.
 870         if (goos == "darwin" || goos == "linux" || goos == "windows") && (goarch == "amd64" || goarch == "arm64") {
 871                 t.tests = append(t.tests, distTest{
 872                         name:    "api",
 873                         heading: "API check",
 874                         fn: func(dt *distTest) error {
 875                                 if t.compileOnly {
 876                                         t.addCmd(dt, "src", "go", "build", "-o", os.DevNull, filepath.Join(goroot, "src/cmd/api/run.go"))
 877                                         return nil
 878                                 }
 879                                 t.addCmd(dt, "src", "go", "run", filepath.Join(goroot, "src/cmd/api/run.go"))
 880                                 return nil
 881                         },
 882                 })
 883         }
 884
 885         // Ensure that the toolchain can bootstrap itself.
 886         // This test adds another ~45s to all.bash if run sequentially, so run it only on the builders.
 887         if os.Getenv("GO_BUILDER_NAME") != "" && goos != "android" && !t.iOS() {
 888                 t.registerHostTest("reboot", "../misc/reboot", "misc/reboot", ".")
 889         }
 890 }
 891
 892 // isRegisteredTestName reports whether a test named testName has already
 893 // been registered.
 894 func (t *tester) isRegisteredTestName(testName string) bool {
 895         for _, tt := range t.tests {
 896                 if tt.name == testName {
 897                         return true
 898                 }
 899         }
 900         return false
 901 }
 902
 903 func (t *tester) registerTest1(seq bool, name, dirBanner string, cmdline ...interface{}) {
 904         bin, args := flattenCmdline(cmdline)
 905         if bin == "time" && !t.haveTime {
 906                 bin, args = args[0], args[1:]
 907         }
 908         if t.isRegisteredTestName(name) {
 909                 panic("duplicate registered test name " + name)
 910         }
 911         t.tests = append(t.tests, distTest{
 912                 name:    name,
 913                 heading: dirBanner,
 914                 fn: func(dt *distTest) error {
 915                         if seq {
 916                                 t.runPending(dt)
 917                                 timelog("start", name)
 918                                 defer timelog("end", name)
 919                                 return t.dirCmd(filepath.Join(goroot, "src", dirBanner), bin, args).Run()
 920                         }
 921                         t.addCmd(dt, filepath.Join(goroot, "src", dirBanner), bin, args)
 922                         return nil
 923                 },
 924         })
 925 }
 926
 927 func (t *tester) registerTest(name, dirBanner string, cmdline ...interface{}) {
 928         t.registerTest1(false, name, dirBanner, cmdline...)
 929 }
 930
 931 func (t *tester) registerSeqTest(name, dirBanner string, cmdline ...interface{}) {
 932         t.registerTest1(true, name, dirBanner, cmdline...)
 933 }
 934
 935 func (t *tester) bgDirCmd(dir, bin string, args ...string) *exec.Cmd {
 936         cmd := exec.Command(bin, args...)
 937         if filepath.IsAbs(dir) {
 938                 setDir(cmd, dir)
 939         } else {
 940                 setDir(cmd, filepath.Join(goroot, dir))
 941         }
 942         return cmd
 943 }
 944
 945 func (t *tester) dirCmd(dir string, cmdline ...interface{}) *exec.Cmd {
 946         bin, args := flattenCmdline(cmdline)
 947         cmd := t.bgDirCmd(dir, bin, args...)
 948         cmd.Stdout = os.Stdout
 949         cmd.Stderr = os.Stderr
 950         if vflag > 1 {
 951                 errprintf("%s\n", strings.Join(cmd.Args, " "))
 952         }
 953         return cmd
 954 }
 955
 956 // flattenCmdline flattens a mixture of string and []string as single list
 957 // and then interprets it as a command line: first element is binary, then args.
 958 func flattenCmdline(cmdline []interface{}) (bin string, args []string) {
 959         var list []string
 960         for _, x := range cmdline {
 961                 switch x := x.(type) {
 962                 case string:
 963                         list = append(list, x)
 964                 case []string:
 965                         list = append(list, x...)
 966                 default:
 967                         panic("invalid addCmd argument type: " + reflect.TypeOf(x).String())
 968                 }
 969         }
 970
 971         // The go command is too picky about duplicated flags.
 972         // Drop all but the last of the allowed duplicated flags.
 973         drop := make([]bool, len(list))
 974         have := map[string]int{}
 975         for i := 1; i < len(list); i++ {
 976                 j := strings.Index(list[i], "=")
 977                 if j < 0 {
 978                         continue
 979                 }
 980                 flag := list[i][:j]
 981                 switch flag {
 982                 case "-run", "-tags":
 983                         if have[flag] != 0 {
 984                                 drop[have[flag]] = true
 985                         }
 986                         have[flag] = i
 987                 }
 988         }
 989         out := list[:0]
 990         for i, x := range list {
 991                 if !drop[i] {
 992                         out = append(out, x)
 993                 }
 994         }
 995         list = out
 996
 997         bin = list[0]
 998         if bin == "go" {
 999                 bin = gorootBinGo
1000         }
1001         return bin, list[1:]
1002 }
1003
1004 func (t *tester) addCmd(dt *distTest, dir string, cmdline ...interface{}) *exec.Cmd {
1005         bin, args := flattenCmdline(cmdline)
1006         w := &work{
1007                 dt:  dt,
1008                 cmd: t.bgDirCmd(dir, bin, args...),
1009         }
1010         t.worklist = append(t.worklist, w)
1011         return w.cmd
1012 }
1013
1014 func (t *tester) iOS() bool {
1015         return goos == "ios"
1016 }
1017
1018 func (t *tester) out(v string) {
1019         if t.banner == "" {
1020                 return
1021         }
1022         fmt.Println("\n" + t.banner + v)
1023 }
1024
1025 func (t *tester) extLink() bool {
1026         pair := gohostos + "-" + goarch
1027         switch pair {
1028         case "aix-ppc64",
1029                 "android-arm", "android-arm64",
1030                 "darwin-amd64", "darwin-arm64",
1031                 "dragonfly-amd64",
1032                 "freebsd-386", "freebsd-amd64", "freebsd-arm", "freebsd-riscv64",
1033                 "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-loong64", "linux-ppc64le", "linux-mips64", "linux-mips64le", "linux-mips", "linux-mipsle", "linux-riscv64", "linux-s390x",
1034                 "netbsd-386", "netbsd-amd64",
1035                 "openbsd-386", "openbsd-amd64",
1036                 "windows-386", "windows-amd64":
1037                 return true
1038         }
1039         return false
1040 }
1041
1042 func (t *tester) internalLink() bool {
1043         if gohostos == "dragonfly" {
1044                 // linkmode=internal fails on dragonfly since errno is a TLS relocation.
1045                 return false
1046         }
1047         if goos == "android" {
1048                 return false
1049         }
1050         if goos == "ios" {
1051                 return false
1052         }
1053         if goos == "windows" && goarch == "arm64" {
1054                 return false
1055         }
1056         // Internally linking cgo is incomplete on some architectures.
1057         // https://golang.org/issue/10373
1058         // https://golang.org/issue/14449
1059         if goarch == "loong64" || goarch == "mips64" || goarch == "mips64le" || goarch == "mips" || goarch == "mipsle" || goarch == "riscv64" {
1060                 return false
1061         }
1062         if goos == "aix" {
1063                 // linkmode=internal isn't supported.
1064                 return false
1065         }
1066         return true
1067 }
1068
1069 func (t *tester) internalLinkPIE() bool {
1070         switch goos + "-" + goarch {
1071         case "darwin-amd64", "darwin-arm64",
1072                 "linux-amd64", "linux-arm64", "linux-ppc64le",
1073                 "android-arm64",
1074                 "windows-amd64", "windows-386", "windows-arm":
1075                 return true
1076         }
1077         return false
1078 }
1079
1080 func (t *tester) supportedBuildmode(mode string) bool {
1081         pair := goos + "-" + goarch
1082         switch mode {
1083         case "c-archive":
1084                 if !t.extLink() {
1085                         return false
1086                 }
1087                 switch pair {
1088                 case "aix-ppc64",
1089                         "darwin-amd64", "darwin-arm64", "ios-arm64",
1090                         "linux-amd64", "linux-386", "linux-ppc64le", "linux-riscv64", "linux-s390x",
1091                         "freebsd-amd64",
1092                         "windows-amd64", "windows-386":
1093                         return true
1094                 }
1095                 return false
1096         case "c-shared":
1097                 switch pair {
1098                 case "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-riscv64", "linux-s390x",
1099                         "darwin-amd64", "darwin-arm64",
1100                         "freebsd-amd64",
1101                         "android-arm", "android-arm64", "android-386",
1102                         "windows-amd64", "windows-386", "windows-arm64":
1103                         return true
1104                 }
1105                 return false
1106         case "shared":
1107                 switch pair {
1108                 case "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-s390x":
1109                         return true
1110                 }
1111                 return false
1112         case "plugin":
1113                 switch pair {
1114                 case "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-s390x", "linux-ppc64le":
1115                         return true
1116                 case "darwin-amd64", "darwin-arm64":
1117                         return true
1118                 case "freebsd-amd64":
1119                         return true
1120                 }
1121                 return false
1122         case "pie":
1123                 switch pair {
1124                 case "aix/ppc64",
1125                         "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-riscv64", "linux-s390x",
1126                         "android-amd64", "android-arm", "android-arm64", "android-386":
1127                         return true
1128                 case "darwin-amd64", "darwin-arm64":
1129                         return true
1130                 case "windows-amd64", "windows-386", "windows-arm":
1131                         return true
1132                 }
1133                 return false
1134
1135         default:
1136                 fatalf("internal error: unknown buildmode %s", mode)
1137                 return false
1138         }
1139 }
1140
1141 func (t *tester) registerHostTest(name, heading, dir, pkg string) {
1142         t.tests = append(t.tests, distTest{
1143                 name:    name,
1144                 heading: heading,
1145                 fn: func(dt *distTest) error {
1146                         t.runPending(dt)
1147                         timelog("start", name)
1148                         defer timelog("end", name)
1149                         return t.runHostTest(dir, pkg)
1150                 },
1151         })
1152 }
1153
1154 func (t *tester) runHostTest(dir, pkg string) error {
1155         out, err := exec.Command(gorootBinGo, "env", "GOEXE", "GOTMPDIR").Output()
1156         if err != nil {
1157                 return err
1158         }
1159
1160         parts := strings.Split(string(out), "\n")
1161         if len(parts) < 2 {
1162                 return fmt.Errorf("'go env GOEXE GOTMPDIR' output contains <2 lines")
1163         }
1164         GOEXE := strings.TrimSpace(parts[0])
1165         GOTMPDIR := strings.TrimSpace(parts[1])
1166
1167         f, err := os.CreateTemp(GOTMPDIR, "test.test-*"+GOEXE)
1168         if err != nil {
1169                 return err
1170         }
1171         f.Close()
1172         defer os.Remove(f.Name())
1173
1174         cmd := t.dirCmd(dir, t.goTest(), "-c", "-o", f.Name(), pkg)
1175         setEnv(cmd, "GOARCH", gohostarch)
1176         setEnv(cmd, "GOOS", gohostos)
1177         if err := cmd.Run(); err != nil {
1178                 return err
1179         }
1180         return t.dirCmd(dir, f.Name(), "-test.short="+short(), "-test.timeout="+t.timeoutDuration(300).String()).Run()
1181 }
1182
1183 func (t *tester) cgoTest(dt *distTest) error {
1184         cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), ".")
1185         setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=auto")
1186
1187         // Stub out various buildmode=pie tests  on alpine until 54354 resolved.
1188         builderName := os.Getenv("GO_BUILDER_NAME")
1189         disablePIE := strings.HasSuffix(builderName, "-alpine")
1190
1191         if t.internalLink() {
1192                 cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=internal", ".")
1193                 setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=internal")
1194         }
1195
1196         pair := gohostos + "-" + goarch
1197         switch pair {
1198         case "darwin-amd64", "darwin-arm64",
1199                 "windows-386", "windows-amd64":
1200                 // test linkmode=external, but __thread not supported, so skip testtls.
1201                 if !t.extLink() {
1202                         break
1203                 }
1204                 cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), ".")
1205                 setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=external")
1206
1207                 t.addCmd(dt, "misc/cgo/test", t.goTest(), "-ldflags", "-linkmode=external -s", ".")
1208
1209                 if t.supportedBuildmode("pie") && !disablePIE {
1210
1211                         t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", ".")
1212                         if t.internalLink() && t.internalLinkPIE() {
1213                                 t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", "-ldflags=-linkmode=internal", "-tags=internal,internal_pie", ".")
1214                         }
1215                 }
1216
1217         case "aix-ppc64",
1218                 "android-arm", "android-arm64",
1219                 "dragonfly-amd64",
1220                 "freebsd-386", "freebsd-amd64", "freebsd-arm", "freebsd-riscv64",
1221                 "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-riscv64", "linux-s390x",
1222                 "netbsd-386", "netbsd-amd64",
1223                 "openbsd-386", "openbsd-amd64", "openbsd-arm", "openbsd-arm64", "openbsd-mips64":
1224
1225                 cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), ".")
1226                 setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=external")
1227                 // cgo should be able to cope with both -g arguments and colored
1228                 // diagnostics.
1229                 setEnv(cmd, "CGO_CFLAGS", "-g0 -fdiagnostics-color")
1230
1231                 t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-ldflags", "-linkmode=auto", ".")
1232                 t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-ldflags", "-linkmode=external", ".")
1233
1234                 switch pair {
1235                 case "aix-ppc64", "netbsd-386", "netbsd-amd64":
1236                         // no static linking
1237                 case "freebsd-arm":
1238                         // -fPIC compiled tls code will use __tls_get_addr instead
1239                         // of __aeabi_read_tp, however, on FreeBSD/ARM, __tls_get_addr
1240                         // is implemented in rtld-elf, so -fPIC isn't compatible with
1241                         // static linking on FreeBSD/ARM with clang. (cgo depends on
1242                         // -fPIC fundamentally.)
1243                 default:
1244                         cmd := t.dirCmd("misc/cgo/test",
1245                                 compilerEnvLookup(defaultcc, goos, goarch), "-xc", "-o", "/dev/null", "-static", "-")
1246                         cmd.Stdin = strings.NewReader("int main() {}")
1247                         if err := cmd.Run(); err != nil {
1248                                 fmt.Println("No support for static linking found (lacks libc.a?), skip cgo static linking test.")
1249                         } else {
1250                                 if goos != "android" {
1251                                         t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-ldflags", `-linkmode=external -extldflags "-static -pthread"`, ".")
1252                                 }
1253                                 t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), ".")
1254                                 t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), "-ldflags", `-linkmode=external`, ".")
1255                                 if goos != "android" {
1256                                         t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), "-ldflags", `-linkmode=external -extldflags "-static -pthread"`, ".")
1257                                         t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=static", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`, ".")
1258                                         // -static in CGO_LDFLAGS triggers a different code path
1259                                         // than -static in -extldflags, so test both.
1260                                         // See issue #16651.
1261                                         cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=static", ".")
1262                                         setEnv(cmd, "CGO_LDFLAGS", "-static -pthread")
1263                                 }
1264                         }
1265
1266                         if t.supportedBuildmode("pie") && !disablePIE {
1267                                 t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", ".")
1268                                 if t.internalLink() && t.internalLinkPIE() {
1269                                         t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", "-ldflags=-linkmode=internal", "-tags=internal,internal_pie", ".")
1270                                 }
1271                                 t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-buildmode=pie", ".")
1272                                 t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), "-buildmode=pie", ".")
1273                         }
1274                 }
1275         }
1276
1277         return nil
1278 }
1279
1280 // run pending test commands, in parallel, emitting headers as appropriate.
1281 // When finished, emit header for nextTest, which is going to run after the
1282 // pending commands are done (and runPending returns).
1283 // A test should call runPending if it wants to make sure that it is not
1284 // running in parallel with earlier tests, or if it has some other reason
1285 // for needing the earlier tests to be done.
1286 func (t *tester) runPending(nextTest *distTest) {
1287         checkNotStale("go", "std")
1288         worklist := t.worklist
1289         t.worklist = nil
1290         for _, w := range worklist {
1291                 w.start = make(chan bool)
1292                 w.end = make(chan bool)
1293                 go func(w *work) {
1294                         if !<-w.start {
1295                                 timelog("skip", w.dt.name)
1296                                 w.out = []byte(fmt.Sprintf("skipped due to earlier error\n"))
1297                         } else {
1298                                 timelog("start", w.dt.name)
1299                                 w.out, w.err = w.cmd.CombinedOutput()
1300                                 if w.err != nil {
1301                                         if isUnsupportedVMASize(w) {
1302                                                 timelog("skip", w.dt.name)
1303                                                 w.out = []byte(fmt.Sprintf("skipped due to unsupported VMA\n"))
1304                                                 w.err = nil
1305                                         }
1306                                 }
1307                         }
1308                         timelog("end", w.dt.name)
1309                         w.end <- true
1310                 }(w)
1311         }
1312
1313         started := 0
1314         ended := 0
1315         var last *distTest
1316         for ended < len(worklist) {
1317                 for started < len(worklist) && started-ended < maxbg {
1318                         w := worklist[started]
1319                         started++
1320                         w.start <- !t.failed || t.keepGoing
1321                 }
1322                 w := worklist[ended]
1323                 dt := w.dt
1324                 if dt.heading != "" && t.lastHeading != dt.heading {
1325                         t.lastHeading = dt.heading
1326                         t.out(dt.heading)
1327                 }
1328                 if dt != last {
1329                         // Assumes all the entries for a single dt are in one worklist.
1330                         last = w.dt
1331                         if vflag > 0 {
1332                                 fmt.Printf("# go tool dist test -run=^%s$\n", dt.name)
1333                         }
1334                 }
1335                 if vflag > 1 {
1336                         errprintf("%s\n", strings.Join(w.cmd.Args, " "))
1337                 }
1338                 ended++
1339                 <-w.end
1340                 os.Stdout.Write(w.out)
1341                 if w.err != nil {
1342                         log.Printf("Failed: %v", w.err)
1343                         t.failed = true
1344                 }
1345                 checkNotStale("go", "std")
1346         }
1347         if t.failed && !t.keepGoing {
1348                 fatalf("FAILED")
1349         }
1350
1351         if dt := nextTest; dt != nil {
1352                 if dt.heading != "" && t.lastHeading != dt.heading {
1353                         t.lastHeading = dt.heading
1354                         t.out(dt.heading)
1355                 }
1356                 if vflag > 0 {
1357                         fmt.Printf("# go tool dist test -run=^%s$\n", dt.name)
1358                 }
1359         }
1360 }
1361
1362 func (t *tester) hasBash() bool {
1363         switch gohostos {
1364         case "windows", "plan9":
1365                 return false
1366         }
1367         return true
1368 }
1369
1370 func (t *tester) hasCxx() bool {
1371         cxx, _ := exec.LookPath(compilerEnvLookup(defaultcxx, goos, goarch))
1372         return cxx != ""
1373 }
1374
1375 func (t *tester) hasSwig() bool {
1376         swig, err := exec.LookPath("swig")
1377         if err != nil {
1378                 return false
1379         }
1380
1381         // Check that swig was installed with Go support by checking
1382         // that a go directory exists inside the swiglib directory.
1383         // See https://golang.org/issue/23469.
1384         output, err := exec.Command(swig, "-go", "-swiglib").Output()
1385         if err != nil {
1386                 return false
1387         }
1388         swigDir := strings.TrimSpace(string(output))
1389
1390         _, err = os.Stat(filepath.Join(swigDir, "go"))
1391         if err != nil {
1392                 return false
1393         }
1394
1395         // Check that swig has a new enough version.
1396         // See https://golang.org/issue/22858.
1397         out, err := exec.Command(swig, "-version").CombinedOutput()
1398         if err != nil {
1399                 return false
1400         }
1401
1402         re := regexp.MustCompile(`[vV]ersion +(\d+)([.]\d+)?([.]\d+)?`)
1403         matches := re.FindSubmatch(out)
1404         if matches == nil {
1405                 // Can't find version number; hope for the best.
1406                 return true
1407         }
1408
1409         major, err := strconv.Atoi(string(matches[1]))
1410         if err != nil {
1411                 // Can't find version number; hope for the best.
1412                 return true
1413         }
1414         if major < 3 {
1415                 return false
1416         }
1417         if major > 3 {
1418                 // 4.0 or later
1419                 return true
1420         }
1421
1422         // We have SWIG version 3.x.
1423         if len(matches[2]) > 0 {
1424                 minor, err := strconv.Atoi(string(matches[2][1:]))
1425                 if err != nil {
1426                         return true
1427                 }
1428                 if minor > 0 {
1429                         // 3.1 or later
1430                         return true
1431                 }
1432         }
1433
1434         // We have SWIG version 3.0.x.
1435         if len(matches[3]) > 0 {
1436                 patch, err := strconv.Atoi(string(matches[3][1:]))
1437                 if err != nil {
1438                         return true
1439                 }
1440                 if patch < 6 {
1441                         // Before 3.0.6.
1442                         return false
1443                 }
1444         }
1445
1446         return true
1447 }
1448
1449 func (t *tester) raceDetectorSupported() bool {
1450         if gohostos != goos {
1451                 return false
1452         }
1453         if !t.cgoEnabled {
1454                 return false
1455         }
1456         if !raceDetectorSupported(goos, goarch) {
1457                 return false
1458         }
1459         // The race detector doesn't work on Alpine Linux:
1460         // golang.org/issue/14481
1461         if isAlpineLinux() {
1462                 return false
1463         }
1464         // NetBSD support is unfinished.
1465         // golang.org/issue/26403
1466         if goos == "netbsd" {
1467                 return false
1468         }
1469         return true
1470 }
1471
1472 func isAlpineLinux() bool {
1473         if runtime.GOOS != "linux" {
1474                 return false
1475         }
1476         fi, err := os.Lstat("/etc/alpine-release")
1477         return err == nil && fi.Mode().IsRegular()
1478 }
1479
1480 func (t *tester) runFlag(rx string) string {
1481         if t.compileOnly {
1482                 return "-run=^$"
1483         }
1484         return "-run=" + rx
1485 }
1486
1487 func (t *tester) raceTest(dt *distTest) error {
1488         t.addCmd(dt, "src", t.goTest(), "-race", t.runFlag("Output"), "runtime/race")
1489         t.addCmd(dt, "src", t.goTest(), "-race", t.runFlag("TestParse|TestEcho|TestStdinCloseRace|TestClosedPipeRace|TestTypeRace|TestFdRace|TestFdReadRace|TestFileCloseRace"), "flag", "net", "os", "os/exec", "encoding/gob")
1490         // We don't want the following line, because it
1491         // slows down all.bash (by 10 seconds on my laptop).
1492         // The race builder should catch any error here, but doesn't.
1493         // TODO(iant): Figure out how to catch this.
1494         // t.addCmd(dt, "src", t.goTest(),  "-race", "-run=TestParallelTest", "cmd/go")
1495         if t.cgoEnabled {
1496                 // Building misc/cgo/test takes a long time.
1497                 // There are already cgo-enabled packages being tested with the race detector.
1498                 // We shouldn't need to redo all of misc/cgo/test too.
1499                 // The race buildler will take care of this.
1500                 // cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), "-race")
1501                 // setEnv(cmd, "GOTRACEBACK", "2")
1502         }
1503         if t.extLink() {
1504                 // Test with external linking; see issue 9133.
1505                 t.addCmd(dt, "src", t.goTest(), "-race", "-ldflags=-linkmode=external", t.runFlag("TestParse|TestEcho|TestStdinCloseRace"), "flag", "os/exec")
1506         }
1507         return nil
1508 }
1509
1510 var runtest struct {
1511         sync.Once
1512         exe string
1513         err error
1514 }
1515
1516 func (t *tester) testDirTest(dt *distTest, shard, shards int) error {
1517         runtest.Do(func() {
1518                 f, err := os.CreateTemp("", "runtest-*.exe") // named exe for Windows, but harmless elsewhere
1519                 if err != nil {
1520                         runtest.err = err
1521                         return
1522                 }
1523                 f.Close()
1524
1525                 runtest.exe = f.Name()
1526                 xatexit(func() {
1527                         os.Remove(runtest.exe)
1528                 })
1529
1530                 cmd := t.dirCmd("test", "go", "build", "-o", runtest.exe, "run.go")
1531                 setEnv(cmd, "GOOS", gohostos)
1532                 setEnv(cmd, "GOARCH", gohostarch)
1533                 runtest.err = cmd.Run()
1534         })
1535         if runtest.err != nil {
1536                 return runtest.err
1537         }
1538         if t.compileOnly {
1539                 return nil
1540         }
1541         t.addCmd(dt, "test", runtest.exe,
1542                 fmt.Sprintf("--shard=%d", shard),
1543                 fmt.Sprintf("--shards=%d", shards),
1544         )
1545         return nil
1546 }
1547
1548 // cgoPackages is the standard packages that use cgo.
1549 var cgoPackages = []string{
1550         "net",
1551         "os/user",
1552 }
1553
1554 var funcBenchmark = []byte("\nfunc Benchmark")
1555
1556 // packageHasBenchmarks reports whether pkg has benchmarks.
1557 // On any error, it conservatively returns true.
1558 //
1559 // This exists just to eliminate work on the builders, since compiling
1560 // a test in race mode just to discover it has no benchmarks costs a
1561 // second or two per package, and this function returns false for
1562 // about 100 packages.
1563 func (t *tester) packageHasBenchmarks(pkg string) bool {
1564         pkgDir := filepath.Join(goroot, "src", pkg)
1565         d, err := os.Open(pkgDir)
1566         if err != nil {
1567                 return true // conservatively
1568         }
1569         defer d.Close()
1570         names, err := d.Readdirnames(-1)
1571         if err != nil {
1572                 return true // conservatively
1573         }
1574         for _, name := range names {
1575                 if !strings.HasSuffix(name, "_test.go") {
1576                         continue
1577                 }
1578                 slurp, err := os.ReadFile(filepath.Join(pkgDir, name))
1579                 if err != nil {
1580                         return true // conservatively
1581                 }
1582                 if bytes.Contains(slurp, funcBenchmark) {
1583                         return true
1584                 }
1585         }
1586         return false
1587 }
1588
1589 // makeGOROOTUnwritable makes all $GOROOT files & directories non-writable to
1590 // check that no tests accidentally write to $GOROOT.
1591 func (t *tester) makeGOROOTUnwritable() (undo func()) {
1592         dir := os.Getenv("GOROOT")
1593         if dir == "" {
1594                 panic("GOROOT not set")
1595         }
1596
1597         type pathMode struct {
1598                 path string
1599                 mode os.FileMode
1600         }
1601         var dirs []pathMode // in lexical order
1602
1603         undo = func() {
1604                 for i := range dirs {
1605                         os.Chmod(dirs[i].path, dirs[i].mode) // best effort
1606                 }
1607         }
1608
1609         gocache := os.Getenv("GOCACHE")
1610         if gocache == "" {
1611                 panic("GOCACHE not set")
1612         }
1613         gocacheSubdir, _ := filepath.Rel(dir, gocache)
1614
1615         // Note: Can't use WalkDir here, because this has to compile with Go 1.4.
1616         filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
1617                 if suffix := strings.TrimPrefix(path, dir+string(filepath.Separator)); suffix != "" {
1618                         if suffix == gocacheSubdir {
1619                                 // Leave GOCACHE writable: we may need to write test binaries into it.
1620                                 return filepath.SkipDir
1621                         }
1622                         if suffix == ".git" {
1623                                 // Leave Git metadata in whatever state it was in. It may contain a lot
1624                                 // of files, and it is highly unlikely that a test will try to modify
1625                                 // anything within that directory.
1626                                 return filepath.SkipDir
1627                         }
1628                 }
1629                 if err == nil {
1630                         mode := info.Mode()
1631                         if mode&0222 != 0 && (mode.IsDir() || mode.IsRegular()) {
1632                                 dirs = append(dirs, pathMode{path, mode})
1633                         }
1634                 }
1635                 return nil
1636         })
1637
1638         // Run over list backward to chmod children before parents.
1639         for i := len(dirs) - 1; i >= 0; i-- {
1640                 err := os.Chmod(dirs[i].path, dirs[i].mode&^0222)
1641                 if err != nil {
1642                         dirs = dirs[i:] // Only undo what we did so far.
1643                         undo()
1644                         fatalf("failed to make GOROOT read-only: %v", err)
1645                 }
1646         }
1647
1648         return undo
1649 }
1650
1651 // raceDetectorSupported is a copy of the function
1652 // internal/platform.RaceDetectorSupported, which can't be used here
1653 // because cmd/dist has to be buildable by Go 1.4.
1654 // The race detector only supports 48-bit VMA on arm64. But we don't have
1655 // a good solution to check VMA size(See https://golang.org/issue/29948)
1656 // raceDetectorSupported will always return true for arm64. But race
1657 // detector tests may abort on non 48-bit VMA configuration, the tests
1658 // will be marked as "skipped" in this case.
1659 func raceDetectorSupported(goos, goarch string) bool {
1660         switch goos {
1661         case "linux":
1662                 return goarch == "amd64" || goarch == "ppc64le" || goarch == "arm64" || goarch == "s390x"
1663         case "darwin":
1664                 return goarch == "amd64" || goarch == "arm64"
1665         case "freebsd", "netbsd", "openbsd", "windows":
1666                 return goarch == "amd64"
1667         default:
1668                 return false
1669         }
1670 }
1671
1672 // isUnsupportedVMASize reports whether the failure is caused by an unsupported
1673 // VMA for the race detector (for example, running the race detector on an
1674 // arm64 machine configured with 39-bit VMA)
1675 func isUnsupportedVMASize(w *work) bool {
1676         unsupportedVMA := []byte("unsupported VMA range")
1677         return w.dt.name == "race" && bytes.Contains(w.out, unsupportedVMA)
1678 }
1679
1680 // isEnvSet reports whether the environment variable evar is
1681 // set in the environment.
1682 func isEnvSet(evar string) bool {
1683         evarEq := evar + "="
1684         for _, e := range os.Environ() {
1685                 if strings.HasPrefix(e, evarEq) {
1686                         return true
1687                 }
1688         }
1689         return false
1690 }