src/cmd/dist/test.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "flag"
  10         "fmt"
  11         "log"
  12         "os"
  13         "os/exec"
  14         "path/filepath"
  15         "reflect"
  16         "regexp"
  17         "runtime"
  18         "strconv"
  19         "strings"
  20         "sync"
  21         "time"
  22 )
  23
  24 func cmdtest() {
  25         gogcflags = os.Getenv("GO_GCFLAGS")
  26         setNoOpt()
  27
  28         var t tester
  29
  30         var noRebuild bool
  31         flag.BoolVar(&t.listMode, "list", false, "list available tests")
  32         flag.BoolVar(&t.rebuild, "rebuild", false, "rebuild everything first")
  33         flag.BoolVar(&noRebuild, "no-rebuild", false, "overrides -rebuild (historical dreg)")
  34         flag.BoolVar(&t.keepGoing, "k", false, "keep going even when error occurred")
  35         flag.BoolVar(&t.race, "race", false, "run in race builder mode (different set of tests)")
  36         flag.BoolVar(&t.compileOnly, "compile-only", false, "compile tests, but don't run them. This is for some builders. Not all dist tests respect this flag, but most do.")
  37         flag.StringVar(&t.banner, "banner", "##### ", "banner prefix; blank means no section banners")
  38         flag.StringVar(&t.runRxStr, "run", os.Getenv("GOTESTONLY"),
  39                 "run only those tests matching the regular expression; empty means to run all. "+
  40                         "Special exception: if the string begins with '!', the match is inverted.")
  41         flag.BoolVar(&t.msan, "msan", false, "run in memory sanitizer builder mode")
  42         flag.BoolVar(&t.asan, "asan", false, "run in address sanitizer builder mode")
  43
  44         xflagparse(-1) // any number of args
  45         if noRebuild {
  46                 t.rebuild = false
  47         }
  48
  49         t.run()
  50 }
  51
  52 // tester executes cmdtest.
  53 type tester struct {
  54         race        bool
  55         msan        bool
  56         asan        bool
  57         listMode    bool
  58         rebuild     bool
  59         failed      bool
  60         keepGoing   bool
  61         compileOnly bool // just try to compile all tests, but no need to run
  62         runRxStr    string
  63         runRx       *regexp.Regexp
  64         runRxWant   bool     // want runRx to match (true) or not match (false)
  65         runNames    []string // tests to run, exclusive with runRx; empty means all
  66         banner      string   // prefix, or "" for none
  67         lastHeading string   // last dir heading printed
  68
  69         cgoEnabled bool
  70         partial    bool
  71         haveTime   bool // the 'time' binary is available
  72
  73         tests        []distTest
  74         timeoutScale int
  75
  76         worklist []*work
  77 }
  78
  79 type work struct {
  80         dt    *distTest
  81         cmd   *exec.Cmd
  82         start chan bool
  83         out   []byte
  84         err   error
  85         end   chan bool
  86 }
  87
  88 // A distTest is a test run by dist test.
  89 // Each test has a unique name and belongs to a group (heading)
  90 type distTest struct {
  91         name    string // unique test name; may be filtered with -run flag
  92         heading string // group section; this header is printed before the test is run.
  93         fn      func(*distTest) error
  94 }
  95
  96 func (t *tester) run() {
  97         timelog("start", "dist test")
  98
  99         os.Setenv("PATH", fmt.Sprintf("%s%c%s", gorootBin, os.PathListSeparator, os.Getenv("PATH")))
 100
 101         cmd := exec.Command(gorootBinGo, "env", "CGO_ENABLED")
 102         cmd.Stderr = new(bytes.Buffer)
 103         slurp, err := cmd.Output()
 104         if err != nil {
 105                 fatalf("Error running go env CGO_ENABLED: %v\n%s", err, cmd.Stderr)
 106         }
 107         t.cgoEnabled, _ = strconv.ParseBool(strings.TrimSpace(string(slurp)))
 108         if flag.NArg() > 0 && t.runRxStr != "" {
 109                 fatalf("the -run regular expression flag is mutually exclusive with test name arguments")
 110         }
 111
 112         t.runNames = flag.Args()
 113
 114         if t.hasBash() {
 115                 if _, err := exec.LookPath("time"); err == nil {
 116                         t.haveTime = true
 117                 }
 118         }
 119
 120         // Set GOTRACEBACK to system if the user didn't set a level explicitly.
 121         // Since we're running tests for Go, we want as much detail as possible
 122         // if something goes wrong.
 123         //
 124         // Set it before running any commands just in case something goes wrong.
 125         if ok := isEnvSet("GOTRACEBACK"); !ok {
 126                 if err := os.Setenv("GOTRACEBACK", "system"); err != nil {
 127                         if t.keepGoing {
 128                                 log.Printf("Failed to set GOTRACEBACK: %v", err)
 129                         } else {
 130                                 fatalf("Failed to set GOTRACEBACK: %v", err)
 131                         }
 132                 }
 133         }
 134
 135         if t.rebuild {
 136                 t.out("Building packages and commands.")
 137                 // Force rebuild the whole toolchain.
 138                 goInstall("go", append([]string{"-a"}, toolchain...)...)
 139         }
 140
 141         if !t.listMode {
 142                 if os.Getenv("GO_BUILDER_NAME") == "" {
 143                         // Complete rebuild bootstrap, even with -no-rebuild.
 144                         // If everything is up-to-date, this is a no-op.
 145                         // If everything is not up-to-date, the first checkNotStale
 146                         // during the test process will kill the tests, so we might
 147                         // as well install the world.
 148                         // Now that for example "go install cmd/compile" does not
 149                         // also install runtime (you need "go install -i cmd/compile"
 150                         // for that), it's easy for previous workflows like
 151                         // "rebuild the compiler and then run run.bash"
 152                         // to break if we don't automatically refresh things here.
 153                         // Rebuilding is a shortened bootstrap.
 154                         // See cmdbootstrap for a description of the overall process.
 155                         goInstall("go", toolchain...)
 156                         goInstall("go", toolchain...)
 157                         goInstall("go", "std", "cmd")
 158                 } else {
 159                         // The Go builder infrastructure should always begin running tests from a
 160                         // clean, non-stale state, so there is no need to rebuild the world.
 161                         // Instead, we can just check that it is not stale, which may be less
 162                         // expensive (and is also more likely to catch bugs in the builder
 163                         // implementation).
 164                         checkNotStale("go", "std", "cmd")
 165                 }
 166         }
 167
 168         t.timeoutScale = 1
 169         switch goarch {
 170         case "arm":
 171                 t.timeoutScale = 2
 172         case "mips", "mipsle", "mips64", "mips64le":
 173                 t.timeoutScale = 4
 174         }
 175         if s := os.Getenv("GO_TEST_TIMEOUT_SCALE"); s != "" {
 176                 t.timeoutScale, err = strconv.Atoi(s)
 177                 if err != nil {
 178                         fatalf("failed to parse $GO_TEST_TIMEOUT_SCALE = %q as integer: %v", s, err)
 179                 }
 180         }
 181
 182         if t.runRxStr != "" {
 183                 if t.runRxStr[0] == '!' {
 184                         t.runRxWant = false
 185                         t.runRxStr = t.runRxStr[1:]
 186                 } else {
 187                         t.runRxWant = true
 188                 }
 189                 t.runRx = regexp.MustCompile(t.runRxStr)
 190         }
 191
 192         t.registerTests()
 193         if t.listMode {
 194                 for _, tt := range t.tests {
 195                         fmt.Println(tt.name)
 196                 }
 197                 return
 198         }
 199
 200         for _, name := range t.runNames {
 201                 if !t.isRegisteredTestName(name) {
 202                         fatalf("unknown test %q", name)
 203                 }
 204         }
 205
 206         // On a few builders, make GOROOT unwritable to catch tests writing to it.
 207         if strings.HasPrefix(os.Getenv("GO_BUILDER_NAME"), "linux-") {
 208                 if os.Getuid() == 0 {
 209                         // Don't bother making GOROOT unwritable:
 210                         // we're running as root, so permissions would have no effect.
 211                 } else {
 212                         xatexit(t.makeGOROOTUnwritable())
 213                 }
 214         }
 215
 216         if err := t.maybeLogMetadata(); err != nil {
 217                 t.failed = true
 218                 if t.keepGoing {
 219                         log.Printf("Failed logging metadata: %v", err)
 220                 } else {
 221                         fatalf("Failed logging metadata: %v", err)
 222                 }
 223         }
 224
 225         for _, dt := range t.tests {
 226                 if !t.shouldRunTest(dt.name) {
 227                         t.partial = true
 228                         continue
 229                 }
 230                 dt := dt // dt used in background after this iteration
 231                 if err := dt.fn(&dt); err != nil {
 232                         t.runPending(&dt) // in case that hasn't been done yet
 233                         t.failed = true
 234                         if t.keepGoing {
 235                                 log.Printf("Failed: %v", err)
 236                         } else {
 237                                 fatalf("Failed: %v", err)
 238                         }
 239                 }
 240         }
 241         t.runPending(nil)
 242         timelog("end", "dist test")
 243
 244         if t.failed {
 245                 fmt.Println("\nFAILED")
 246                 xexit(1)
 247         } else if incomplete[goos+"/"+goarch] {
 248                 // The test succeeded, but consider it as failed so we don't
 249                 // forget to remove the port from the incomplete map once the
 250                 // port is complete.
 251                 fmt.Println("\nFAILED (incomplete port)")
 252                 xexit(1)
 253         } else if t.partial {
 254                 fmt.Println("\nALL TESTS PASSED (some were excluded)")
 255         } else {
 256                 fmt.Println("\nALL TESTS PASSED")
 257         }
 258 }
 259
 260 func (t *tester) shouldRunTest(name string) bool {
 261         if t.runRx != nil {
 262                 return t.runRx.MatchString(name) == t.runRxWant
 263         }
 264         if len(t.runNames) == 0 {
 265                 return true
 266         }
 267         for _, runName := range t.runNames {
 268                 if runName == name {
 269                         return true
 270                 }
 271         }
 272         return false
 273 }
 274
 275 func (t *tester) maybeLogMetadata() error {
 276         if t.compileOnly {
 277                 // We need to run a subprocess to log metadata. Don't do that
 278                 // on compile-only runs.
 279                 return nil
 280         }
 281         t.out("Test execution environment.")
 282         // Helper binary to print system metadata (CPU model, etc). This is a
 283         // separate binary from dist so it need not build with the bootstrap
 284         // toolchain.
 285         //
 286         // TODO(prattmic): If we split dist bootstrap and dist test then this
 287         // could be simplified to directly use internal/sysinfo here.
 288         return t.dirCmd(filepath.Join(goroot, "src/cmd/internal/metadata"), "go", []string{"run", "main.go"}).Run()
 289 }
 290
 291 // short returns a -short flag value to use with 'go test'
 292 // or a test binary for tests intended to run in short mode.
 293 // It returns "true", unless the environment variable
 294 // GO_TEST_SHORT is set to a non-empty, false-ish string.
 295 //
 296 // This environment variable is meant to be an internal
 297 // detail between the Go build system and cmd/dist for
 298 // the purpose of longtest builders, and is not intended
 299 // for use by users. See golang.org/issue/12508.
 300 func short() string {
 301         if v := os.Getenv("GO_TEST_SHORT"); v != "" {
 302                 short, err := strconv.ParseBool(v)
 303                 if err != nil {
 304                         fatalf("invalid GO_TEST_SHORT %q: %v", v, err)
 305                 }
 306                 if !short {
 307                         return "false"
 308                 }
 309         }
 310         return "true"
 311 }
 312
 313 // goTest returns the beginning of the go test command line.
 314 // Callers should use goTest and then pass flags overriding these
 315 // defaults as later arguments in the command line.
 316 func (t *tester) goTest() []string {
 317         return []string{
 318                 "go", "test", "-short=" + short(), "-count=1", t.tags(), t.runFlag(""),
 319         }
 320 }
 321
 322 func (t *tester) tags() string {
 323         ios := t.iOS()
 324         switch {
 325         case ios && noOpt:
 326                 return "-tags=lldb,noopt"
 327         case ios:
 328                 return "-tags=lldb"
 329         case noOpt:
 330                 return "-tags=noopt"
 331         default:
 332                 return "-tags="
 333         }
 334 }
 335
 336 // timeoutDuration converts the provided number of seconds into a
 337 // time.Duration, scaled by the t.timeoutScale factor.
 338 func (t *tester) timeoutDuration(sec int) time.Duration {
 339         return time.Duration(sec) * time.Second * time.Duration(t.timeoutScale)
 340 }
 341
 342 // timeout returns the "-timeout=" string argument to "go test" given
 343 // the number of seconds of timeout. It scales it by the
 344 // t.timeoutScale factor.
 345 func (t *tester) timeout(sec int) string {
 346         return "-timeout=" + t.timeoutDuration(sec).String()
 347 }
 348
 349 // ranGoTest and stdMatches are state closed over by the stdlib
 350 // testing func in registerStdTest below. The tests are run
 351 // sequentially, so there's no need for locks.
 352 //
 353 // ranGoBench and benchMatches are the same, but are only used
 354 // in -race mode.
 355 var (
 356         ranGoTest  bool
 357         stdMatches []string
 358
 359         ranGoBench   bool
 360         benchMatches []string
 361 )
 362
 363 func (t *tester) registerStdTest(pkg string) {
 364         heading := "Testing packages."
 365         testPrefix := "go_test:"
 366         gcflags := gogcflags
 367
 368         testName := testPrefix + pkg
 369         if t.runRx == nil || t.runRx.MatchString(testName) == t.runRxWant {
 370                 stdMatches = append(stdMatches, pkg)
 371         }
 372
 373         t.tests = append(t.tests, distTest{
 374                 name:    testName,
 375                 heading: heading,
 376                 fn: func(dt *distTest) error {
 377                         if ranGoTest {
 378                                 return nil
 379                         }
 380                         t.runPending(dt)
 381                         timelog("start", dt.name)
 382                         defer timelog("end", dt.name)
 383                         ranGoTest = true
 384
 385                         timeoutSec := 180
 386                         for _, pkg := range stdMatches {
 387                                 if pkg == "cmd/go" {
 388                                         timeoutSec *= 3
 389                                         break
 390                                 }
 391                         }
 392                         args := []string{
 393                                 "test",
 394                                 "-short=" + short(),
 395                                 t.tags(),
 396                                 t.timeout(timeoutSec),
 397                         }
 398                         if gcflags != "" {
 399                                 args = append(args, "-gcflags=all="+gcflags)
 400                         }
 401                         if t.race {
 402                                 args = append(args, "-race")
 403                         }
 404                         if t.msan {
 405                                 args = append(args, "-msan")
 406                         }
 407                         if t.asan {
 408                                 args = append(args, "-asan")
 409                         }
 410                         if t.compileOnly {
 411                                 args = append(args, "-run=^$")
 412                         }
 413                         args = append(args, stdMatches...)
 414                         cmd := exec.Command(gorootBinGo, args...)
 415                         cmd.Stdout = os.Stdout
 416                         cmd.Stderr = os.Stderr
 417                         return cmd.Run()
 418                 },
 419         })
 420 }
 421
 422 func (t *tester) registerRaceBenchTest(pkg string) {
 423         testName := "go_test_bench:" + pkg
 424         if t.runRx == nil || t.runRx.MatchString(testName) == t.runRxWant {
 425                 benchMatches = append(benchMatches, pkg)
 426         }
 427         t.tests = append(t.tests, distTest{
 428                 name:    testName,
 429                 heading: "Running benchmarks briefly.",
 430                 fn: func(dt *distTest) error {
 431                         if ranGoBench {
 432                                 return nil
 433                         }
 434                         t.runPending(dt)
 435                         timelog("start", dt.name)
 436                         defer timelog("end", dt.name)
 437                         ranGoBench = true
 438                         args := []string{
 439                                 "test",
 440                                 "-short=" + short(),
 441                                 "-race",
 442                                 t.timeout(1200), // longer timeout for race with benchmarks
 443                                 "-run=^$",       // nothing. only benchmarks.
 444                                 "-benchtime=.1s",
 445                                 "-cpu=4",
 446                         }
 447                         if !t.compileOnly {
 448                                 args = append(args, "-bench=.*")
 449                         }
 450                         args = append(args, benchMatches...)
 451                         cmd := exec.Command(gorootBinGo, args...)
 452                         cmd.Stdout = os.Stdout
 453                         cmd.Stderr = os.Stderr
 454                         return cmd.Run()
 455                 },
 456         })
 457 }
 458
 459 // stdOutErrAreTerminals is defined in test_linux.go, to report
 460 // whether stdout & stderr are terminals.
 461 var stdOutErrAreTerminals func() bool
 462
 463 func (t *tester) registerTests() {
 464         // Fast path to avoid the ~1 second of `go list std cmd` when
 465         // the caller lists specific tests to run. (as the continuous
 466         // build coordinator does).
 467         if len(t.runNames) > 0 {
 468                 for _, name := range t.runNames {
 469                         if strings.HasPrefix(name, "go_test:") {
 470                                 t.registerStdTest(strings.TrimPrefix(name, "go_test:"))
 471                         }
 472                         if strings.HasPrefix(name, "go_test_bench:") {
 473                                 t.registerRaceBenchTest(strings.TrimPrefix(name, "go_test_bench:"))
 474                         }
 475                 }
 476         } else {
 477                 // Use a format string to only list packages and commands that have tests.
 478                 const format = "{{if (or .TestGoFiles .XTestGoFiles)}}{{.ImportPath}}{{end}}"
 479                 cmd := exec.Command(gorootBinGo, "list", "-f", format)
 480                 if t.race {
 481                         cmd.Args = append(cmd.Args, "-tags=race")
 482                 }
 483                 cmd.Args = append(cmd.Args, "std", "cmd")
 484                 cmd.Stderr = new(bytes.Buffer)
 485                 all, err := cmd.Output()
 486                 if err != nil {
 487                         fatalf("Error running go list std cmd: %v:\n%s", err, cmd.Stderr)
 488                 }
 489                 pkgs := strings.Fields(string(all))
 490                 for _, pkg := range pkgs {
 491                         t.registerStdTest(pkg)
 492                 }
 493                 if t.race {
 494                         for _, pkg := range pkgs {
 495                                 if t.packageHasBenchmarks(pkg) {
 496                                         t.registerRaceBenchTest(pkg)
 497                                 }
 498                         }
 499                 }
 500         }
 501
 502         // Test the os/user package in the pure-Go mode too.
 503         if !t.compileOnly {
 504                 t.tests = append(t.tests, distTest{
 505                         name:    "osusergo",
 506                         heading: "os/user with tag osusergo",
 507                         fn: func(dt *distTest) error {
 508                                 t.addCmd(dt, "src", t.goTest(), t.timeout(300), "-tags=osusergo", "os/user")
 509                                 return nil
 510                         },
 511                 })
 512         }
 513
 514         // Test ios/amd64 for the iOS simulator.
 515         if goos == "darwin" && goarch == "amd64" && t.cgoEnabled {
 516                 t.tests = append(t.tests, distTest{
 517                         name:    "amd64ios",
 518                         heading: "GOOS=ios on darwin/amd64",
 519                         fn: func(dt *distTest) error {
 520                                 cmd := t.addCmd(dt, "src", t.goTest(), t.timeout(300), "-run=SystemRoots", "crypto/x509")
 521                                 setEnv(cmd, "GOOS", "ios")
 522                                 setEnv(cmd, "CGO_ENABLED", "1")
 523                                 return nil
 524                         },
 525                 })
 526         }
 527
 528         if t.race {
 529                 return
 530         }
 531
 532         // Runtime CPU tests.
 533         if !t.compileOnly && goos != "js" { // js can't handle -cpu != 1
 534                 testName := "runtime:cpu124"
 535                 t.tests = append(t.tests, distTest{
 536                         name:    testName,
 537                         heading: "GOMAXPROCS=2 runtime -cpu=1,2,4 -quick",
 538                         fn: func(dt *distTest) error {
 539                                 cmd := t.addCmd(dt, "src", t.goTest(), "-short=true", t.timeout(300), "runtime", "-cpu=1,2,4", "-quick")
 540                                 // We set GOMAXPROCS=2 in addition to -cpu=1,2,4 in order to test runtime bootstrap code,
 541                                 // creation of first goroutines and first garbage collections in the parallel setting.
 542                                 setEnv(cmd, "GOMAXPROCS", "2")
 543                                 return nil
 544                         },
 545                 })
 546         }
 547
 548         // morestack tests. We only run these on in long-test mode
 549         // (with GO_TEST_SHORT=false) because the runtime test is
 550         // already quite long and mayMoreStackMove makes it about
 551         // twice as slow.
 552         if !t.compileOnly && short() == "false" {
 553                 // hooks is the set of maymorestack hooks to test with.
 554                 hooks := []string{"mayMoreStackPreempt", "mayMoreStackMove"}
 555                 // pkgs is the set of test packages to run.
 556                 pkgs := []string{"runtime", "reflect", "sync"}
 557                 // hookPkgs is the set of package patterns to apply
 558                 // the maymorestack hook to.
 559                 hookPkgs := []string{"runtime/...", "reflect", "sync"}
 560                 // unhookPkgs is the set of package patterns to
 561                 // exclude from hookPkgs.
 562                 unhookPkgs := []string{"runtime/testdata/..."}
 563                 for _, hook := range hooks {
 564                         // Construct the build flags to use the
 565                         // maymorestack hook in the compiler and
 566                         // assembler. We pass this via the GOFLAGS
 567                         // environment variable so that it applies to
 568                         // both the test itself and to binaries built
 569                         // by the test.
 570                         goFlagsList := []string{}
 571                         for _, flag := range []string{"-gcflags", "-asmflags"} {
 572                                 for _, hookPkg := range hookPkgs {
 573                                         goFlagsList = append(goFlagsList, flag+"="+hookPkg+"=-d=maymorestack=runtime."+hook)
 574                                 }
 575                                 for _, unhookPkg := range unhookPkgs {
 576                                         goFlagsList = append(goFlagsList, flag+"="+unhookPkg+"=")
 577                                 }
 578                         }
 579                         goFlags := strings.Join(goFlagsList, " ")
 580
 581                         for _, pkg := range pkgs {
 582                                 pkg := pkg
 583                                 testName := hook + ":" + pkg
 584                                 t.tests = append(t.tests, distTest{
 585                                         name:    testName,
 586                                         heading: "maymorestack=" + hook,
 587                                         fn: func(dt *distTest) error {
 588                                                 cmd := t.addCmd(dt, "src", t.goTest(), t.timeout(600), pkg, "-short")
 589                                                 setEnv(cmd, "GOFLAGS", goFlags)
 590                                                 return nil
 591                                         },
 592                                 })
 593                         }
 594                 }
 595         }
 596
 597         // This test needs its stdout/stderr to be terminals, so we don't run it from cmd/go's tests.
 598         // See issue 18153.
 599         if goos == "linux" {
 600                 t.tests = append(t.tests, distTest{
 601                         name:    "cmd_go_test_terminal",
 602                         heading: "cmd/go terminal test",
 603                         fn: func(dt *distTest) error {
 604                                 t.runPending(dt)
 605                                 timelog("start", dt.name)
 606                                 defer timelog("end", dt.name)
 607                                 if !stdOutErrAreTerminals() {
 608                                         fmt.Println("skipping terminal test; stdout/stderr not terminals")
 609                                         return nil
 610                                 }
 611                                 cmd := exec.Command(gorootBinGo, "test")
 612                                 setDir(cmd, filepath.Join(os.Getenv("GOROOT"), "src/cmd/go/testdata/testterminal18153"))
 613                                 cmd.Stdout = os.Stdout
 614                                 cmd.Stderr = os.Stderr
 615                                 return cmd.Run()
 616                         },
 617                 })
 618         }
 619
 620         // On the builders only, test that a moved GOROOT still works.
 621         // Fails on iOS because CC_FOR_TARGET refers to clangwrap.sh
 622         // in the unmoved GOROOT.
 623         // Fails on Android and js/wasm with an exec format error.
 624         // Fails on plan9 with "cannot find GOROOT" (issue #21016).
 625         if os.Getenv("GO_BUILDER_NAME") != "" && goos != "android" && !t.iOS() && goos != "plan9" && goos != "js" {
 626                 t.tests = append(t.tests, distTest{
 627                         name:    "moved_goroot",
 628                         heading: "moved GOROOT",
 629                         fn: func(dt *distTest) error {
 630                                 t.runPending(dt)
 631                                 timelog("start", dt.name)
 632                                 defer timelog("end", dt.name)
 633                                 moved := goroot + "-moved"
 634                                 if err := os.Rename(goroot, moved); err != nil {
 635                                         if goos == "windows" {
 636                                                 // Fails on Windows (with "Access is denied") if a process
 637                                                 // or binary is in this directory. For instance, using all.bat
 638                                                 // when run from c:\workdir\go\src fails here
 639                                                 // if GO_BUILDER_NAME is set. Our builders invoke tests
 640                                                 // a different way which happens to work when sharding
 641                                                 // tests, but we should be tolerant of the non-sharded
 642                                                 // all.bat case.
 643                                                 log.Printf("skipping test on Windows")
 644                                                 return nil
 645                                         }
 646                                         return err
 647                                 }
 648
 649                                 // Run `go test fmt` in the moved GOROOT, without explicitly setting
 650                                 // GOROOT in the environment. The 'go' command should find itself.
 651                                 cmd := exec.Command(filepath.Join(moved, "bin", "go"), "test", "fmt")
 652                                 cmd.Stdout = os.Stdout
 653                                 cmd.Stderr = os.Stderr
 654                                 unsetEnv(cmd, "GOROOT")
 655                                 unsetEnv(cmd, "GOCACHE") // TODO(bcmills): ...why‽
 656                                 err := cmd.Run()
 657
 658                                 if rerr := os.Rename(moved, goroot); rerr != nil {
 659                                         fatalf("failed to restore GOROOT: %v", rerr)
 660                                 }
 661                                 return err
 662                         },
 663                 })
 664         }
 665
 666         // Test that internal linking of standard packages does not
 667         // require libgcc. This ensures that we can install a Go
 668         // release on a system that does not have a C compiler
 669         // installed and still build Go programs (that don't use cgo).
 670         for _, pkg := range cgoPackages {
 671                 if !t.internalLink() {
 672                         break
 673                 }
 674
 675                 // ARM libgcc may be Thumb, which internal linking does not support.
 676                 if goarch == "arm" {
 677                         break
 678                 }
 679
 680                 pkg := pkg
 681                 var run string
 682                 if pkg == "net" {
 683                         run = "TestTCPStress"
 684                 }
 685                 t.tests = append(t.tests, distTest{
 686                         name:    "nolibgcc:" + pkg,
 687                         heading: "Testing without libgcc.",
 688                         fn: func(dt *distTest) error {
 689                                 // What matters is that the tests build and start up.
 690                                 // Skip expensive tests, especially x509 TestSystemRoots.
 691                                 t.addCmd(dt, "src", t.goTest(), "-ldflags=-linkmode=internal -libgcc=none", "-run=^Test[^CS]", pkg, t.runFlag(run))
 692                                 return nil
 693                         },
 694                 })
 695         }
 696
 697         // Stub out following test on alpine until 54354 resolved.
 698         builderName := os.Getenv("GO_BUILDER_NAME")
 699         disablePIE := strings.HasSuffix(builderName, "-alpine")
 700
 701         // Test internal linking of PIE binaries where it is supported.
 702         if t.internalLinkPIE() && !disablePIE {
 703                 t.tests = append(t.tests, distTest{
 704                         name:    "pie_internal",
 705                         heading: "internal linking of -buildmode=pie",
 706                         fn: func(dt *distTest) error {
 707                                 cmd := t.addCmd(dt, "src", t.goTest(), "reflect", "-buildmode=pie", "-ldflags=-linkmode=internal", t.timeout(60))
 708                                 setEnv(cmd, "CGO_ENABLED", "0")
 709                                 return nil
 710                         },
 711                 })
 712                 // Also test a cgo package.
 713                 if t.cgoEnabled && t.internalLink() && !disablePIE {
 714                         t.tests = append(t.tests, distTest{
 715                                 name:    "pie_internal_cgo",
 716                                 heading: "internal linking of -buildmode=pie",
 717                                 fn: func(dt *distTest) error {
 718                                         t.addCmd(dt, "src", t.goTest(), "os/user", "-buildmode=pie", "-ldflags=-linkmode=internal", t.timeout(60))
 719                                         return nil
 720                                 },
 721                         })
 722                 }
 723         }
 724
 725         // sync tests
 726         if goos != "js" { // js doesn't support -cpu=10
 727                 t.tests = append(t.tests, distTest{
 728                         name:    "sync_cpu",
 729                         heading: "sync -cpu=10",
 730                         fn: func(dt *distTest) error {
 731                                 t.addCmd(dt, "src", t.goTest(), "sync", t.timeout(120), "-cpu=10", t.runFlag(""))
 732                                 return nil
 733                         },
 734                 })
 735         }
 736
 737         if t.raceDetectorSupported() {
 738                 t.tests = append(t.tests, distTest{
 739                         name:    "race",
 740                         heading: "Testing race detector",
 741                         fn:      t.raceTest,
 742                 })
 743         }
 744
 745         if t.cgoEnabled && !t.iOS() {
 746                 // Disabled on iOS. golang.org/issue/15919
 747                 t.registerHostTest("cgo_stdio", "../misc/cgo/stdio", "misc/cgo/stdio", ".")
 748                 t.registerHostTest("cgo_life", "../misc/cgo/life", "misc/cgo/life", ".")
 749                 if goos != "android" {
 750                         t.registerHostTest("cgo_fortran", "../misc/cgo/fortran", "misc/cgo/fortran", ".")
 751                 }
 752                 if t.hasSwig() && goos != "android" {
 753                         t.tests = append(t.tests, distTest{
 754                                 name:    "swig_stdio",
 755                                 heading: "../misc/swig/stdio",
 756                                 fn: func(dt *distTest) error {
 757                                         t.addCmd(dt, "misc/swig/stdio", t.goTest(), ".")
 758                                         return nil
 759                                 },
 760                         })
 761                         if t.hasCxx() {
 762                                 t.tests = append(t.tests,
 763                                         distTest{
 764                                                 name:    "swig_callback",
 765                                                 heading: "../misc/swig/callback",
 766                                                 fn: func(dt *distTest) error {
 767                                                         t.addCmd(dt, "misc/swig/callback", t.goTest(), ".")
 768                                                         return nil
 769                                                 },
 770                                         },
 771                                         distTest{
 772                                                 name:    "swig_callback_lto",
 773                                                 heading: "../misc/swig/callback",
 774                                                 fn: func(dt *distTest) error {
 775                                                         cmd := t.addCmd(dt, "misc/swig/callback", t.goTest(), ".")
 776                                                         setEnv(cmd, "CGO_CFLAGS", "-flto -Wno-lto-type-mismatch -Wno-unknown-warning-option")
 777                                                         setEnv(cmd, "CGO_CXXFLAGS", "-flto -Wno-lto-type-mismatch -Wno-unknown-warning-option")
 778                                                         setEnv(cmd, "CGO_LDFLAGS", "-flto -Wno-lto-type-mismatch -Wno-unknown-warning-option")
 779                                                         return nil
 780                                                 },
 781                                         },
 782                                 )
 783                         }
 784                 }
 785         }
 786         if t.cgoEnabled {
 787                 t.tests = append(t.tests, distTest{
 788                         name:    "cgo_test",
 789                         heading: "../misc/cgo/test",
 790                         fn:      t.cgoTest,
 791                 })
 792         }
 793
 794         // Don't run these tests with $GO_GCFLAGS because most of them
 795         // assume that they can run "go install" with no -gcflags and not
 796         // recompile the entire standard library. If make.bash ran with
 797         // special -gcflags, that's not true.
 798         if t.cgoEnabled && gogcflags == "" {
 799                 t.registerHostTest("testgodefs", "../misc/cgo/testgodefs", "misc/cgo/testgodefs", ".")
 800
 801                 t.registerTest("testso", "../misc/cgo/testso", t.goTest(), t.timeout(600), ".")
 802                 t.registerTest("testsovar", "../misc/cgo/testsovar", t.goTest(), t.timeout(600), ".")
 803                 if t.supportedBuildmode("c-archive") {
 804                         t.registerHostTest("testcarchive", "../misc/cgo/testcarchive", "misc/cgo/testcarchive", ".")
 805                 }
 806                 if t.supportedBuildmode("c-shared") {
 807                         t.registerHostTest("testcshared", "../misc/cgo/testcshared", "misc/cgo/testcshared", ".")
 808                 }
 809                 if t.supportedBuildmode("shared") {
 810                         t.registerTest("testshared", "../misc/cgo/testshared", t.goTest(), t.timeout(600), ".")
 811                 }
 812                 if t.supportedBuildmode("plugin") {
 813                         t.registerTest("testplugin", "../misc/cgo/testplugin", t.goTest(), t.timeout(600), ".")
 814                 }
 815                 if goos == "linux" || (goos == "freebsd" && goarch == "amd64") {
 816                         // because Pdeathsig of syscall.SysProcAttr struct used in misc/cgo/testsanitizers is only
 817                         // supported on Linux and FreeBSD.
 818                         t.registerHostTest("testsanitizers", "../misc/cgo/testsanitizers", "misc/cgo/testsanitizers", ".")
 819                 }
 820                 if t.hasBash() && goos != "android" && !t.iOS() && gohostos != "windows" {
 821                         t.registerHostTest("cgo_errors", "../misc/cgo/errors", "misc/cgo/errors", ".")
 822                 }
 823         }
 824
 825         if goos != "android" && !t.iOS() {
 826                 // There are no tests in this directory, only benchmarks.
 827                 // Check that the test binary builds but don't bother running it.
 828                 // (It has init-time work to set up for the benchmarks that is not worth doing unnecessarily.)
 829                 t.registerTest("bench_go1", "../test/bench/go1", t.goTest(), "-c", "-o="+os.DevNull)
 830         }
 831         if goos != "android" && !t.iOS() {
 832                 // Only start multiple test dir shards on builders,
 833                 // where they get distributed to multiple machines.
 834                 // See issues 20141 and 31834.
 835                 nShards := 1
 836                 if os.Getenv("GO_BUILDER_NAME") != "" {
 837                         nShards = 10
 838                 }
 839                 if n, err := strconv.Atoi(os.Getenv("GO_TEST_SHARDS")); err == nil {
 840                         nShards = n
 841                 }
 842                 for shard := 0; shard < nShards; shard++ {
 843                         shard := shard
 844                         t.tests = append(t.tests, distTest{
 845                                 name:    fmt.Sprintf("test:%d_%d", shard, nShards),
 846                                 heading: "../test",
 847                                 fn:      func(dt *distTest) error { return t.testDirTest(dt, shard, nShards) },
 848                         })
 849                 }
 850         }
 851         // Only run the API check on fast development platforms.
 852         // Every platform checks the API on every GOOS/GOARCH/CGO_ENABLED combination anyway,
 853         // so we really only need to run this check once anywhere to get adequate coverage.
 854         // To help developers avoid trybot-only failures, we try to run on typical developer machines
 855         // which is darwin/linux/windows and amd64/arm64.
 856         if (goos == "darwin" || goos == "linux" || goos == "windows") && (goarch == "amd64" || goarch == "arm64") {
 857                 t.tests = append(t.tests, distTest{
 858                         name:    "api",
 859                         heading: "API check",
 860                         fn: func(dt *distTest) error {
 861                                 if t.compileOnly {
 862                                         t.addCmd(dt, "src", "go", "build", "-o", os.DevNull, filepath.Join(goroot, "src/cmd/api/run.go"))
 863                                         return nil
 864                                 }
 865                                 t.addCmd(dt, "src", "go", "run", filepath.Join(goroot, "src/cmd/api/run.go"))
 866                                 return nil
 867                         },
 868                 })
 869         }
 870
 871         // Ensure that the toolchain can bootstrap itself.
 872         // This test adds another ~45s to all.bash if run sequentially, so run it only on the builders.
 873         if os.Getenv("GO_BUILDER_NAME") != "" && goos != "android" && !t.iOS() {
 874                 t.registerHostTest("reboot", "../misc/reboot", "misc/reboot", ".")
 875         }
 876 }
 877
 878 // isRegisteredTestName reports whether a test named testName has already
 879 // been registered.
 880 func (t *tester) isRegisteredTestName(testName string) bool {
 881         for _, tt := range t.tests {
 882                 if tt.name == testName {
 883                         return true
 884                 }
 885         }
 886         return false
 887 }
 888
 889 func (t *tester) registerTest1(seq bool, name, dirBanner string, cmdline ...interface{}) {
 890         bin, args := flattenCmdline(cmdline)
 891         if bin == "time" && !t.haveTime {
 892                 bin, args = args[0], args[1:]
 893         }
 894         if t.isRegisteredTestName(name) {
 895                 panic("duplicate registered test name " + name)
 896         }
 897         t.tests = append(t.tests, distTest{
 898                 name:    name,
 899                 heading: dirBanner,
 900                 fn: func(dt *distTest) error {
 901                         if seq {
 902                                 t.runPending(dt)
 903                                 timelog("start", name)
 904                                 defer timelog("end", name)
 905                                 return t.dirCmd(filepath.Join(goroot, "src", dirBanner), bin, args).Run()
 906                         }
 907                         t.addCmd(dt, filepath.Join(goroot, "src", dirBanner), bin, args)
 908                         return nil
 909                 },
 910         })
 911 }
 912
 913 func (t *tester) registerTest(name, dirBanner string, cmdline ...interface{}) {
 914         t.registerTest1(false, name, dirBanner, cmdline...)
 915 }
 916
 917 func (t *tester) registerSeqTest(name, dirBanner string, cmdline ...interface{}) {
 918         t.registerTest1(true, name, dirBanner, cmdline...)
 919 }
 920
 921 func (t *tester) bgDirCmd(dir, bin string, args ...string) *exec.Cmd {
 922         cmd := exec.Command(bin, args...)
 923         if filepath.IsAbs(dir) {
 924                 setDir(cmd, dir)
 925         } else {
 926                 setDir(cmd, filepath.Join(goroot, dir))
 927         }
 928         return cmd
 929 }
 930
 931 func (t *tester) dirCmd(dir string, cmdline ...interface{}) *exec.Cmd {
 932         bin, args := flattenCmdline(cmdline)
 933         cmd := t.bgDirCmd(dir, bin, args...)
 934         cmd.Stdout = os.Stdout
 935         cmd.Stderr = os.Stderr
 936         if vflag > 1 {
 937                 errprintf("%s\n", strings.Join(cmd.Args, " "))
 938         }
 939         return cmd
 940 }
 941
 942 // flattenCmdline flattens a mixture of string and []string as single list
 943 // and then interprets it as a command line: first element is binary, then args.
 944 func flattenCmdline(cmdline []interface{}) (bin string, args []string) {
 945         var list []string
 946         for _, x := range cmdline {
 947                 switch x := x.(type) {
 948                 case string:
 949                         list = append(list, x)
 950                 case []string:
 951                         list = append(list, x...)
 952                 default:
 953                         panic("invalid addCmd argument type: " + reflect.TypeOf(x).String())
 954                 }
 955         }
 956
 957         // The go command is too picky about duplicated flags.
 958         // Drop all but the last of the allowed duplicated flags.
 959         drop := make([]bool, len(list))
 960         have := map[string]int{}
 961         for i := 1; i < len(list); i++ {
 962                 j := strings.Index(list[i], "=")
 963                 if j < 0 {
 964                         continue
 965                 }
 966                 flag := list[i][:j]
 967                 switch flag {
 968                 case "-run", "-tags":
 969                         if have[flag] != 0 {
 970                                 drop[have[flag]] = true
 971                         }
 972                         have[flag] = i
 973                 }
 974         }
 975         out := list[:0]
 976         for i, x := range list {
 977                 if !drop[i] {
 978                         out = append(out, x)
 979                 }
 980         }
 981         list = out
 982
 983         bin = list[0]
 984         if bin == "go" {
 985                 bin = gorootBinGo
 986         }
 987         return bin, list[1:]
 988 }
 989
 990 func (t *tester) addCmd(dt *distTest, dir string, cmdline ...interface{}) *exec.Cmd {
 991         bin, args := flattenCmdline(cmdline)
 992         w := &work{
 993                 dt:  dt,
 994                 cmd: t.bgDirCmd(dir, bin, args...),
 995         }
 996         t.worklist = append(t.worklist, w)
 997         return w.cmd
 998 }
 999
1000 func (t *tester) iOS() bool {
1001         return goos == "ios"
1002 }
1003
1004 func (t *tester) out(v string) {
1005         if t.banner == "" {
1006                 return
1007         }
1008         fmt.Println("\n" + t.banner + v)
1009 }
1010
1011 func (t *tester) extLink() bool {
1012         pair := gohostos + "-" + goarch
1013         switch pair {
1014         case "aix-ppc64",
1015                 "android-arm", "android-arm64",
1016                 "darwin-amd64", "darwin-arm64",
1017                 "dragonfly-amd64",
1018                 "freebsd-386", "freebsd-amd64", "freebsd-arm", "freebsd-riscv64",
1019                 "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-loong64", "linux-ppc64le", "linux-mips64", "linux-mips64le", "linux-mips", "linux-mipsle", "linux-riscv64", "linux-s390x",
1020                 "netbsd-386", "netbsd-amd64",
1021                 "openbsd-386", "openbsd-amd64",
1022                 "windows-386", "windows-amd64":
1023                 return true
1024         }
1025         return false
1026 }
1027
1028 func (t *tester) internalLink() bool {
1029         if gohostos == "dragonfly" {
1030                 // linkmode=internal fails on dragonfly since errno is a TLS relocation.
1031                 return false
1032         }
1033         if goos == "android" {
1034                 return false
1035         }
1036         if goos == "ios" {
1037                 return false
1038         }
1039         if goos == "windows" && goarch == "arm64" {
1040                 return false
1041         }
1042         // Internally linking cgo is incomplete on some architectures.
1043         // https://golang.org/issue/10373
1044         // https://golang.org/issue/14449
1045         if goarch == "loong64" || goarch == "mips64" || goarch == "mips64le" || goarch == "mips" || goarch == "mipsle" || goarch == "riscv64" {
1046                 return false
1047         }
1048         if goos == "aix" {
1049                 // linkmode=internal isn't supported.
1050                 return false
1051         }
1052         return true
1053 }
1054
1055 func (t *tester) internalLinkPIE() bool {
1056         switch goos + "-" + goarch {
1057         case "darwin-amd64", "darwin-arm64",
1058                 "linux-amd64", "linux-arm64", "linux-ppc64le",
1059                 "android-arm64",
1060                 "windows-amd64", "windows-386", "windows-arm":
1061                 return true
1062         }
1063         return false
1064 }
1065
1066 func (t *tester) supportedBuildmode(mode string) bool {
1067         pair := goos + "-" + goarch
1068         switch mode {
1069         case "c-archive":
1070                 if !t.extLink() {
1071                         return false
1072                 }
1073                 switch pair {
1074                 case "aix-ppc64",
1075                         "darwin-amd64", "darwin-arm64", "ios-arm64",
1076                         "linux-amd64", "linux-386", "linux-ppc64le", "linux-riscv64", "linux-s390x",
1077                         "freebsd-amd64",
1078                         "windows-amd64", "windows-386":
1079                         return true
1080                 }
1081                 return false
1082         case "c-shared":
1083                 switch pair {
1084                 case "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-riscv64", "linux-s390x",
1085                         "darwin-amd64", "darwin-arm64",
1086                         "freebsd-amd64",
1087                         "android-arm", "android-arm64", "android-386",
1088                         "windows-amd64", "windows-386", "windows-arm64":
1089                         return true
1090                 }
1091                 return false
1092         case "shared":
1093                 switch pair {
1094                 case "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-s390x":
1095                         return true
1096                 }
1097                 return false
1098         case "plugin":
1099                 switch pair {
1100                 case "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-s390x", "linux-ppc64le":
1101                         return true
1102                 case "darwin-amd64", "darwin-arm64":
1103                         return true
1104                 case "freebsd-amd64":
1105                         return true
1106                 }
1107                 return false
1108         case "pie":
1109                 switch pair {
1110                 case "aix/ppc64",
1111                         "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-riscv64", "linux-s390x",
1112                         "android-amd64", "android-arm", "android-arm64", "android-386":
1113                         return true
1114                 case "darwin-amd64", "darwin-arm64":
1115                         return true
1116                 case "windows-amd64", "windows-386", "windows-arm":
1117                         return true
1118                 }
1119                 return false
1120
1121         default:
1122                 fatalf("internal error: unknown buildmode %s", mode)
1123                 return false
1124         }
1125 }
1126
1127 func (t *tester) registerHostTest(name, heading, dir, pkg string) {
1128         t.tests = append(t.tests, distTest{
1129                 name:    name,
1130                 heading: heading,
1131                 fn: func(dt *distTest) error {
1132                         t.runPending(dt)
1133                         timelog("start", name)
1134                         defer timelog("end", name)
1135                         return t.runHostTest(dir, pkg)
1136                 },
1137         })
1138 }
1139
1140 func (t *tester) runHostTest(dir, pkg string) error {
1141         out, err := exec.Command(gorootBinGo, "env", "GOEXE", "GOTMPDIR").Output()
1142         if err != nil {
1143                 return err
1144         }
1145
1146         parts := strings.Split(string(out), "\n")
1147         if len(parts) < 2 {
1148                 return fmt.Errorf("'go env GOEXE GOTMPDIR' output contains <2 lines")
1149         }
1150         GOEXE := strings.TrimSpace(parts[0])
1151         GOTMPDIR := strings.TrimSpace(parts[1])
1152
1153         f, err := os.CreateTemp(GOTMPDIR, "test.test-*"+GOEXE)
1154         if err != nil {
1155                 return err
1156         }
1157         f.Close()
1158         defer os.Remove(f.Name())
1159
1160         cmd := t.dirCmd(dir, t.goTest(), "-c", "-o", f.Name(), pkg)
1161         setEnv(cmd, "GOARCH", gohostarch)
1162         setEnv(cmd, "GOOS", gohostos)
1163         if err := cmd.Run(); err != nil {
1164                 return err
1165         }
1166         return t.dirCmd(dir, f.Name(), "-test.short="+short(), "-test.timeout="+t.timeoutDuration(300).String()).Run()
1167 }
1168
1169 func (t *tester) cgoTest(dt *distTest) error {
1170         cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), ".")
1171         setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=auto")
1172
1173         // Stub out various buildmode=pie tests  on alpine until 54354 resolved.
1174         builderName := os.Getenv("GO_BUILDER_NAME")
1175         disablePIE := strings.HasSuffix(builderName, "-alpine")
1176
1177         if t.internalLink() {
1178                 cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=internal", ".")
1179                 setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=internal")
1180         }
1181
1182         pair := gohostos + "-" + goarch
1183         switch pair {
1184         case "darwin-amd64", "darwin-arm64",
1185                 "windows-386", "windows-amd64":
1186                 // test linkmode=external, but __thread not supported, so skip testtls.
1187                 if !t.extLink() {
1188                         break
1189                 }
1190                 cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), ".")
1191                 setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=external")
1192
1193                 t.addCmd(dt, "misc/cgo/test", t.goTest(), "-ldflags", "-linkmode=external -s", ".")
1194
1195                 if t.supportedBuildmode("pie") && !disablePIE {
1196
1197                         t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", ".")
1198                         if t.internalLink() && t.internalLinkPIE() {
1199                                 t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", "-ldflags=-linkmode=internal", "-tags=internal,internal_pie", ".")
1200                         }
1201                 }
1202
1203         case "aix-ppc64",
1204                 "android-arm", "android-arm64",
1205                 "dragonfly-amd64",
1206                 "freebsd-386", "freebsd-amd64", "freebsd-arm", "freebsd-riscv64",
1207                 "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-riscv64", "linux-s390x",
1208                 "netbsd-386", "netbsd-amd64",
1209                 "openbsd-386", "openbsd-amd64", "openbsd-arm", "openbsd-arm64", "openbsd-mips64":
1210
1211                 cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), ".")
1212                 setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=external")
1213                 // cgo should be able to cope with both -g arguments and colored
1214                 // diagnostics.
1215                 setEnv(cmd, "CGO_CFLAGS", "-g0 -fdiagnostics-color")
1216
1217                 t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-ldflags", "-linkmode=auto", ".")
1218                 t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-ldflags", "-linkmode=external", ".")
1219
1220                 switch pair {
1221                 case "aix-ppc64", "netbsd-386", "netbsd-amd64":
1222                         // no static linking
1223                 case "freebsd-arm":
1224                         // -fPIC compiled tls code will use __tls_get_addr instead
1225                         // of __aeabi_read_tp, however, on FreeBSD/ARM, __tls_get_addr
1226                         // is implemented in rtld-elf, so -fPIC isn't compatible with
1227                         // static linking on FreeBSD/ARM with clang. (cgo depends on
1228                         // -fPIC fundamentally.)
1229                 default:
1230                         cmd := t.dirCmd("misc/cgo/test",
1231                                 compilerEnvLookup(defaultcc, goos, goarch), "-xc", "-o", "/dev/null", "-static", "-")
1232                         cmd.Stdin = strings.NewReader("int main() {}")
1233                         if err := cmd.Run(); err != nil {
1234                                 fmt.Println("No support for static linking found (lacks libc.a?), skip cgo static linking test.")
1235                         } else {
1236                                 if goos != "android" {
1237                                         t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-ldflags", `-linkmode=external -extldflags "-static -pthread"`, ".")
1238                                 }
1239                                 t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), ".")
1240                                 t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), "-ldflags", `-linkmode=external`, ".")
1241                                 if goos != "android" {
1242                                         t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), "-ldflags", `-linkmode=external -extldflags "-static -pthread"`, ".")
1243                                         t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=static", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`, ".")
1244                                         // -static in CGO_LDFLAGS triggers a different code path
1245                                         // than -static in -extldflags, so test both.
1246                                         // See issue #16651.
1247                                         cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=static", ".")
1248                                         setEnv(cmd, "CGO_LDFLAGS", "-static -pthread")
1249                                 }
1250                         }
1251
1252                         if t.supportedBuildmode("pie") && !disablePIE {
1253                                 t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", ".")
1254                                 if t.internalLink() && t.internalLinkPIE() {
1255                                         t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", "-ldflags=-linkmode=internal", "-tags=internal,internal_pie", ".")
1256                                 }
1257                                 t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-buildmode=pie", ".")
1258                                 t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), "-buildmode=pie", ".")
1259                         }
1260                 }
1261         }
1262
1263         return nil
1264 }
1265
1266 // run pending test commands, in parallel, emitting headers as appropriate.
1267 // When finished, emit header for nextTest, which is going to run after the
1268 // pending commands are done (and runPending returns).
1269 // A test should call runPending if it wants to make sure that it is not
1270 // running in parallel with earlier tests, or if it has some other reason
1271 // for needing the earlier tests to be done.
1272 func (t *tester) runPending(nextTest *distTest) {
1273         checkNotStale("go", "std")
1274         worklist := t.worklist
1275         t.worklist = nil
1276         for _, w := range worklist {
1277                 w.start = make(chan bool)
1278                 w.end = make(chan bool)
1279                 go func(w *work) {
1280                         if !<-w.start {
1281                                 timelog("skip", w.dt.name)
1282                                 w.out = []byte(fmt.Sprintf("skipped due to earlier error\n"))
1283                         } else {
1284                                 timelog("start", w.dt.name)
1285                                 w.out, w.err = w.cmd.CombinedOutput()
1286                                 if w.err != nil {
1287                                         if isUnsupportedVMASize(w) {
1288                                                 timelog("skip", w.dt.name)
1289                                                 w.out = []byte(fmt.Sprintf("skipped due to unsupported VMA\n"))
1290                                                 w.err = nil
1291                                         }
1292                                 }
1293                         }
1294                         timelog("end", w.dt.name)
1295                         w.end <- true
1296                 }(w)
1297         }
1298
1299         started := 0
1300         ended := 0
1301         var last *distTest
1302         for ended < len(worklist) {
1303                 for started < len(worklist) && started-ended < maxbg {
1304                         w := worklist[started]
1305                         started++
1306                         w.start <- !t.failed || t.keepGoing
1307                 }
1308                 w := worklist[ended]
1309                 dt := w.dt
1310                 if dt.heading != "" && t.lastHeading != dt.heading {
1311                         t.lastHeading = dt.heading
1312                         t.out(dt.heading)
1313                 }
1314                 if dt != last {
1315                         // Assumes all the entries for a single dt are in one worklist.
1316                         last = w.dt
1317                         if vflag > 0 {
1318                                 fmt.Printf("# go tool dist test -run=^%s$\n", dt.name)
1319                         }
1320                 }
1321                 if vflag > 1 {
1322                         errprintf("%s\n", strings.Join(w.cmd.Args, " "))
1323                 }
1324                 ended++
1325                 <-w.end
1326                 os.Stdout.Write(w.out)
1327                 if w.err != nil {
1328                         log.Printf("Failed: %v", w.err)
1329                         t.failed = true
1330                 }
1331                 checkNotStale("go", "std")
1332         }
1333         if t.failed && !t.keepGoing {
1334                 fatalf("FAILED")
1335         }
1336
1337         if dt := nextTest; dt != nil {
1338                 if dt.heading != "" && t.lastHeading != dt.heading {
1339                         t.lastHeading = dt.heading
1340                         t.out(dt.heading)
1341                 }
1342                 if vflag > 0 {
1343                         fmt.Printf("# go tool dist test -run=^%s$\n", dt.name)
1344                 }
1345         }
1346 }
1347
1348 func (t *tester) hasBash() bool {
1349         switch gohostos {
1350         case "windows", "plan9":
1351                 return false
1352         }
1353         return true
1354 }
1355
1356 func (t *tester) hasCxx() bool {
1357         cxx, _ := exec.LookPath(compilerEnvLookup(defaultcxx, goos, goarch))
1358         return cxx != ""
1359 }
1360
1361 func (t *tester) hasSwig() bool {
1362         swig, err := exec.LookPath("swig")
1363         if err != nil {
1364                 return false
1365         }
1366
1367         // Check that swig was installed with Go support by checking
1368         // that a go directory exists inside the swiglib directory.
1369         // See https://golang.org/issue/23469.
1370         output, err := exec.Command(swig, "-go", "-swiglib").Output()
1371         if err != nil {
1372                 return false
1373         }
1374         swigDir := strings.TrimSpace(string(output))
1375
1376         _, err = os.Stat(filepath.Join(swigDir, "go"))
1377         if err != nil {
1378                 return false
1379         }
1380
1381         // Check that swig has a new enough version.
1382         // See https://golang.org/issue/22858.
1383         out, err := exec.Command(swig, "-version").CombinedOutput()
1384         if err != nil {
1385                 return false
1386         }
1387
1388         re := regexp.MustCompile(`[vV]ersion +(\d+)([.]\d+)?([.]\d+)?`)
1389         matches := re.FindSubmatch(out)
1390         if matches == nil {
1391                 // Can't find version number; hope for the best.
1392                 return true
1393         }
1394
1395         major, err := strconv.Atoi(string(matches[1]))
1396         if err != nil {
1397                 // Can't find version number; hope for the best.
1398                 return true
1399         }
1400         if major < 3 {
1401                 return false
1402         }
1403         if major > 3 {
1404                 // 4.0 or later
1405                 return true
1406         }
1407
1408         // We have SWIG version 3.x.
1409         if len(matches[2]) > 0 {
1410                 minor, err := strconv.Atoi(string(matches[2][1:]))
1411                 if err != nil {
1412                         return true
1413                 }
1414                 if minor > 0 {
1415                         // 3.1 or later
1416                         return true
1417                 }
1418         }
1419
1420         // We have SWIG version 3.0.x.
1421         if len(matches[3]) > 0 {
1422                 patch, err := strconv.Atoi(string(matches[3][1:]))
1423                 if err != nil {
1424                         return true
1425                 }
1426                 if patch < 6 {
1427                         // Before 3.0.6.
1428                         return false
1429                 }
1430         }
1431
1432         return true
1433 }
1434
1435 func (t *tester) raceDetectorSupported() bool {
1436         if gohostos != goos {
1437                 return false
1438         }
1439         if !t.cgoEnabled {
1440                 return false
1441         }
1442         if !raceDetectorSupported(goos, goarch) {
1443                 return false
1444         }
1445         // The race detector doesn't work on Alpine Linux:
1446         // golang.org/issue/14481
1447         if isAlpineLinux() {
1448                 return false
1449         }
1450         // NetBSD support is unfinished.
1451         // golang.org/issue/26403
1452         if goos == "netbsd" {
1453                 return false
1454         }
1455         return true
1456 }
1457
1458 func isAlpineLinux() bool {
1459         if runtime.GOOS != "linux" {
1460                 return false
1461         }
1462         fi, err := os.Lstat("/etc/alpine-release")
1463         return err == nil && fi.Mode().IsRegular()
1464 }
1465
1466 func (t *tester) runFlag(rx string) string {
1467         if t.compileOnly {
1468                 return "-run=^$"
1469         }
1470         return "-run=" + rx
1471 }
1472
1473 func (t *tester) raceTest(dt *distTest) error {
1474         t.addCmd(dt, "src", t.goTest(), "-race", t.runFlag("Output"), "runtime/race")
1475         t.addCmd(dt, "src", t.goTest(), "-race", t.runFlag("TestParse|TestEcho|TestStdinCloseRace|TestClosedPipeRace|TestTypeRace|TestFdRace|TestFdReadRace|TestFileCloseRace"), "flag", "net", "os", "os/exec", "encoding/gob")
1476         // We don't want the following line, because it
1477         // slows down all.bash (by 10 seconds on my laptop).
1478         // The race builder should catch any error here, but doesn't.
1479         // TODO(iant): Figure out how to catch this.
1480         // t.addCmd(dt, "src", t.goTest(),  "-race", "-run=TestParallelTest", "cmd/go")
1481         if t.cgoEnabled {
1482                 // Building misc/cgo/test takes a long time.
1483                 // There are already cgo-enabled packages being tested with the race detector.
1484                 // We shouldn't need to redo all of misc/cgo/test too.
1485                 // The race buildler will take care of this.
1486                 // cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), "-race")
1487                 // setEnv(cmd, "GOTRACEBACK", "2")
1488         }
1489         if t.extLink() {
1490                 // Test with external linking; see issue 9133.
1491                 t.addCmd(dt, "src", t.goTest(), "-race", "-ldflags=-linkmode=external", t.runFlag("TestParse|TestEcho|TestStdinCloseRace"), "flag", "os/exec")
1492         }
1493         return nil
1494 }
1495
1496 var runtest struct {
1497         sync.Once
1498         exe string
1499         err error
1500 }
1501
1502 func (t *tester) testDirTest(dt *distTest, shard, shards int) error {
1503         runtest.Do(func() {
1504                 f, err := os.CreateTemp("", "runtest-*.exe") // named exe for Windows, but harmless elsewhere
1505                 if err != nil {
1506                         runtest.err = err
1507                         return
1508                 }
1509                 f.Close()
1510
1511                 runtest.exe = f.Name()
1512                 xatexit(func() {
1513                         os.Remove(runtest.exe)
1514                 })
1515
1516                 cmd := t.dirCmd("test", "go", "build", "-o", runtest.exe, "run.go")
1517                 setEnv(cmd, "GOOS", gohostos)
1518                 setEnv(cmd, "GOARCH", gohostarch)
1519                 runtest.err = cmd.Run()
1520         })
1521         if runtest.err != nil {
1522                 return runtest.err
1523         }
1524         if t.compileOnly {
1525                 return nil
1526         }
1527         t.addCmd(dt, "test", runtest.exe,
1528                 fmt.Sprintf("--shard=%d", shard),
1529                 fmt.Sprintf("--shards=%d", shards),
1530         )
1531         return nil
1532 }
1533
1534 // cgoPackages is the standard packages that use cgo.
1535 var cgoPackages = []string{
1536         "net",
1537         "os/user",
1538 }
1539
1540 var funcBenchmark = []byte("\nfunc Benchmark")
1541
1542 // packageHasBenchmarks reports whether pkg has benchmarks.
1543 // On any error, it conservatively returns true.
1544 //
1545 // This exists just to eliminate work on the builders, since compiling
1546 // a test in race mode just to discover it has no benchmarks costs a
1547 // second or two per package, and this function returns false for
1548 // about 100 packages.
1549 func (t *tester) packageHasBenchmarks(pkg string) bool {
1550         pkgDir := filepath.Join(goroot, "src", pkg)
1551         d, err := os.Open(pkgDir)
1552         if err != nil {
1553                 return true // conservatively
1554         }
1555         defer d.Close()
1556         names, err := d.Readdirnames(-1)
1557         if err != nil {
1558                 return true // conservatively
1559         }
1560         for _, name := range names {
1561                 if !strings.HasSuffix(name, "_test.go") {
1562                         continue
1563                 }
1564                 slurp, err := os.ReadFile(filepath.Join(pkgDir, name))
1565                 if err != nil {
1566                         return true // conservatively
1567                 }
1568                 if bytes.Contains(slurp, funcBenchmark) {
1569                         return true
1570                 }
1571         }
1572         return false
1573 }
1574
1575 // makeGOROOTUnwritable makes all $GOROOT files & directories non-writable to
1576 // check that no tests accidentally write to $GOROOT.
1577 func (t *tester) makeGOROOTUnwritable() (undo func()) {
1578         dir := os.Getenv("GOROOT")
1579         if dir == "" {
1580                 panic("GOROOT not set")
1581         }
1582
1583         type pathMode struct {
1584                 path string
1585                 mode os.FileMode
1586         }
1587         var dirs []pathMode // in lexical order
1588
1589         undo = func() {
1590                 for i := range dirs {
1591                         os.Chmod(dirs[i].path, dirs[i].mode) // best effort
1592                 }
1593         }
1594
1595         gocache := os.Getenv("GOCACHE")
1596         if gocache == "" {
1597                 panic("GOCACHE not set")
1598         }
1599         gocacheSubdir, _ := filepath.Rel(dir, gocache)
1600
1601         // Note: Can't use WalkDir here, because this has to compile with Go 1.4.
1602         filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
1603                 if suffix := strings.TrimPrefix(path, dir+string(filepath.Separator)); suffix != "" {
1604                         if suffix == gocacheSubdir {
1605                                 // Leave GOCACHE writable: we may need to write test binaries into it.
1606                                 return filepath.SkipDir
1607                         }
1608                         if suffix == ".git" {
1609                                 // Leave Git metadata in whatever state it was in. It may contain a lot
1610                                 // of files, and it is highly unlikely that a test will try to modify
1611                                 // anything within that directory.
1612                                 return filepath.SkipDir
1613                         }
1614                 }
1615                 if err == nil {
1616                         mode := info.Mode()
1617                         if mode&0222 != 0 && (mode.IsDir() || mode.IsRegular()) {
1618                                 dirs = append(dirs, pathMode{path, mode})
1619                         }
1620                 }
1621                 return nil
1622         })
1623
1624         // Run over list backward to chmod children before parents.
1625         for i := len(dirs) - 1; i >= 0; i-- {
1626                 err := os.Chmod(dirs[i].path, dirs[i].mode&^0222)
1627                 if err != nil {
1628                         dirs = dirs[i:] // Only undo what we did so far.
1629                         undo()
1630                         fatalf("failed to make GOROOT read-only: %v", err)
1631                 }
1632         }
1633
1634         return undo
1635 }
1636
1637 // raceDetectorSupported is a copy of the function
1638 // internal/platform.RaceDetectorSupported, which can't be used here
1639 // because cmd/dist has to be buildable by Go 1.4.
1640 // The race detector only supports 48-bit VMA on arm64. But we don't have
1641 // a good solution to check VMA size(See https://golang.org/issue/29948)
1642 // raceDetectorSupported will always return true for arm64. But race
1643 // detector tests may abort on non 48-bit VMA configuration, the tests
1644 // will be marked as "skipped" in this case.
1645 func raceDetectorSupported(goos, goarch string) bool {
1646         switch goos {
1647         case "linux":
1648                 return goarch == "amd64" || goarch == "ppc64le" || goarch == "arm64" || goarch == "s390x"
1649         case "darwin":
1650                 return goarch == "amd64" || goarch == "arm64"
1651         case "freebsd", "netbsd", "openbsd", "windows":
1652                 return goarch == "amd64"
1653         default:
1654                 return false
1655         }
1656 }
1657
1658 // isUnsupportedVMASize reports whether the failure is caused by an unsupported
1659 // VMA for the race detector (for example, running the race detector on an
1660 // arm64 machine configured with 39-bit VMA)
1661 func isUnsupportedVMASize(w *work) bool {
1662         unsupportedVMA := []byte("unsupported VMA range")
1663         return w.dt.name == "race" && bytes.Contains(w.out, unsupportedVMA)
1664 }
1665
1666 // isEnvSet reports whether the environment variable evar is
1667 // set in the environment.
1668 func isEnvSet(evar string) bool {
1669         evarEq := evar + "="
1670         for _, e := range os.Environ() {
1671                 if strings.HasPrefix(e, evarEq) {
1672                         return true
1673                 }
1674         }
1675         return false
1676 }