src/cmd/dist/test.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "flag"
  10         "fmt"
  11         "log"
  12         "os"
  13         "os/exec"
  14         "path/filepath"
  15         "reflect"
  16         "regexp"
  17         "runtime"
  18         "strconv"
  19         "strings"
  20         "sync"
  21         "time"
  22 )
  23
  24 func cmdtest() {
  25         gogcflags = os.Getenv("GO_GCFLAGS")
  26         setNoOpt()
  27
  28         var t tester
  29
  30         var noRebuild bool
  31         flag.BoolVar(&t.listMode, "list", false, "list available tests")
  32         flag.BoolVar(&t.rebuild, "rebuild", false, "rebuild everything first")
  33         flag.BoolVar(&noRebuild, "no-rebuild", false, "overrides -rebuild (historical dreg)")
  34         flag.BoolVar(&t.keepGoing, "k", false, "keep going even when error occurred")
  35         flag.BoolVar(&t.race, "race", false, "run in race builder mode (different set of tests)")
  36         flag.BoolVar(&t.compileOnly, "compile-only", false, "compile tests, but don't run them. This is for some builders. Not all dist tests respect this flag, but most do.")
  37         flag.StringVar(&t.banner, "banner", "##### ", "banner prefix; blank means no section banners")
  38         flag.StringVar(&t.runRxStr, "run", os.Getenv("GOTESTONLY"),
  39                 "run only those tests matching the regular expression; empty means to run all. "+
  40                         "Special exception: if the string begins with '!', the match is inverted.")
  41         flag.BoolVar(&t.msan, "msan", false, "run in memory sanitizer builder mode")
  42         flag.BoolVar(&t.asan, "asan", false, "run in address sanitizer builder mode")
  43
  44         xflagparse(-1) // any number of args
  45         if noRebuild {
  46                 t.rebuild = false
  47         }
  48
  49         t.run()
  50 }
  51
  52 // tester executes cmdtest.
  53 type tester struct {
  54         race        bool
  55         msan        bool
  56         asan        bool
  57         listMode    bool
  58         rebuild     bool
  59         failed      bool
  60         keepGoing   bool
  61         compileOnly bool // just try to compile all tests, but no need to run
  62         runRxStr    string
  63         runRx       *regexp.Regexp
  64         runRxWant   bool     // want runRx to match (true) or not match (false)
  65         runNames    []string // tests to run, exclusive with runRx; empty means all
  66         banner      string   // prefix, or "" for none
  67         lastHeading string   // last dir heading printed
  68
  69         cgoEnabled bool
  70         partial    bool
  71         haveTime   bool // the 'time' binary is available
  72
  73         tests        []distTest
  74         timeoutScale int
  75
  76         worklist []*work
  77 }
  78
  79 type work struct {
  80         dt    *distTest
  81         cmd   *exec.Cmd
  82         start chan bool
  83         out   []byte
  84         err   error
  85         end   chan bool
  86 }
  87
  88 // A distTest is a test run by dist test.
  89 // Each test has a unique name and belongs to a group (heading)
  90 type distTest struct {
  91         name    string // unique test name; may be filtered with -run flag
  92         heading string // group section; this header is printed before the test is run.
  93         fn      func(*distTest) error
  94 }
  95
  96 func (t *tester) run() {
  97         timelog("start", "dist test")
  98
  99         os.Setenv("PATH", fmt.Sprintf("%s%c%s", gorootBin, os.PathListSeparator, os.Getenv("PATH")))
 100
 101         cmd := exec.Command(gorootBinGo, "env", "CGO_ENABLED")
 102         cmd.Stderr = new(bytes.Buffer)
 103         slurp, err := cmd.Output()
 104         if err != nil {
 105                 fatalf("Error running go env CGO_ENABLED: %v\n%s", err, cmd.Stderr)
 106         }
 107         t.cgoEnabled, _ = strconv.ParseBool(strings.TrimSpace(string(slurp)))
 108         if flag.NArg() > 0 && t.runRxStr != "" {
 109                 fatalf("the -run regular expression flag is mutually exclusive with test name arguments")
 110         }
 111
 112         t.runNames = flag.Args()
 113
 114         if t.hasBash() {
 115                 if _, err := exec.LookPath("time"); err == nil {
 116                         t.haveTime = true
 117                 }
 118         }
 119
 120         // Set GOTRACEBACK to system if the user didn't set a level explicitly.
 121         // Since we're running tests for Go, we want as much detail as possible
 122         // if something goes wrong.
 123         //
 124         // Set it before running any commands just in case something goes wrong.
 125         if ok := isEnvSet("GOTRACEBACK"); !ok {
 126                 if err := os.Setenv("GOTRACEBACK", "system"); err != nil {
 127                         if t.keepGoing {
 128                                 log.Printf("Failed to set GOTRACEBACK: %v", err)
 129                         } else {
 130                                 fatalf("Failed to set GOTRACEBACK: %v", err)
 131                         }
 132                 }
 133         }
 134
 135         if t.rebuild {
 136                 t.out("Building packages and commands.")
 137                 // Force rebuild the whole toolchain.
 138                 goInstall("go", append([]string{"-a"}, toolchain...)...)
 139         }
 140
 141         if !t.listMode {
 142                 if os.Getenv("GO_BUILDER_NAME") == "" {
 143                         // Complete rebuild bootstrap, even with -no-rebuild.
 144                         // If everything is up-to-date, this is a no-op.
 145                         // If everything is not up-to-date, the first checkNotStale
 146                         // during the test process will kill the tests, so we might
 147                         // as well install the world.
 148                         // Now that for example "go install cmd/compile" does not
 149                         // also install runtime (you need "go install -i cmd/compile"
 150                         // for that), it's easy for previous workflows like
 151                         // "rebuild the compiler and then run run.bash"
 152                         // to break if we don't automatically refresh things here.
 153                         // Rebuilding is a shortened bootstrap.
 154                         // See cmdbootstrap for a description of the overall process.
 155                         goInstall("go", toolchain...)
 156                         goInstall("go", toolchain...)
 157                         goInstall("go", "std", "cmd")
 158                 } else {
 159                         // The Go builder infrastructure should always begin running tests from a
 160                         // clean, non-stale state, so there is no need to rebuild the world.
 161                         // Instead, we can just check that it is not stale, which may be less
 162                         // expensive (and is also more likely to catch bugs in the builder
 163                         // implementation).
 164                         // The cache used by dist when building is different from that used when
 165                         // running dist test, so rebuild (but don't install) std and cmd to make
 166                         // sure packages without install targets are cached so they are not stale.
 167                         goCmd("go", "build", "std", "cmd") // make sure dependencies of targets are cached
 168                         checkNotStale("go", "std", "cmd")
 169                 }
 170         }
 171
 172         t.timeoutScale = 1
 173         switch goarch {
 174         case "arm":
 175                 t.timeoutScale = 2
 176         case "mips", "mipsle", "mips64", "mips64le":
 177                 t.timeoutScale = 4
 178         }
 179         if s := os.Getenv("GO_TEST_TIMEOUT_SCALE"); s != "" {
 180                 t.timeoutScale, err = strconv.Atoi(s)
 181                 if err != nil {
 182                         fatalf("failed to parse $GO_TEST_TIMEOUT_SCALE = %q as integer: %v", s, err)
 183                 }
 184         }
 185
 186         if t.runRxStr != "" {
 187                 if t.runRxStr[0] == '!' {
 188                         t.runRxWant = false
 189                         t.runRxStr = t.runRxStr[1:]
 190                 } else {
 191                         t.runRxWant = true
 192                 }
 193                 t.runRx = regexp.MustCompile(t.runRxStr)
 194         }
 195
 196         t.registerTests()
 197         if t.listMode {
 198                 for _, tt := range t.tests {
 199                         fmt.Println(tt.name)
 200                 }
 201                 return
 202         }
 203
 204         for _, name := range t.runNames {
 205                 if !t.isRegisteredTestName(name) {
 206                         fatalf("unknown test %q", name)
 207                 }
 208         }
 209
 210         // On a few builders, make GOROOT unwritable to catch tests writing to it.
 211         if strings.HasPrefix(os.Getenv("GO_BUILDER_NAME"), "linux-") {
 212                 if os.Getuid() == 0 {
 213                         // Don't bother making GOROOT unwritable:
 214                         // we're running as root, so permissions would have no effect.
 215                 } else {
 216                         xatexit(t.makeGOROOTUnwritable())
 217                 }
 218         }
 219
 220         if err := t.maybeLogMetadata(); err != nil {
 221                 t.failed = true
 222                 if t.keepGoing {
 223                         log.Printf("Failed logging metadata: %v", err)
 224                 } else {
 225                         fatalf("Failed logging metadata: %v", err)
 226                 }
 227         }
 228
 229         for _, dt := range t.tests {
 230                 if !t.shouldRunTest(dt.name) {
 231                         t.partial = true
 232                         continue
 233                 }
 234                 dt := dt // dt used in background after this iteration
 235                 if err := dt.fn(&dt); err != nil {
 236                         t.runPending(&dt) // in case that hasn't been done yet
 237                         t.failed = true
 238                         if t.keepGoing {
 239                                 log.Printf("Failed: %v", err)
 240                         } else {
 241                                 fatalf("Failed: %v", err)
 242                         }
 243                 }
 244         }
 245         t.runPending(nil)
 246         timelog("end", "dist test")
 247
 248         if t.failed {
 249                 fmt.Println("\nFAILED")
 250                 xexit(1)
 251         } else if incomplete[goos+"/"+goarch] {
 252                 // The test succeeded, but consider it as failed so we don't
 253                 // forget to remove the port from the incomplete map once the
 254                 // port is complete.
 255                 fmt.Println("\nFAILED (incomplete port)")
 256                 xexit(1)
 257         } else if t.partial {
 258                 fmt.Println("\nALL TESTS PASSED (some were excluded)")
 259         } else {
 260                 fmt.Println("\nALL TESTS PASSED")
 261         }
 262 }
 263
 264 func (t *tester) shouldRunTest(name string) bool {
 265         if t.runRx != nil {
 266                 return t.runRx.MatchString(name) == t.runRxWant
 267         }
 268         if len(t.runNames) == 0 {
 269                 return true
 270         }
 271         for _, runName := range t.runNames {
 272                 if runName == name {
 273                         return true
 274                 }
 275         }
 276         return false
 277 }
 278
 279 func (t *tester) maybeLogMetadata() error {
 280         if t.compileOnly {
 281                 // We need to run a subprocess to log metadata. Don't do that
 282                 // on compile-only runs.
 283                 return nil
 284         }
 285         t.out("Test execution environment.")
 286         // Helper binary to print system metadata (CPU model, etc). This is a
 287         // separate binary from dist so it need not build with the bootstrap
 288         // toolchain.
 289         //
 290         // TODO(prattmic): If we split dist bootstrap and dist test then this
 291         // could be simplified to directly use internal/sysinfo here.
 292         return t.dirCmd(filepath.Join(goroot, "src/cmd/internal/metadata"), "go", []string{"run", "main.go"}).Run()
 293 }
 294
 295 // short returns a -short flag value to use with 'go test'
 296 // or a test binary for tests intended to run in short mode.
 297 // It returns "true", unless the environment variable
 298 // GO_TEST_SHORT is set to a non-empty, false-ish string.
 299 //
 300 // This environment variable is meant to be an internal
 301 // detail between the Go build system and cmd/dist for
 302 // the purpose of longtest builders, and is not intended
 303 // for use by users. See golang.org/issue/12508.
 304 func short() string {
 305         if v := os.Getenv("GO_TEST_SHORT"); v != "" {
 306                 short, err := strconv.ParseBool(v)
 307                 if err != nil {
 308                         fatalf("invalid GO_TEST_SHORT %q: %v", v, err)
 309                 }
 310                 if !short {
 311                         return "false"
 312                 }
 313         }
 314         return "true"
 315 }
 316
 317 // goTest returns the beginning of the go test command line.
 318 // Callers should use goTest and then pass flags overriding these
 319 // defaults as later arguments in the command line.
 320 func (t *tester) goTest() []string {
 321         return []string{
 322                 "go", "test", "-short=" + short(), "-count=1", t.tags(), t.runFlag(""),
 323         }
 324 }
 325
 326 func (t *tester) tags() string {
 327         ios := t.iOS()
 328         switch {
 329         case ios && noOpt:
 330                 return "-tags=lldb,noopt"
 331         case ios:
 332                 return "-tags=lldb"
 333         case noOpt:
 334                 return "-tags=noopt"
 335         default:
 336                 return "-tags="
 337         }
 338 }
 339
 340 // timeoutDuration converts the provided number of seconds into a
 341 // time.Duration, scaled by the t.timeoutScale factor.
 342 func (t *tester) timeoutDuration(sec int) time.Duration {
 343         return time.Duration(sec) * time.Second * time.Duration(t.timeoutScale)
 344 }
 345
 346 // timeout returns the "-timeout=" string argument to "go test" given
 347 // the number of seconds of timeout. It scales it by the
 348 // t.timeoutScale factor.
 349 func (t *tester) timeout(sec int) string {
 350         return "-timeout=" + t.timeoutDuration(sec).String()
 351 }
 352
 353 // ranGoTest and stdMatches are state closed over by the stdlib
 354 // testing func in registerStdTest below. The tests are run
 355 // sequentially, so there's no need for locks.
 356 //
 357 // ranGoBench and benchMatches are the same, but are only used
 358 // in -race mode.
 359 var (
 360         ranGoTest  bool
 361         stdMatches []string
 362
 363         ranGoBench   bool
 364         benchMatches []string
 365 )
 366
 367 func (t *tester) registerStdTest(pkg string) {
 368         heading := "Testing packages."
 369         testPrefix := "go_test:"
 370         gcflags := gogcflags
 371
 372         testName := testPrefix + pkg
 373         if t.runRx == nil || t.runRx.MatchString(testName) == t.runRxWant {
 374                 stdMatches = append(stdMatches, pkg)
 375         }
 376
 377         t.tests = append(t.tests, distTest{
 378                 name:    testName,
 379                 heading: heading,
 380                 fn: func(dt *distTest) error {
 381                         if ranGoTest {
 382                                 return nil
 383                         }
 384                         t.runPending(dt)
 385                         timelog("start", dt.name)
 386                         defer timelog("end", dt.name)
 387                         ranGoTest = true
 388
 389                         timeoutSec := 180
 390                         for _, pkg := range stdMatches {
 391                                 if pkg == "cmd/go" {
 392                                         timeoutSec *= 3
 393                                         break
 394                                 }
 395                         }
 396                         args := []string{
 397                                 "test",
 398                                 "-short=" + short(),
 399                                 t.tags(),
 400                                 t.timeout(timeoutSec),
 401                         }
 402                         if gcflags != "" {
 403                                 args = append(args, "-gcflags=all="+gcflags)
 404                         }
 405                         if t.race {
 406                                 args = append(args, "-race")
 407                         }
 408                         if t.msan {
 409                                 args = append(args, "-msan")
 410                         }
 411                         if t.asan {
 412                                 args = append(args, "-asan")
 413                         }
 414                         if t.compileOnly {
 415                                 args = append(args, "-run=^$")
 416                         }
 417                         args = append(args, stdMatches...)
 418                         cmd := exec.Command(gorootBinGo, args...)
 419                         cmd.Stdout = os.Stdout
 420                         cmd.Stderr = os.Stderr
 421                         return cmd.Run()
 422                 },
 423         })
 424 }
 425
 426 func (t *tester) registerRaceBenchTest(pkg string) {
 427         testName := "go_test_bench:" + pkg
 428         if t.runRx == nil || t.runRx.MatchString(testName) == t.runRxWant {
 429                 benchMatches = append(benchMatches, pkg)
 430         }
 431         t.tests = append(t.tests, distTest{
 432                 name:    testName,
 433                 heading: "Running benchmarks briefly.",
 434                 fn: func(dt *distTest) error {
 435                         if ranGoBench {
 436                                 return nil
 437                         }
 438                         t.runPending(dt)
 439                         timelog("start", dt.name)
 440                         defer timelog("end", dt.name)
 441                         ranGoBench = true
 442                         args := []string{
 443                                 "test",
 444                                 "-short=" + short(),
 445                                 "-race",
 446                                 t.timeout(1200), // longer timeout for race with benchmarks
 447                                 "-run=^$",       // nothing. only benchmarks.
 448                                 "-benchtime=.1s",
 449                                 "-cpu=4",
 450                         }
 451                         if !t.compileOnly {
 452                                 args = append(args, "-bench=.*")
 453                         }
 454                         args = append(args, benchMatches...)
 455                         cmd := exec.Command(gorootBinGo, args...)
 456                         cmd.Stdout = os.Stdout
 457                         cmd.Stderr = os.Stderr
 458                         return cmd.Run()
 459                 },
 460         })
 461 }
 462
 463 // stdOutErrAreTerminals is defined in test_linux.go, to report
 464 // whether stdout & stderr are terminals.
 465 var stdOutErrAreTerminals func() bool
 466
 467 func (t *tester) registerTests() {
 468         // Fast path to avoid the ~1 second of `go list std cmd` when
 469         // the caller lists specific tests to run. (as the continuous
 470         // build coordinator does).
 471         if len(t.runNames) > 0 {
 472                 for _, name := range t.runNames {
 473                         if strings.HasPrefix(name, "go_test:") {
 474                                 t.registerStdTest(strings.TrimPrefix(name, "go_test:"))
 475                         }
 476                         if strings.HasPrefix(name, "go_test_bench:") {
 477                                 t.registerRaceBenchTest(strings.TrimPrefix(name, "go_test_bench:"))
 478                         }
 479                 }
 480         } else {
 481                 // Use a format string to only list packages and commands that have tests.
 482                 const format = "{{if (or .TestGoFiles .XTestGoFiles)}}{{.ImportPath}}{{end}}"
 483                 cmd := exec.Command(gorootBinGo, "list", "-f", format)
 484                 if t.race {
 485                         cmd.Args = append(cmd.Args, "-tags=race")
 486                 }
 487                 cmd.Args = append(cmd.Args, "std", "cmd")
 488                 cmd.Stderr = new(bytes.Buffer)
 489                 all, err := cmd.Output()
 490                 if err != nil {
 491                         fatalf("Error running go list std cmd: %v:\n%s", err, cmd.Stderr)
 492                 }
 493                 pkgs := strings.Fields(string(all))
 494                 for _, pkg := range pkgs {
 495                         t.registerStdTest(pkg)
 496                 }
 497                 if t.race {
 498                         for _, pkg := range pkgs {
 499                                 if t.packageHasBenchmarks(pkg) {
 500                                         t.registerRaceBenchTest(pkg)
 501                                 }
 502                         }
 503                 }
 504         }
 505
 506         // Test the os/user package in the pure-Go mode too.
 507         if !t.compileOnly {
 508                 t.tests = append(t.tests, distTest{
 509                         name:    "osusergo",
 510                         heading: "os/user with tag osusergo",
 511                         fn: func(dt *distTest) error {
 512                                 t.addCmd(dt, "src", t.goTest(), t.timeout(300), "-tags=osusergo", "os/user")
 513                                 return nil
 514                         },
 515                 })
 516         }
 517
 518         // Test ios/amd64 for the iOS simulator.
 519         if goos == "darwin" && goarch == "amd64" && t.cgoEnabled {
 520                 t.tests = append(t.tests, distTest{
 521                         name:    "amd64ios",
 522                         heading: "GOOS=ios on darwin/amd64",
 523                         fn: func(dt *distTest) error {
 524                                 cmd := t.addCmd(dt, "src", t.goTest(), t.timeout(300), "-run=SystemRoots", "crypto/x509")
 525                                 setEnv(cmd, "GOOS", "ios")
 526                                 setEnv(cmd, "CGO_ENABLED", "1")
 527                                 return nil
 528                         },
 529                 })
 530         }
 531
 532         if t.race {
 533                 return
 534         }
 535
 536         // Runtime CPU tests.
 537         if !t.compileOnly && goos != "js" { // js can't handle -cpu != 1
 538                 testName := "runtime:cpu124"
 539                 t.tests = append(t.tests, distTest{
 540                         name:    testName,
 541                         heading: "GOMAXPROCS=2 runtime -cpu=1,2,4 -quick",
 542                         fn: func(dt *distTest) error {
 543                                 cmd := t.addCmd(dt, "src", t.goTest(), "-short=true", t.timeout(300), "runtime", "-cpu=1,2,4", "-quick")
 544                                 // We set GOMAXPROCS=2 in addition to -cpu=1,2,4 in order to test runtime bootstrap code,
 545                                 // creation of first goroutines and first garbage collections in the parallel setting.
 546                                 setEnv(cmd, "GOMAXPROCS", "2")
 547                                 return nil
 548                         },
 549                 })
 550         }
 551
 552         // morestack tests. We only run these on in long-test mode
 553         // (with GO_TEST_SHORT=false) because the runtime test is
 554         // already quite long and mayMoreStackMove makes it about
 555         // twice as slow.
 556         if !t.compileOnly && short() == "false" {
 557                 // hooks is the set of maymorestack hooks to test with.
 558                 hooks := []string{"mayMoreStackPreempt", "mayMoreStackMove"}
 559                 // pkgs is the set of test packages to run.
 560                 pkgs := []string{"runtime", "reflect", "sync"}
 561                 // hookPkgs is the set of package patterns to apply
 562                 // the maymorestack hook to.
 563                 hookPkgs := []string{"runtime/...", "reflect", "sync"}
 564                 // unhookPkgs is the set of package patterns to
 565                 // exclude from hookPkgs.
 566                 unhookPkgs := []string{"runtime/testdata/..."}
 567                 for _, hook := range hooks {
 568                         // Construct the build flags to use the
 569                         // maymorestack hook in the compiler and
 570                         // assembler. We pass this via the GOFLAGS
 571                         // environment variable so that it applies to
 572                         // both the test itself and to binaries built
 573                         // by the test.
 574                         goFlagsList := []string{}
 575                         for _, flag := range []string{"-gcflags", "-asmflags"} {
 576                                 for _, hookPkg := range hookPkgs {
 577                                         goFlagsList = append(goFlagsList, flag+"="+hookPkg+"=-d=maymorestack=runtime."+hook)
 578                                 }
 579                                 for _, unhookPkg := range unhookPkgs {
 580                                         goFlagsList = append(goFlagsList, flag+"="+unhookPkg+"=")
 581                                 }
 582                         }
 583                         goFlags := strings.Join(goFlagsList, " ")
 584
 585                         for _, pkg := range pkgs {
 586                                 pkg := pkg
 587                                 testName := hook + ":" + pkg
 588                                 t.tests = append(t.tests, distTest{
 589                                         name:    testName,
 590                                         heading: "maymorestack=" + hook,
 591                                         fn: func(dt *distTest) error {
 592                                                 cmd := t.addCmd(dt, "src", t.goTest(), t.timeout(600), pkg, "-short")
 593                                                 setEnv(cmd, "GOFLAGS", goFlags)
 594                                                 return nil
 595                                         },
 596                                 })
 597                         }
 598                 }
 599         }
 600
 601         // This test needs its stdout/stderr to be terminals, so we don't run it from cmd/go's tests.
 602         // See issue 18153.
 603         if goos == "linux" {
 604                 t.tests = append(t.tests, distTest{
 605                         name:    "cmd_go_test_terminal",
 606                         heading: "cmd/go terminal test",
 607                         fn: func(dt *distTest) error {
 608                                 t.runPending(dt)
 609                                 timelog("start", dt.name)
 610                                 defer timelog("end", dt.name)
 611                                 if !stdOutErrAreTerminals() {
 612                                         fmt.Println("skipping terminal test; stdout/stderr not terminals")
 613                                         return nil
 614                                 }
 615                                 cmd := exec.Command(gorootBinGo, "test")
 616                                 setDir(cmd, filepath.Join(os.Getenv("GOROOT"), "src/cmd/go/testdata/testterminal18153"))
 617                                 cmd.Stdout = os.Stdout
 618                                 cmd.Stderr = os.Stderr
 619                                 return cmd.Run()
 620                         },
 621                 })
 622         }
 623
 624         // On the builders only, test that a moved GOROOT still works.
 625         // Fails on iOS because CC_FOR_TARGET refers to clangwrap.sh
 626         // in the unmoved GOROOT.
 627         // Fails on Android and js/wasm with an exec format error.
 628         // Fails on plan9 with "cannot find GOROOT" (issue #21016).
 629         if os.Getenv("GO_BUILDER_NAME") != "" && goos != "android" && !t.iOS() && goos != "plan9" && goos != "js" {
 630                 t.tests = append(t.tests, distTest{
 631                         name:    "moved_goroot",
 632                         heading: "moved GOROOT",
 633                         fn: func(dt *distTest) error {
 634                                 t.runPending(dt)
 635                                 timelog("start", dt.name)
 636                                 defer timelog("end", dt.name)
 637                                 moved := goroot + "-moved"
 638                                 if err := os.Rename(goroot, moved); err != nil {
 639                                         if goos == "windows" {
 640                                                 // Fails on Windows (with "Access is denied") if a process
 641                                                 // or binary is in this directory. For instance, using all.bat
 642                                                 // when run from c:\workdir\go\src fails here
 643                                                 // if GO_BUILDER_NAME is set. Our builders invoke tests
 644                                                 // a different way which happens to work when sharding
 645                                                 // tests, but we should be tolerant of the non-sharded
 646                                                 // all.bat case.
 647                                                 log.Printf("skipping test on Windows")
 648                                                 return nil
 649                                         }
 650                                         return err
 651                                 }
 652
 653                                 // Run `go test fmt` in the moved GOROOT, without explicitly setting
 654                                 // GOROOT in the environment. The 'go' command should find itself.
 655                                 cmd := exec.Command(filepath.Join(moved, "bin", "go"), "test", "fmt")
 656                                 cmd.Stdout = os.Stdout
 657                                 cmd.Stderr = os.Stderr
 658                                 unsetEnv(cmd, "GOROOT")
 659                                 unsetEnv(cmd, "GOCACHE") // TODO(bcmills): ...why‽
 660                                 err := cmd.Run()
 661
 662                                 if rerr := os.Rename(moved, goroot); rerr != nil {
 663                                         fatalf("failed to restore GOROOT: %v", rerr)
 664                                 }
 665                                 return err
 666                         },
 667                 })
 668         }
 669
 670         // Test that internal linking of standard packages does not
 671         // require libgcc. This ensures that we can install a Go
 672         // release on a system that does not have a C compiler
 673         // installed and still build Go programs (that don't use cgo).
 674         for _, pkg := range cgoPackages {
 675                 if !t.internalLink() {
 676                         break
 677                 }
 678
 679                 // ARM libgcc may be Thumb, which internal linking does not support.
 680                 if goarch == "arm" {
 681                         break
 682                 }
 683
 684                 pkg := pkg
 685                 var run string
 686                 if pkg == "net" {
 687                         run = "TestTCPStress"
 688                 }
 689                 t.tests = append(t.tests, distTest{
 690                         name:    "nolibgcc:" + pkg,
 691                         heading: "Testing without libgcc.",
 692                         fn: func(dt *distTest) error {
 693                                 // What matters is that the tests build and start up.
 694                                 // Skip expensive tests, especially x509 TestSystemRoots.
 695                                 t.addCmd(dt, "src", t.goTest(), "-ldflags=-linkmode=internal -libgcc=none", "-run=^Test[^CS]", pkg, t.runFlag(run))
 696                                 return nil
 697                         },
 698                 })
 699         }
 700
 701         // Stub out following test on alpine until 54354 resolved.
 702         builderName := os.Getenv("GO_BUILDER_NAME")
 703         disablePIE := strings.HasSuffix(builderName, "-alpine")
 704
 705         // Test internal linking of PIE binaries where it is supported.
 706         if t.internalLinkPIE() && !disablePIE {
 707                 t.tests = append(t.tests, distTest{
 708                         name:    "pie_internal",
 709                         heading: "internal linking of -buildmode=pie",
 710                         fn: func(dt *distTest) error {
 711                                 cmd := t.addCmd(dt, "src", t.goTest(), "reflect", "-buildmode=pie", "-ldflags=-linkmode=internal", t.timeout(60))
 712                                 setEnv(cmd, "CGO_ENABLED", "0")
 713                                 return nil
 714                         },
 715                 })
 716                 // Also test a cgo package.
 717                 if t.cgoEnabled && t.internalLink() && !disablePIE {
 718                         t.tests = append(t.tests, distTest{
 719                                 name:    "pie_internal_cgo",
 720                                 heading: "internal linking of -buildmode=pie",
 721                                 fn: func(dt *distTest) error {
 722                                         t.addCmd(dt, "src", t.goTest(), "os/user", "-buildmode=pie", "-ldflags=-linkmode=internal", t.timeout(60))
 723                                         return nil
 724                                 },
 725                         })
 726                 }
 727         }
 728
 729         // sync tests
 730         if goos != "js" { // js doesn't support -cpu=10
 731                 t.tests = append(t.tests, distTest{
 732                         name:    "sync_cpu",
 733                         heading: "sync -cpu=10",
 734                         fn: func(dt *distTest) error {
 735                                 t.addCmd(dt, "src", t.goTest(), "sync", t.timeout(120), "-cpu=10", t.runFlag(""))
 736                                 return nil
 737                         },
 738                 })
 739         }
 740
 741         if t.raceDetectorSupported() {
 742                 t.tests = append(t.tests, distTest{
 743                         name:    "race",
 744                         heading: "Testing race detector",
 745                         fn:      t.raceTest,
 746                 })
 747         }
 748
 749         if t.cgoEnabled && !t.iOS() {
 750                 // Disabled on iOS. golang.org/issue/15919
 751                 t.registerHostTest("cgo_stdio", "../misc/cgo/stdio", "misc/cgo/stdio", ".")
 752                 t.registerHostTest("cgo_life", "../misc/cgo/life", "misc/cgo/life", ".")
 753                 if goos != "android" {
 754                         t.registerHostTest("cgo_fortran", "../misc/cgo/fortran", "misc/cgo/fortran", ".")
 755                 }
 756                 if t.hasSwig() && goos != "android" {
 757                         t.tests = append(t.tests, distTest{
 758                                 name:    "swig_stdio",
 759                                 heading: "../misc/swig/stdio",
 760                                 fn: func(dt *distTest) error {
 761                                         t.addCmd(dt, "misc/swig/stdio", t.goTest(), ".")
 762                                         return nil
 763                                 },
 764                         })
 765                         if t.hasCxx() {
 766                                 t.tests = append(t.tests,
 767                                         distTest{
 768                                                 name:    "swig_callback",
 769                                                 heading: "../misc/swig/callback",
 770                                                 fn: func(dt *distTest) error {
 771                                                         t.addCmd(dt, "misc/swig/callback", t.goTest(), ".")
 772                                                         return nil
 773                                                 },
 774                                         },
 775                                         distTest{
 776                                                 name:    "swig_callback_lto",
 777                                                 heading: "../misc/swig/callback",
 778                                                 fn: func(dt *distTest) error {
 779                                                         cmd := t.addCmd(dt, "misc/swig/callback", t.goTest(), ".")
 780                                                         setEnv(cmd, "CGO_CFLAGS", "-flto -Wno-lto-type-mismatch -Wno-unknown-warning-option")
 781                                                         setEnv(cmd, "CGO_CXXFLAGS", "-flto -Wno-lto-type-mismatch -Wno-unknown-warning-option")
 782                                                         setEnv(cmd, "CGO_LDFLAGS", "-flto -Wno-lto-type-mismatch -Wno-unknown-warning-option")
 783                                                         return nil
 784                                                 },
 785                                         },
 786                                 )
 787                         }
 788                 }
 789         }
 790         if t.cgoEnabled {
 791                 t.tests = append(t.tests, distTest{
 792                         name:    "cgo_test",
 793                         heading: "../misc/cgo/test",
 794                         fn:      t.cgoTest,
 795                 })
 796         }
 797
 798         // Don't run these tests with $GO_GCFLAGS because most of them
 799         // assume that they can run "go install" with no -gcflags and not
 800         // recompile the entire standard library. If make.bash ran with
 801         // special -gcflags, that's not true.
 802         if t.cgoEnabled && gogcflags == "" {
 803                 t.registerHostTest("testgodefs", "../misc/cgo/testgodefs", "misc/cgo/testgodefs", ".")
 804
 805                 t.registerTest("testso", "../misc/cgo/testso", t.goTest(), t.timeout(600), ".")
 806                 t.registerTest("testsovar", "../misc/cgo/testsovar", t.goTest(), t.timeout(600), ".")
 807                 if t.supportedBuildmode("c-archive") {
 808                         t.registerHostTest("testcarchive", "../misc/cgo/testcarchive", "misc/cgo/testcarchive", ".")
 809                 }
 810                 if t.supportedBuildmode("c-shared") {
 811                         t.registerHostTest("testcshared", "../misc/cgo/testcshared", "misc/cgo/testcshared", ".")
 812                 }
 813                 if t.supportedBuildmode("shared") {
 814                         t.registerTest("testshared", "../misc/cgo/testshared", t.goTest(), t.timeout(600), ".")
 815                 }
 816                 if t.supportedBuildmode("plugin") {
 817                         t.registerTest("testplugin", "../misc/cgo/testplugin", t.goTest(), t.timeout(600), ".")
 818                 }
 819                 if goos == "linux" || (goos == "freebsd" && goarch == "amd64") {
 820                         // because Pdeathsig of syscall.SysProcAttr struct used in misc/cgo/testsanitizers is only
 821                         // supported on Linux and FreeBSD.
 822                         t.registerHostTest("testsanitizers", "../misc/cgo/testsanitizers", "misc/cgo/testsanitizers", ".")
 823                 }
 824                 if t.hasBash() && goos != "android" && !t.iOS() && gohostos != "windows" {
 825                         t.registerHostTest("cgo_errors", "../misc/cgo/errors", "misc/cgo/errors", ".")
 826                 }
 827         }
 828
 829         if goos != "android" && !t.iOS() {
 830                 // There are no tests in this directory, only benchmarks.
 831                 // Check that the test binary builds but don't bother running it.
 832                 // (It has init-time work to set up for the benchmarks that is not worth doing unnecessarily.)
 833                 t.registerTest("bench_go1", "../test/bench/go1", t.goTest(), "-c", "-o="+os.DevNull)
 834         }
 835         if goos != "android" && !t.iOS() {
 836                 // Only start multiple test dir shards on builders,
 837                 // where they get distributed to multiple machines.
 838                 // See issues 20141 and 31834.
 839                 nShards := 1
 840                 if os.Getenv("GO_BUILDER_NAME") != "" {
 841                         nShards = 10
 842                 }
 843                 if n, err := strconv.Atoi(os.Getenv("GO_TEST_SHARDS")); err == nil {
 844                         nShards = n
 845                 }
 846                 for shard := 0; shard < nShards; shard++ {
 847                         shard := shard
 848                         t.tests = append(t.tests, distTest{
 849                                 name:    fmt.Sprintf("test:%d_%d", shard, nShards),
 850                                 heading: "../test",
 851                                 fn:      func(dt *distTest) error { return t.testDirTest(dt, shard, nShards) },
 852                         })
 853                 }
 854         }
 855         // Only run the API check on fast development platforms.
 856         // Every platform checks the API on every GOOS/GOARCH/CGO_ENABLED combination anyway,
 857         // so we really only need to run this check once anywhere to get adequate coverage.
 858         // To help developers avoid trybot-only failures, we try to run on typical developer machines
 859         // which is darwin/linux/windows and amd64/arm64.
 860         if (goos == "darwin" || goos == "linux" || goos == "windows") && (goarch == "amd64" || goarch == "arm64") {
 861                 t.tests = append(t.tests, distTest{
 862                         name:    "api",
 863                         heading: "API check",
 864                         fn: func(dt *distTest) error {
 865                                 if t.compileOnly {
 866                                         t.addCmd(dt, "src", "go", "build", "-o", os.DevNull, filepath.Join(goroot, "src/cmd/api/run.go"))
 867                                         return nil
 868                                 }
 869                                 t.addCmd(dt, "src", "go", "run", filepath.Join(goroot, "src/cmd/api/run.go"))
 870                                 return nil
 871                         },
 872                 })
 873         }
 874
 875         // Ensure that the toolchain can bootstrap itself.
 876         // This test adds another ~45s to all.bash if run sequentially, so run it only on the builders.
 877         if os.Getenv("GO_BUILDER_NAME") != "" && goos != "android" && !t.iOS() {
 878                 t.registerHostTest("reboot", "../misc/reboot", "misc/reboot", ".")
 879         }
 880 }
 881
 882 // isRegisteredTestName reports whether a test named testName has already
 883 // been registered.
 884 func (t *tester) isRegisteredTestName(testName string) bool {
 885         for _, tt := range t.tests {
 886                 if tt.name == testName {
 887                         return true
 888                 }
 889         }
 890         return false
 891 }
 892
 893 func (t *tester) registerTest1(seq bool, name, dirBanner string, cmdline ...interface{}) {
 894         bin, args := flattenCmdline(cmdline)
 895         if bin == "time" && !t.haveTime {
 896                 bin, args = args[0], args[1:]
 897         }
 898         if t.isRegisteredTestName(name) {
 899                 panic("duplicate registered test name " + name)
 900         }
 901         t.tests = append(t.tests, distTest{
 902                 name:    name,
 903                 heading: dirBanner,
 904                 fn: func(dt *distTest) error {
 905                         if seq {
 906                                 t.runPending(dt)
 907                                 timelog("start", name)
 908                                 defer timelog("end", name)
 909                                 return t.dirCmd(filepath.Join(goroot, "src", dirBanner), bin, args).Run()
 910                         }
 911                         t.addCmd(dt, filepath.Join(goroot, "src", dirBanner), bin, args)
 912                         return nil
 913                 },
 914         })
 915 }
 916
 917 func (t *tester) registerTest(name, dirBanner string, cmdline ...interface{}) {
 918         t.registerTest1(false, name, dirBanner, cmdline...)
 919 }
 920
 921 func (t *tester) registerSeqTest(name, dirBanner string, cmdline ...interface{}) {
 922         t.registerTest1(true, name, dirBanner, cmdline...)
 923 }
 924
 925 func (t *tester) bgDirCmd(dir, bin string, args ...string) *exec.Cmd {
 926         cmd := exec.Command(bin, args...)
 927         if filepath.IsAbs(dir) {
 928                 setDir(cmd, dir)
 929         } else {
 930                 setDir(cmd, filepath.Join(goroot, dir))
 931         }
 932         return cmd
 933 }
 934
 935 func (t *tester) dirCmd(dir string, cmdline ...interface{}) *exec.Cmd {
 936         bin, args := flattenCmdline(cmdline)
 937         cmd := t.bgDirCmd(dir, bin, args...)
 938         cmd.Stdout = os.Stdout
 939         cmd.Stderr = os.Stderr
 940         if vflag > 1 {
 941                 errprintf("%s\n", strings.Join(cmd.Args, " "))
 942         }
 943         return cmd
 944 }
 945
 946 // flattenCmdline flattens a mixture of string and []string as single list
 947 // and then interprets it as a command line: first element is binary, then args.
 948 func flattenCmdline(cmdline []interface{}) (bin string, args []string) {
 949         var list []string
 950         for _, x := range cmdline {
 951                 switch x := x.(type) {
 952                 case string:
 953                         list = append(list, x)
 954                 case []string:
 955                         list = append(list, x...)
 956                 default:
 957                         panic("invalid addCmd argument type: " + reflect.TypeOf(x).String())
 958                 }
 959         }
 960
 961         // The go command is too picky about duplicated flags.
 962         // Drop all but the last of the allowed duplicated flags.
 963         drop := make([]bool, len(list))
 964         have := map[string]int{}
 965         for i := 1; i < len(list); i++ {
 966                 j := strings.Index(list[i], "=")
 967                 if j < 0 {
 968                         continue
 969                 }
 970                 flag := list[i][:j]
 971                 switch flag {
 972                 case "-run", "-tags":
 973                         if have[flag] != 0 {
 974                                 drop[have[flag]] = true
 975                         }
 976                         have[flag] = i
 977                 }
 978         }
 979         out := list[:0]
 980         for i, x := range list {
 981                 if !drop[i] {
 982                         out = append(out, x)
 983                 }
 984         }
 985         list = out
 986
 987         bin = list[0]
 988         if bin == "go" {
 989                 bin = gorootBinGo
 990         }
 991         return bin, list[1:]
 992 }
 993
 994 func (t *tester) addCmd(dt *distTest, dir string, cmdline ...interface{}) *exec.Cmd {
 995         bin, args := flattenCmdline(cmdline)
 996         w := &work{
 997                 dt:  dt,
 998                 cmd: t.bgDirCmd(dir, bin, args...),
 999         }
1000         t.worklist = append(t.worklist, w)
1001         return w.cmd
1002 }
1003
1004 func (t *tester) iOS() bool {
1005         return goos == "ios"
1006 }
1007
1008 func (t *tester) out(v string) {
1009         if t.banner == "" {
1010                 return
1011         }
1012         fmt.Println("\n" + t.banner + v)
1013 }
1014
1015 func (t *tester) extLink() bool {
1016         pair := gohostos + "-" + goarch
1017         switch pair {
1018         case "aix-ppc64",
1019                 "android-arm", "android-arm64",
1020                 "darwin-amd64", "darwin-arm64",
1021                 "dragonfly-amd64",
1022                 "freebsd-386", "freebsd-amd64", "freebsd-arm", "freebsd-riscv64",
1023                 "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-loong64", "linux-ppc64le", "linux-mips64", "linux-mips64le", "linux-mips", "linux-mipsle", "linux-riscv64", "linux-s390x",
1024                 "netbsd-386", "netbsd-amd64",
1025                 "openbsd-386", "openbsd-amd64",
1026                 "windows-386", "windows-amd64":
1027                 return true
1028         }
1029         return false
1030 }
1031
1032 func (t *tester) internalLink() bool {
1033         if gohostos == "dragonfly" {
1034                 // linkmode=internal fails on dragonfly since errno is a TLS relocation.
1035                 return false
1036         }
1037         if goos == "android" {
1038                 return false
1039         }
1040         if goos == "ios" {
1041                 return false
1042         }
1043         if goos == "windows" && goarch == "arm64" {
1044                 return false
1045         }
1046         // Internally linking cgo is incomplete on some architectures.
1047         // https://golang.org/issue/10373
1048         // https://golang.org/issue/14449
1049         if goarch == "loong64" || goarch == "mips64" || goarch == "mips64le" || goarch == "mips" || goarch == "mipsle" || goarch == "riscv64" {
1050                 return false
1051         }
1052         if goos == "aix" {
1053                 // linkmode=internal isn't supported.
1054                 return false
1055         }
1056         return true
1057 }
1058
1059 func (t *tester) internalLinkPIE() bool {
1060         switch goos + "-" + goarch {
1061         case "darwin-amd64", "darwin-arm64",
1062                 "linux-amd64", "linux-arm64", "linux-ppc64le",
1063                 "android-arm64",
1064                 "windows-amd64", "windows-386", "windows-arm":
1065                 return true
1066         }
1067         return false
1068 }
1069
1070 func (t *tester) supportedBuildmode(mode string) bool {
1071         pair := goos + "-" + goarch
1072         switch mode {
1073         case "c-archive":
1074                 if !t.extLink() {
1075                         return false
1076                 }
1077                 switch pair {
1078                 case "aix-ppc64",
1079                         "darwin-amd64", "darwin-arm64", "ios-arm64",
1080                         "linux-amd64", "linux-386", "linux-ppc64le", "linux-riscv64", "linux-s390x",
1081                         "freebsd-amd64",
1082                         "windows-amd64", "windows-386":
1083                         return true
1084                 }
1085                 return false
1086         case "c-shared":
1087                 switch pair {
1088                 case "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-riscv64", "linux-s390x",
1089                         "darwin-amd64", "darwin-arm64",
1090                         "freebsd-amd64",
1091                         "android-arm", "android-arm64", "android-386",
1092                         "windows-amd64", "windows-386", "windows-arm64":
1093                         return true
1094                 }
1095                 return false
1096         case "shared":
1097                 switch pair {
1098                 case "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-s390x":
1099                         return true
1100                 }
1101                 return false
1102         case "plugin":
1103                 switch pair {
1104                 case "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-s390x", "linux-ppc64le":
1105                         return true
1106                 case "darwin-amd64", "darwin-arm64":
1107                         return true
1108                 case "freebsd-amd64":
1109                         return true
1110                 }
1111                 return false
1112         case "pie":
1113                 switch pair {
1114                 case "aix/ppc64",
1115                         "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-riscv64", "linux-s390x",
1116                         "android-amd64", "android-arm", "android-arm64", "android-386":
1117                         return true
1118                 case "darwin-amd64", "darwin-arm64":
1119                         return true
1120                 case "windows-amd64", "windows-386", "windows-arm":
1121                         return true
1122                 }
1123                 return false
1124
1125         default:
1126                 fatalf("internal error: unknown buildmode %s", mode)
1127                 return false
1128         }
1129 }
1130
1131 func (t *tester) registerHostTest(name, heading, dir, pkg string) {
1132         t.tests = append(t.tests, distTest{
1133                 name:    name,
1134                 heading: heading,
1135                 fn: func(dt *distTest) error {
1136                         t.runPending(dt)
1137                         timelog("start", name)
1138                         defer timelog("end", name)
1139                         return t.runHostTest(dir, pkg)
1140                 },
1141         })
1142 }
1143
1144 func (t *tester) runHostTest(dir, pkg string) error {
1145         out, err := exec.Command(gorootBinGo, "env", "GOEXE", "GOTMPDIR").Output()
1146         if err != nil {
1147                 return err
1148         }
1149
1150         parts := strings.Split(string(out), "\n")
1151         if len(parts) < 2 {
1152                 return fmt.Errorf("'go env GOEXE GOTMPDIR' output contains <2 lines")
1153         }
1154         GOEXE := strings.TrimSpace(parts[0])
1155         GOTMPDIR := strings.TrimSpace(parts[1])
1156
1157         f, err := os.CreateTemp(GOTMPDIR, "test.test-*"+GOEXE)
1158         if err != nil {
1159                 return err
1160         }
1161         f.Close()
1162         defer os.Remove(f.Name())
1163
1164         cmd := t.dirCmd(dir, t.goTest(), "-c", "-o", f.Name(), pkg)
1165         setEnv(cmd, "GOARCH", gohostarch)
1166         setEnv(cmd, "GOOS", gohostos)
1167         if err := cmd.Run(); err != nil {
1168                 return err
1169         }
1170         return t.dirCmd(dir, f.Name(), "-test.short="+short(), "-test.timeout="+t.timeoutDuration(300).String()).Run()
1171 }
1172
1173 func (t *tester) cgoTest(dt *distTest) error {
1174         cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), ".")
1175         setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=auto")
1176
1177         // Stub out various buildmode=pie tests  on alpine until 54354 resolved.
1178         builderName := os.Getenv("GO_BUILDER_NAME")
1179         disablePIE := strings.HasSuffix(builderName, "-alpine")
1180
1181         if t.internalLink() {
1182                 cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=internal", ".")
1183                 setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=internal")
1184         }
1185
1186         pair := gohostos + "-" + goarch
1187         switch pair {
1188         case "darwin-amd64", "darwin-arm64",
1189                 "windows-386", "windows-amd64":
1190                 // test linkmode=external, but __thread not supported, so skip testtls.
1191                 if !t.extLink() {
1192                         break
1193                 }
1194                 cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), ".")
1195                 setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=external")
1196
1197                 t.addCmd(dt, "misc/cgo/test", t.goTest(), "-ldflags", "-linkmode=external -s", ".")
1198
1199                 if t.supportedBuildmode("pie") && !disablePIE {
1200
1201                         t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", ".")
1202                         if t.internalLink() && t.internalLinkPIE() {
1203                                 t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", "-ldflags=-linkmode=internal", "-tags=internal,internal_pie", ".")
1204                         }
1205                 }
1206
1207         case "aix-ppc64",
1208                 "android-arm", "android-arm64",
1209                 "dragonfly-amd64",
1210                 "freebsd-386", "freebsd-amd64", "freebsd-arm", "freebsd-riscv64",
1211                 "linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-riscv64", "linux-s390x",
1212                 "netbsd-386", "netbsd-amd64",
1213                 "openbsd-386", "openbsd-amd64", "openbsd-arm", "openbsd-arm64", "openbsd-mips64":
1214
1215                 cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), ".")
1216                 setEnv(cmd, "GOFLAGS", "-ldflags=-linkmode=external")
1217                 // cgo should be able to cope with both -g arguments and colored
1218                 // diagnostics.
1219                 setEnv(cmd, "CGO_CFLAGS", "-g0 -fdiagnostics-color")
1220
1221                 t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-ldflags", "-linkmode=auto", ".")
1222                 t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-ldflags", "-linkmode=external", ".")
1223
1224                 switch pair {
1225                 case "aix-ppc64", "netbsd-386", "netbsd-amd64":
1226                         // no static linking
1227                 case "freebsd-arm":
1228                         // -fPIC compiled tls code will use __tls_get_addr instead
1229                         // of __aeabi_read_tp, however, on FreeBSD/ARM, __tls_get_addr
1230                         // is implemented in rtld-elf, so -fPIC isn't compatible with
1231                         // static linking on FreeBSD/ARM with clang. (cgo depends on
1232                         // -fPIC fundamentally.)
1233                 default:
1234                         cmd := t.dirCmd("misc/cgo/test",
1235                                 compilerEnvLookup(defaultcc, goos, goarch), "-xc", "-o", "/dev/null", "-static", "-")
1236                         cmd.Stdin = strings.NewReader("int main() {}")
1237                         if err := cmd.Run(); err != nil {
1238                                 fmt.Println("No support for static linking found (lacks libc.a?), skip cgo static linking test.")
1239                         } else {
1240                                 if goos != "android" {
1241                                         t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-ldflags", `-linkmode=external -extldflags "-static -pthread"`, ".")
1242                                 }
1243                                 t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), ".")
1244                                 t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), "-ldflags", `-linkmode=external`, ".")
1245                                 if goos != "android" {
1246                                         t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), "-ldflags", `-linkmode=external -extldflags "-static -pthread"`, ".")
1247                                         t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=static", "-ldflags", `-linkmode=external -extldflags "-static -pthread"`, ".")
1248                                         // -static in CGO_LDFLAGS triggers a different code path
1249                                         // than -static in -extldflags, so test both.
1250                                         // See issue #16651.
1251                                         cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), "-tags=static", ".")
1252                                         setEnv(cmd, "CGO_LDFLAGS", "-static -pthread")
1253                                 }
1254                         }
1255
1256                         if t.supportedBuildmode("pie") && !disablePIE {
1257                                 t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", ".")
1258                                 if t.internalLink() && t.internalLinkPIE() {
1259                                         t.addCmd(dt, "misc/cgo/test", t.goTest(), "-buildmode=pie", "-ldflags=-linkmode=internal", "-tags=internal,internal_pie", ".")
1260                                 }
1261                                 t.addCmd(dt, "misc/cgo/testtls", t.goTest(), "-buildmode=pie", ".")
1262                                 t.addCmd(dt, "misc/cgo/nocgo", t.goTest(), "-buildmode=pie", ".")
1263                         }
1264                 }
1265         }
1266
1267         return nil
1268 }
1269
1270 // run pending test commands, in parallel, emitting headers as appropriate.
1271 // When finished, emit header for nextTest, which is going to run after the
1272 // pending commands are done (and runPending returns).
1273 // A test should call runPending if it wants to make sure that it is not
1274 // running in parallel with earlier tests, or if it has some other reason
1275 // for needing the earlier tests to be done.
1276 func (t *tester) runPending(nextTest *distTest) {
1277         checkNotStale("go", "std")
1278         worklist := t.worklist
1279         t.worklist = nil
1280         for _, w := range worklist {
1281                 w.start = make(chan bool)
1282                 w.end = make(chan bool)
1283                 go func(w *work) {
1284                         if !<-w.start {
1285                                 timelog("skip", w.dt.name)
1286                                 w.out = []byte(fmt.Sprintf("skipped due to earlier error\n"))
1287                         } else {
1288                                 timelog("start", w.dt.name)
1289                                 w.out, w.err = w.cmd.CombinedOutput()
1290                                 if w.err != nil {
1291                                         if isUnsupportedVMASize(w) {
1292                                                 timelog("skip", w.dt.name)
1293                                                 w.out = []byte(fmt.Sprintf("skipped due to unsupported VMA\n"))
1294                                                 w.err = nil
1295                                         }
1296                                 }
1297                         }
1298                         timelog("end", w.dt.name)
1299                         w.end <- true
1300                 }(w)
1301         }
1302
1303         started := 0
1304         ended := 0
1305         var last *distTest
1306         for ended < len(worklist) {
1307                 for started < len(worklist) && started-ended < maxbg {
1308                         w := worklist[started]
1309                         started++
1310                         w.start <- !t.failed || t.keepGoing
1311                 }
1312                 w := worklist[ended]
1313                 dt := w.dt
1314                 if dt.heading != "" && t.lastHeading != dt.heading {
1315                         t.lastHeading = dt.heading
1316                         t.out(dt.heading)
1317                 }
1318                 if dt != last {
1319                         // Assumes all the entries for a single dt are in one worklist.
1320                         last = w.dt
1321                         if vflag > 0 {
1322                                 fmt.Printf("# go tool dist test -run=^%s$\n", dt.name)
1323                         }
1324                 }
1325                 if vflag > 1 {
1326                         errprintf("%s\n", strings.Join(w.cmd.Args, " "))
1327                 }
1328                 ended++
1329                 <-w.end
1330                 os.Stdout.Write(w.out)
1331                 if w.err != nil {
1332                         log.Printf("Failed: %v", w.err)
1333                         t.failed = true
1334                 }
1335                 checkNotStale("go", "std")
1336         }
1337         if t.failed && !t.keepGoing {
1338                 fatalf("FAILED")
1339         }
1340
1341         if dt := nextTest; dt != nil {
1342                 if dt.heading != "" && t.lastHeading != dt.heading {
1343                         t.lastHeading = dt.heading
1344                         t.out(dt.heading)
1345                 }
1346                 if vflag > 0 {
1347                         fmt.Printf("# go tool dist test -run=^%s$\n", dt.name)
1348                 }
1349         }
1350 }
1351
1352 func (t *tester) hasBash() bool {
1353         switch gohostos {
1354         case "windows", "plan9":
1355                 return false
1356         }
1357         return true
1358 }
1359
1360 func (t *tester) hasCxx() bool {
1361         cxx, _ := exec.LookPath(compilerEnvLookup(defaultcxx, goos, goarch))
1362         return cxx != ""
1363 }
1364
1365 func (t *tester) hasSwig() bool {
1366         swig, err := exec.LookPath("swig")
1367         if err != nil {
1368                 return false
1369         }
1370
1371         // Check that swig was installed with Go support by checking
1372         // that a go directory exists inside the swiglib directory.
1373         // See https://golang.org/issue/23469.
1374         output, err := exec.Command(swig, "-go", "-swiglib").Output()
1375         if err != nil {
1376                 return false
1377         }
1378         swigDir := strings.TrimSpace(string(output))
1379
1380         _, err = os.Stat(filepath.Join(swigDir, "go"))
1381         if err != nil {
1382                 return false
1383         }
1384
1385         // Check that swig has a new enough version.
1386         // See https://golang.org/issue/22858.
1387         out, err := exec.Command(swig, "-version").CombinedOutput()
1388         if err != nil {
1389                 return false
1390         }
1391
1392         re := regexp.MustCompile(`[vV]ersion +(\d+)([.]\d+)?([.]\d+)?`)
1393         matches := re.FindSubmatch(out)
1394         if matches == nil {
1395                 // Can't find version number; hope for the best.
1396                 return true
1397         }
1398
1399         major, err := strconv.Atoi(string(matches[1]))
1400         if err != nil {
1401                 // Can't find version number; hope for the best.
1402                 return true
1403         }
1404         if major < 3 {
1405                 return false
1406         }
1407         if major > 3 {
1408                 // 4.0 or later
1409                 return true
1410         }
1411
1412         // We have SWIG version 3.x.
1413         if len(matches[2]) > 0 {
1414                 minor, err := strconv.Atoi(string(matches[2][1:]))
1415                 if err != nil {
1416                         return true
1417                 }
1418                 if minor > 0 {
1419                         // 3.1 or later
1420                         return true
1421                 }
1422         }
1423
1424         // We have SWIG version 3.0.x.
1425         if len(matches[3]) > 0 {
1426                 patch, err := strconv.Atoi(string(matches[3][1:]))
1427                 if err != nil {
1428                         return true
1429                 }
1430                 if patch < 6 {
1431                         // Before 3.0.6.
1432                         return false
1433                 }
1434         }
1435
1436         return true
1437 }
1438
1439 func (t *tester) raceDetectorSupported() bool {
1440         if gohostos != goos {
1441                 return false
1442         }
1443         if !t.cgoEnabled {
1444                 return false
1445         }
1446         if !raceDetectorSupported(goos, goarch) {
1447                 return false
1448         }
1449         // The race detector doesn't work on Alpine Linux:
1450         // golang.org/issue/14481
1451         if isAlpineLinux() {
1452                 return false
1453         }
1454         // NetBSD support is unfinished.
1455         // golang.org/issue/26403
1456         if goos == "netbsd" {
1457                 return false
1458         }
1459         return true
1460 }
1461
1462 func isAlpineLinux() bool {
1463         if runtime.GOOS != "linux" {
1464                 return false
1465         }
1466         fi, err := os.Lstat("/etc/alpine-release")
1467         return err == nil && fi.Mode().IsRegular()
1468 }
1469
1470 func (t *tester) runFlag(rx string) string {
1471         if t.compileOnly {
1472                 return "-run=^$"
1473         }
1474         return "-run=" + rx
1475 }
1476
1477 func (t *tester) raceTest(dt *distTest) error {
1478         t.addCmd(dt, "src", t.goTest(), "-race", t.runFlag("Output"), "runtime/race")
1479         t.addCmd(dt, "src", t.goTest(), "-race", t.runFlag("TestParse|TestEcho|TestStdinCloseRace|TestClosedPipeRace|TestTypeRace|TestFdRace|TestFdReadRace|TestFileCloseRace"), "flag", "net", "os", "os/exec", "encoding/gob")
1480         // We don't want the following line, because it
1481         // slows down all.bash (by 10 seconds on my laptop).
1482         // The race builder should catch any error here, but doesn't.
1483         // TODO(iant): Figure out how to catch this.
1484         // t.addCmd(dt, "src", t.goTest(),  "-race", "-run=TestParallelTest", "cmd/go")
1485         if t.cgoEnabled {
1486                 // Building misc/cgo/test takes a long time.
1487                 // There are already cgo-enabled packages being tested with the race detector.
1488                 // We shouldn't need to redo all of misc/cgo/test too.
1489                 // The race buildler will take care of this.
1490                 // cmd := t.addCmd(dt, "misc/cgo/test", t.goTest(), "-race")
1491                 // setEnv(cmd, "GOTRACEBACK", "2")
1492         }
1493         if t.extLink() {
1494                 // Test with external linking; see issue 9133.
1495                 t.addCmd(dt, "src", t.goTest(), "-race", "-ldflags=-linkmode=external", t.runFlag("TestParse|TestEcho|TestStdinCloseRace"), "flag", "os/exec")
1496         }
1497         return nil
1498 }
1499
1500 var runtest struct {
1501         sync.Once
1502         exe string
1503         err error
1504 }
1505
1506 func (t *tester) testDirTest(dt *distTest, shard, shards int) error {
1507         runtest.Do(func() {
1508                 f, err := os.CreateTemp("", "runtest-*.exe") // named exe for Windows, but harmless elsewhere
1509                 if err != nil {
1510                         runtest.err = err
1511                         return
1512                 }
1513                 f.Close()
1514
1515                 runtest.exe = f.Name()
1516                 xatexit(func() {
1517                         os.Remove(runtest.exe)
1518                 })
1519
1520                 cmd := t.dirCmd("test", "go", "build", "-o", runtest.exe, "run.go")
1521                 setEnv(cmd, "GOOS", gohostos)
1522                 setEnv(cmd, "GOARCH", gohostarch)
1523                 runtest.err = cmd.Run()
1524         })
1525         if runtest.err != nil {
1526                 return runtest.err
1527         }
1528         if t.compileOnly {
1529                 return nil
1530         }
1531         t.addCmd(dt, "test", runtest.exe,
1532                 fmt.Sprintf("--shard=%d", shard),
1533                 fmt.Sprintf("--shards=%d", shards),
1534         )
1535         return nil
1536 }
1537
1538 // cgoPackages is the standard packages that use cgo.
1539 var cgoPackages = []string{
1540         "net",
1541         "os/user",
1542 }
1543
1544 var funcBenchmark = []byte("\nfunc Benchmark")
1545
1546 // packageHasBenchmarks reports whether pkg has benchmarks.
1547 // On any error, it conservatively returns true.
1548 //
1549 // This exists just to eliminate work on the builders, since compiling
1550 // a test in race mode just to discover it has no benchmarks costs a
1551 // second or two per package, and this function returns false for
1552 // about 100 packages.
1553 func (t *tester) packageHasBenchmarks(pkg string) bool {
1554         pkgDir := filepath.Join(goroot, "src", pkg)
1555         d, err := os.Open(pkgDir)
1556         if err != nil {
1557                 return true // conservatively
1558         }
1559         defer d.Close()
1560         names, err := d.Readdirnames(-1)
1561         if err != nil {
1562                 return true // conservatively
1563         }
1564         for _, name := range names {
1565                 if !strings.HasSuffix(name, "_test.go") {
1566                         continue
1567                 }
1568                 slurp, err := os.ReadFile(filepath.Join(pkgDir, name))
1569                 if err != nil {
1570                         return true // conservatively
1571                 }
1572                 if bytes.Contains(slurp, funcBenchmark) {
1573                         return true
1574                 }
1575         }
1576         return false
1577 }
1578
1579 // makeGOROOTUnwritable makes all $GOROOT files & directories non-writable to
1580 // check that no tests accidentally write to $GOROOT.
1581 func (t *tester) makeGOROOTUnwritable() (undo func()) {
1582         dir := os.Getenv("GOROOT")
1583         if dir == "" {
1584                 panic("GOROOT not set")
1585         }
1586
1587         type pathMode struct {
1588                 path string
1589                 mode os.FileMode
1590         }
1591         var dirs []pathMode // in lexical order
1592
1593         undo = func() {
1594                 for i := range dirs {
1595                         os.Chmod(dirs[i].path, dirs[i].mode) // best effort
1596                 }
1597         }
1598
1599         gocache := os.Getenv("GOCACHE")
1600         if gocache == "" {
1601                 panic("GOCACHE not set")
1602         }
1603         gocacheSubdir, _ := filepath.Rel(dir, gocache)
1604
1605         // Note: Can't use WalkDir here, because this has to compile with Go 1.4.
1606         filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
1607                 if suffix := strings.TrimPrefix(path, dir+string(filepath.Separator)); suffix != "" {
1608                         if suffix == gocacheSubdir {
1609                                 // Leave GOCACHE writable: we may need to write test binaries into it.
1610                                 return filepath.SkipDir
1611                         }
1612                         if suffix == ".git" {
1613                                 // Leave Git metadata in whatever state it was in. It may contain a lot
1614                                 // of files, and it is highly unlikely that a test will try to modify
1615                                 // anything within that directory.
1616                                 return filepath.SkipDir
1617                         }
1618                 }
1619                 if err == nil {
1620                         mode := info.Mode()
1621                         if mode&0222 != 0 && (mode.IsDir() || mode.IsRegular()) {
1622                                 dirs = append(dirs, pathMode{path, mode})
1623                         }
1624                 }
1625                 return nil
1626         })
1627
1628         // Run over list backward to chmod children before parents.
1629         for i := len(dirs) - 1; i >= 0; i-- {
1630                 err := os.Chmod(dirs[i].path, dirs[i].mode&^0222)
1631                 if err != nil {
1632                         dirs = dirs[i:] // Only undo what we did so far.
1633                         undo()
1634                         fatalf("failed to make GOROOT read-only: %v", err)
1635                 }
1636         }
1637
1638         return undo
1639 }
1640
1641 // raceDetectorSupported is a copy of the function
1642 // internal/platform.RaceDetectorSupported, which can't be used here
1643 // because cmd/dist has to be buildable by Go 1.4.
1644 // The race detector only supports 48-bit VMA on arm64. But we don't have
1645 // a good solution to check VMA size(See https://golang.org/issue/29948)
1646 // raceDetectorSupported will always return true for arm64. But race
1647 // detector tests may abort on non 48-bit VMA configuration, the tests
1648 // will be marked as "skipped" in this case.
1649 func raceDetectorSupported(goos, goarch string) bool {
1650         switch goos {
1651         case "linux":
1652                 return goarch == "amd64" || goarch == "ppc64le" || goarch == "arm64" || goarch == "s390x"
1653         case "darwin":
1654                 return goarch == "amd64" || goarch == "arm64"
1655         case "freebsd", "netbsd", "openbsd", "windows":
1656                 return goarch == "amd64"
1657         default:
1658                 return false
1659         }
1660 }
1661
1662 // isUnsupportedVMASize reports whether the failure is caused by an unsupported
1663 // VMA for the race detector (for example, running the race detector on an
1664 // arm64 machine configured with 39-bit VMA)
1665 func isUnsupportedVMASize(w *work) bool {
1666         unsupportedVMA := []byte("unsupported VMA range")
1667         return w.dt.name == "race" && bytes.Contains(w.out, unsupportedVMA)
1668 }
1669
1670 // isEnvSet reports whether the environment variable evar is
1671 // set in the environment.
1672 func isEnvSet(evar string) bool {
1673         evarEq := evar + "="
1674         for _, e := range os.Environ() {
1675                 if strings.HasPrefix(e, evarEq) {
1676                         return true
1677                 }
1678         }
1679         return false
1680 }