src/cmd/dist/buildtool.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // Build toolchain using Go 1.4.
   6 //
   7 // The general strategy is to copy the source files we need into
   8 // a new GOPATH workspace, adjust import paths appropriately,
   9 // invoke the Go 1.4 go command to build those sources,
  10 // and then copy the binaries back.
  11
  12 package main
  13
  14 import (
  15         "fmt"
  16         "os"
  17         "path/filepath"
  18         "regexp"
  19         "runtime"
  20         "strings"
  21 )
  22
  23 // bootstrapDirs is a list of directories holding code that must be
  24 // compiled with a Go 1.4 toolchain to produce the bootstrapTargets.
  25 // All directories in this list are relative to and must be below $GOROOT/src.
  26 //
  27 // The list has two kinds of entries: names beginning with cmd/ with
  28 // no other slashes, which are commands, and other paths, which are packages
  29 // supporting the commands. Packages in the standard library can be listed
  30 // if a newer copy needs to be substituted for the Go 1.4 copy when used
  31 // by the command packages. Paths ending with /... automatically
  32 // include all packages within subdirectories as well.
  33 // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
  34 var bootstrapDirs = []string{
  35         "cmd/asm",
  36         "cmd/asm/internal/...",
  37         "cmd/cgo",
  38         "cmd/compile",
  39         "cmd/compile/internal/...",
  40         "cmd/internal/archive",
  41         "cmd/internal/bio",
  42         "cmd/internal/codesign",
  43         "cmd/internal/dwarf",
  44         "cmd/internal/edit",
  45         "cmd/internal/gcprog",
  46         "cmd/internal/goobj",
  47         "cmd/internal/obj/...",
  48         "cmd/internal/objabi",
  49         "cmd/internal/pkgpath",
  50         "cmd/internal/quoted",
  51         "cmd/internal/src",
  52         "cmd/internal/sys",
  53         "cmd/link",
  54         "cmd/link/internal/...",
  55         "compress/flate",
  56         "compress/zlib",
  57         "container/heap",
  58         "debug/dwarf",
  59         "debug/elf",
  60         "debug/macho",
  61         "debug/pe",
  62         "go/constant",
  63         "internal/buildcfg",
  64         "internal/goexperiment",
  65         "internal/goversion",
  66         "internal/race",
  67         "internal/unsafeheader",
  68         "internal/xcoff",
  69         "math/big",
  70         "math/bits",
  71         "sort",
  72         "strconv",
  73 }
  74
  75 // File prefixes that are ignored by go/build anyway, and cause
  76 // problems with editor generated temporary files (#18931).
  77 var ignorePrefixes = []string{
  78         ".",
  79         "_",
  80         "#",
  81 }
  82
  83 // File suffixes that use build tags introduced since Go 1.4.
  84 // These must not be copied into the bootstrap build directory.
  85 // Also ignore test files.
  86 var ignoreSuffixes = []string{
  87         "_arm64.s",
  88         "_arm64.go",
  89         "_riscv64.s",
  90         "_riscv64.go",
  91         "_wasm.s",
  92         "_wasm.go",
  93         "_test.s",
  94         "_test.go",
  95 }
  96
  97 var tryDirs = []string{
  98         "sdk/go1.17",
  99         "go1.17",
 100 }
 101
 102 func bootstrapBuildTools() {
 103         goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
 104         if goroot_bootstrap == "" {
 105                 home := os.Getenv("HOME")
 106                 goroot_bootstrap = pathf("%s/go1.4", home)
 107                 for _, d := range tryDirs {
 108                         if p := pathf("%s/%s", home, d); isdir(p) {
 109                                 goroot_bootstrap = p
 110                         }
 111                 }
 112         }
 113         xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
 114
 115         mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
 116         mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
 117
 118         // Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
 119         // We use a subdirectory of $GOROOT/pkg because that's the
 120         // space within $GOROOT where we store all generated objects.
 121         // We could use a temporary directory outside $GOROOT instead,
 122         // but it is easier to debug on failure if the files are in a known location.
 123         workspace := pathf("%s/pkg/bootstrap", goroot)
 124         xremoveall(workspace)
 125         xatexit(func() { xremoveall(workspace) })
 126         base := pathf("%s/src/bootstrap", workspace)
 127         xmkdirall(base)
 128
 129         // Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
 130         writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
 131         for _, dir := range bootstrapDirs {
 132                 recurse := strings.HasSuffix(dir, "/...")
 133                 dir = strings.TrimSuffix(dir, "/...")
 134                 filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 135                         if err != nil {
 136                                 fatalf("walking bootstrap dirs failed: %v: %v", path, err)
 137                         }
 138
 139                         name := filepath.Base(path)
 140                         src := pathf("%s/src/%s", goroot, path)
 141                         dst := pathf("%s/%s", base, path)
 142
 143                         if info.IsDir() {
 144                                 if !recurse && path != dir || name == "testdata" {
 145                                         return filepath.SkipDir
 146                                 }
 147
 148                                 xmkdirall(dst)
 149                                 if path == "cmd/cgo" {
 150                                         // Write to src because we need the file both for bootstrap
 151                                         // and for later in the main build.
 152                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
 153                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
 154                                 }
 155                                 return nil
 156                         }
 157
 158                         for _, pre := range ignorePrefixes {
 159                                 if strings.HasPrefix(name, pre) {
 160                                         return nil
 161                                 }
 162                         }
 163                         for _, suf := range ignoreSuffixes {
 164                                 if strings.HasSuffix(name, suf) {
 165                                         return nil
 166                                 }
 167                         }
 168
 169                         text := bootstrapRewriteFile(src)
 170                         writefile(text, dst, 0)
 171                         return nil
 172                 })
 173         }
 174
 175         // Set up environment for invoking Go 1.4 go command.
 176         // GOROOT points at Go 1.4 GOROOT,
 177         // GOPATH points at our bootstrap workspace,
 178         // GOBIN is empty, so that binaries are installed to GOPATH/bin,
 179         // and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
 180         // so that Go 1.4 builds whatever kind of binary it knows how to build.
 181         // Restore GOROOT, GOPATH, and GOBIN when done.
 182         // Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
 183         // because setup will take care of those when bootstrapBuildTools returns.
 184
 185         defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
 186         os.Setenv("GOROOT", goroot_bootstrap)
 187
 188         defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
 189         os.Setenv("GOPATH", workspace)
 190
 191         defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
 192         os.Setenv("GOBIN", "")
 193
 194         os.Setenv("GOOS", "")
 195         os.Setenv("GOHOSTOS", "")
 196         os.Setenv("GOARCH", "")
 197         os.Setenv("GOHOSTARCH", "")
 198
 199         // Run Go 1.4 to build binaries. Use -gcflags=-l to disable inlining to
 200         // workaround bugs in Go 1.4's compiler. See discussion thread:
 201         // https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ
 202         // Use the math_big_pure_go build tag to disable the assembly in math/big
 203         // which may contain unsupported instructions.
 204         // Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l
 205         // only applies to the final cmd/go binary, but that's OK: if this is Go 1.10
 206         // or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler.
 207         cmd := []string{
 208                 pathf("%s/bin/go", goroot_bootstrap),
 209                 "install",
 210                 "-gcflags=-l",
 211                 "-tags=math_big_pure_go compiler_bootstrap",
 212         }
 213         if vflag > 0 {
 214                 cmd = append(cmd, "-v")
 215         }
 216         if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
 217                 cmd = append(cmd, "-toolexec="+tool)
 218         }
 219         cmd = append(cmd, "bootstrap/cmd/...")
 220         run(base, ShowOutput|CheckExit, cmd...)
 221
 222         // Copy binaries into tool binary directory.
 223         for _, name := range bootstrapDirs {
 224                 if !strings.HasPrefix(name, "cmd/") {
 225                         continue
 226                 }
 227                 name = name[len("cmd/"):]
 228                 if !strings.Contains(name, "/") {
 229                         copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
 230                 }
 231         }
 232
 233         if vflag > 0 {
 234                 xprintf("\n")
 235         }
 236 }
 237
 238 var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
 239
 240 // isUnneededSSARewriteFile reports whether srcFile is a
 241 // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
 242 // architecture that isn't for the current runtime.GOARCH.
 243 //
 244 // When unneeded is true archCaps is the rewrite base filename without
 245 // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
 246 func isUnneededSSARewriteFile(srcFile string) (archCaps string, unneeded bool) {
 247         if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
 248                 return "", false
 249         }
 250         fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
 251         if fileArch == "" {
 252                 return "", false
 253         }
 254         b := fileArch[0]
 255         if b == '_' || ('a' <= b && b <= 'z') {
 256                 return "", false
 257         }
 258         archCaps = fileArch
 259         fileArch = strings.ToLower(fileArch)
 260         fileArch = strings.TrimSuffix(fileArch, "splitload")
 261         if fileArch == os.Getenv("GOHOSTARCH") {
 262                 return "", false
 263         }
 264         if fileArch == strings.TrimSuffix(runtime.GOARCH, "le") {
 265                 return "", false
 266         }
 267         if fileArch == strings.TrimSuffix(os.Getenv("GOARCH"), "le") {
 268                 return "", false
 269         }
 270         return archCaps, true
 271 }
 272
 273 func bootstrapRewriteFile(srcFile string) string {
 274         // During bootstrap, generate dummy rewrite files for
 275         // irrelevant architectures. We only need to build a bootstrap
 276         // binary that works for the current runtime.GOARCH.
 277         // This saves 6+ seconds of bootstrap.
 278         if archCaps, ok := isUnneededSSARewriteFile(srcFile); ok {
 279                 return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.
 280
 281 package ssa
 282
 283 func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
 284 func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
 285 `, archCaps, archCaps)
 286         }
 287
 288         return bootstrapFixImports(srcFile)
 289 }
 290
 291 func bootstrapFixImports(srcFile string) string {
 292         text := readfile(srcFile)
 293         if !strings.Contains(srcFile, "/cmd/") && !strings.Contains(srcFile, `\cmd\`) {
 294                 text = regexp.MustCompile(`\bany\b`).ReplaceAllString(text, "interface{}")
 295         }
 296         lines := strings.SplitAfter(text, "\n")
 297         inBlock := false
 298         for i, line := range lines {
 299                 if strings.HasPrefix(line, "import (") {
 300                         inBlock = true
 301                         continue
 302                 }
 303                 if inBlock && strings.HasPrefix(line, ")") {
 304                         inBlock = false
 305                         continue
 306                 }
 307                 if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
 308                         inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"")) {
 309                         line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
 310                         // During bootstrap, must use plain os/exec.
 311                         line = strings.Replace(line, `exec "internal/execabs"`, `"os/exec"`, -1)
 312                         for _, dir := range bootstrapDirs {
 313                                 if strings.HasPrefix(dir, "cmd/") {
 314                                         continue
 315                                 }
 316                                 line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
 317                         }
 318                         lines[i] = line
 319                 }
 320         }
 321
 322         lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
 323
 324         return strings.Join(lines, "")
 325 }