src/cmd/dist/buildtool.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // Build toolchain using Go 1.4.
   6 //
   7 // The general strategy is to copy the source files we need into
   8 // a new GOPATH workspace, adjust import paths appropriately,
   9 // invoke the Go 1.4 go command to build those sources,
  10 // and then copy the binaries back.
  11
  12 package main
  13
  14 import (
  15         "fmt"
  16         "os"
  17         "path/filepath"
  18         "runtime"
  19         "strings"
  20 )
  21
  22 // bootstrapDirs is a list of directories holding code that must be
  23 // compiled with a Go 1.4 toolchain to produce the bootstrapTargets.
  24 // All directories in this list are relative to and must be below $GOROOT/src.
  25 //
  26 // The list has two kinds of entries: names beginning with cmd/ with
  27 // no other slashes, which are commands, and other paths, which are packages
  28 // supporting the commands. Packages in the standard library can be listed
  29 // if a newer copy needs to be substituted for the Go 1.4 copy when used
  30 // by the command packages. Paths ending with /... automatically
  31 // include all packages within subdirectories as well.
  32 // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
  33 var bootstrapDirs = []string{
  34         "cmd/asm",
  35         "cmd/asm/internal/...",
  36         "cmd/cgo",
  37         "cmd/compile",
  38         "cmd/compile/internal/...",
  39         "cmd/internal/archive",
  40         "cmd/internal/bio",
  41         "cmd/internal/codesign",
  42         "cmd/internal/dwarf",
  43         "cmd/internal/edit",
  44         "cmd/internal/gcprog",
  45         "cmd/internal/goobj",
  46         "cmd/internal/obj/...",
  47         "cmd/internal/objabi",
  48         "cmd/internal/pkgpath",
  49         "cmd/internal/src",
  50         "cmd/internal/sys",
  51         "cmd/link",
  52         "cmd/link/internal/...",
  53         "compress/flate",
  54         "compress/zlib",
  55         "container/heap",
  56         "debug/dwarf",
  57         "debug/elf",
  58         "debug/macho",
  59         "debug/pe",
  60         "go/constant",
  61         "internal/goexperiment",
  62         "internal/goversion",
  63         "internal/race",
  64         "internal/unsafeheader",
  65         "internal/xcoff",
  66         "math/big",
  67         "math/bits",
  68         "sort",
  69         "strconv",
  70 }
  71
  72 // File prefixes that are ignored by go/build anyway, and cause
  73 // problems with editor generated temporary files (#18931).
  74 var ignorePrefixes = []string{
  75         ".",
  76         "_",
  77         "#",
  78 }
  79
  80 // File suffixes that use build tags introduced since Go 1.4.
  81 // These must not be copied into the bootstrap build directory.
  82 // Also ignore test files.
  83 var ignoreSuffixes = []string{
  84         "_arm64.s",
  85         "_arm64.go",
  86         "_riscv64.s",
  87         "_riscv64.go",
  88         "_wasm.s",
  89         "_wasm.go",
  90         "_test.s",
  91         "_test.go",
  92 }
  93
  94 func bootstrapBuildTools() {
  95         goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
  96         if goroot_bootstrap == "" {
  97                 goroot_bootstrap = pathf("%s/go1.4", os.Getenv("HOME"))
  98         }
  99         xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
 100
 101         mkzbootstrap(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
 102
 103         // Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
 104         // We use a subdirectory of $GOROOT/pkg because that's the
 105         // space within $GOROOT where we store all generated objects.
 106         // We could use a temporary directory outside $GOROOT instead,
 107         // but it is easier to debug on failure if the files are in a known location.
 108         workspace := pathf("%s/pkg/bootstrap", goroot)
 109         xremoveall(workspace)
 110         xatexit(func() { xremoveall(workspace) })
 111         base := pathf("%s/src/bootstrap", workspace)
 112         xmkdirall(base)
 113
 114         // Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
 115         writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
 116         for _, dir := range bootstrapDirs {
 117                 recurse := strings.HasSuffix(dir, "/...")
 118                 dir = strings.TrimSuffix(dir, "/...")
 119                 filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 120                         if err != nil {
 121                                 fatalf("walking bootstrap dirs failed: %v: %v", path, err)
 122                         }
 123
 124                         name := filepath.Base(path)
 125                         src := pathf("%s/src/%s", goroot, path)
 126                         dst := pathf("%s/%s", base, path)
 127
 128                         if info.IsDir() {
 129                                 if !recurse && path != dir || name == "testdata" {
 130                                         return filepath.SkipDir
 131                                 }
 132
 133                                 xmkdirall(dst)
 134                                 if path == "cmd/cgo" {
 135                                         // Write to src because we need the file both for bootstrap
 136                                         // and for later in the main build.
 137                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
 138                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
 139                                 }
 140                                 return nil
 141                         }
 142
 143                         for _, pre := range ignorePrefixes {
 144                                 if strings.HasPrefix(name, pre) {
 145                                         return nil
 146                                 }
 147                         }
 148                         for _, suf := range ignoreSuffixes {
 149                                 if strings.HasSuffix(name, suf) {
 150                                         return nil
 151                                 }
 152                         }
 153
 154                         text := bootstrapRewriteFile(src)
 155                         writefile(text, dst, 0)
 156                         return nil
 157                 })
 158         }
 159
 160         // Set up environment for invoking Go 1.4 go command.
 161         // GOROOT points at Go 1.4 GOROOT,
 162         // GOPATH points at our bootstrap workspace,
 163         // GOBIN is empty, so that binaries are installed to GOPATH/bin,
 164         // and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
 165         // so that Go 1.4 builds whatever kind of binary it knows how to build.
 166         // Restore GOROOT, GOPATH, and GOBIN when done.
 167         // Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
 168         // because setup will take care of those when bootstrapBuildTools returns.
 169
 170         defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
 171         os.Setenv("GOROOT", goroot_bootstrap)
 172
 173         defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
 174         os.Setenv("GOPATH", workspace)
 175
 176         defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
 177         os.Setenv("GOBIN", "")
 178
 179         os.Setenv("GOOS", "")
 180         os.Setenv("GOHOSTOS", "")
 181         os.Setenv("GOARCH", "")
 182         os.Setenv("GOHOSTARCH", "")
 183
 184         // Run Go 1.4 to build binaries. Use -gcflags=-l to disable inlining to
 185         // workaround bugs in Go 1.4's compiler. See discussion thread:
 186         // https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ
 187         // Use the math_big_pure_go build tag to disable the assembly in math/big
 188         // which may contain unsupported instructions.
 189         // Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l
 190         // only applies to the final cmd/go binary, but that's OK: if this is Go 1.10
 191         // or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler.
 192         cmd := []string{
 193                 pathf("%s/bin/go", goroot_bootstrap),
 194                 "install",
 195                 "-gcflags=-l",
 196                 "-tags=math_big_pure_go compiler_bootstrap",
 197         }
 198         if vflag > 0 {
 199                 cmd = append(cmd, "-v")
 200         }
 201         if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
 202                 cmd = append(cmd, "-toolexec="+tool)
 203         }
 204         cmd = append(cmd, "bootstrap/cmd/...")
 205         run(base, ShowOutput|CheckExit, cmd...)
 206
 207         // Copy binaries into tool binary directory.
 208         for _, name := range bootstrapDirs {
 209                 if !strings.HasPrefix(name, "cmd/") {
 210                         continue
 211                 }
 212                 name = name[len("cmd/"):]
 213                 if !strings.Contains(name, "/") {
 214                         copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
 215                 }
 216         }
 217
 218         if vflag > 0 {
 219                 xprintf("\n")
 220         }
 221 }
 222
 223 var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
 224
 225 // isUnneededSSARewriteFile reports whether srcFile is a
 226 // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
 227 // architecture that isn't for the current runtime.GOARCH.
 228 //
 229 // When unneeded is true archCaps is the rewrite base filename without
 230 // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
 231 func isUnneededSSARewriteFile(srcFile string) (archCaps string, unneeded bool) {
 232         if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
 233                 return "", false
 234         }
 235         fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
 236         if fileArch == "" {
 237                 return "", false
 238         }
 239         b := fileArch[0]
 240         if b == '_' || ('a' <= b && b <= 'z') {
 241                 return "", false
 242         }
 243         archCaps = fileArch
 244         fileArch = strings.ToLower(fileArch)
 245         fileArch = strings.TrimSuffix(fileArch, "splitload")
 246         if fileArch == os.Getenv("GOHOSTARCH") {
 247                 return "", false
 248         }
 249         if fileArch == strings.TrimSuffix(runtime.GOARCH, "le") {
 250                 return "", false
 251         }
 252         if fileArch == strings.TrimSuffix(os.Getenv("GOARCH"), "le") {
 253                 return "", false
 254         }
 255         return archCaps, true
 256 }
 257
 258 func bootstrapRewriteFile(srcFile string) string {
 259         // During bootstrap, generate dummy rewrite files for
 260         // irrelevant architectures. We only need to build a bootstrap
 261         // binary that works for the current runtime.GOARCH.
 262         // This saves 6+ seconds of bootstrap.
 263         if archCaps, ok := isUnneededSSARewriteFile(srcFile); ok {
 264                 return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.
 265
 266 package ssa
 267
 268 func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
 269 func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
 270 `, archCaps, archCaps)
 271         }
 272
 273         return bootstrapFixImports(srcFile)
 274 }
 275
 276 func bootstrapFixImports(srcFile string) string {
 277         lines := strings.SplitAfter(readfile(srcFile), "\n")
 278         inBlock := false
 279         for i, line := range lines {
 280                 if strings.HasPrefix(line, "import (") {
 281                         inBlock = true
 282                         continue
 283                 }
 284                 if inBlock && strings.HasPrefix(line, ")") {
 285                         inBlock = false
 286                         continue
 287                 }
 288                 if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
 289                         inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"")) {
 290                         line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
 291                         // During bootstrap, must use plain os/exec.
 292                         line = strings.Replace(line, `exec "internal/execabs"`, `"os/exec"`, -1)
 293                         for _, dir := range bootstrapDirs {
 294                                 if strings.HasPrefix(dir, "cmd/") {
 295                                         continue
 296                                 }
 297                                 line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
 298                         }
 299                         lines[i] = line
 300                 }
 301         }
 302
 303         lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
 304
 305         return strings.Join(lines, "")
 306 }