src/cmd/dist/buildtool.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // Build toolchain using Go 1.4.
   6 //
   7 // The general strategy is to copy the source files we need into
   8 // a new GOPATH workspace, adjust import paths appropriately,
   9 // invoke the Go 1.4 go command to build those sources,
  10 // and then copy the binaries back.
  11
  12 package main
  13
  14 import (
  15         "fmt"
  16         "os"
  17         "path/filepath"
  18         "runtime"
  19         "strings"
  20 )
  21
  22 // bootstrapDirs is a list of directories holding code that must be
  23 // compiled with a Go 1.4 toolchain to produce the bootstrapTargets.
  24 // All directories in this list are relative to and must be below $GOROOT/src.
  25 //
  26 // The list has have two kinds of entries: names beginning with cmd/ with
  27 // no other slashes, which are commands, and other paths, which are packages
  28 // supporting the commands. Packages in the standard library can be listed
  29 // if a newer copy needs to be substituted for the Go 1.4 copy when used
  30 // by the command packages.
  31 // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
  32 var bootstrapDirs = []string{
  33         "cmd/asm",
  34         "cmd/asm/internal/arch",
  35         "cmd/asm/internal/asm",
  36         "cmd/asm/internal/flags",
  37         "cmd/asm/internal/lex",
  38         "cmd/cgo",
  39         "cmd/compile",
  40         "cmd/compile/internal/amd64",
  41         "cmd/compile/internal/arm",
  42         "cmd/compile/internal/arm64",
  43         "cmd/compile/internal/gc",
  44         "cmd/compile/internal/logopt",
  45         "cmd/compile/internal/mips",
  46         "cmd/compile/internal/mips64",
  47         "cmd/compile/internal/ppc64",
  48         "cmd/compile/internal/riscv64",
  49         "cmd/compile/internal/s390x",
  50         "cmd/compile/internal/ssa",
  51         "cmd/compile/internal/syntax",
  52         "cmd/compile/internal/types",
  53         "cmd/compile/internal/x86",
  54         "cmd/compile/internal/wasm",
  55         "cmd/internal/bio",
  56         "cmd/internal/codesign",
  57         "cmd/internal/gcprog",
  58         "cmd/internal/dwarf",
  59         "cmd/internal/edit",
  60         "cmd/internal/goobj",
  61         "cmd/internal/objabi",
  62         "cmd/internal/obj",
  63         "cmd/internal/obj/arm",
  64         "cmd/internal/obj/arm64",
  65         "cmd/internal/obj/mips",
  66         "cmd/internal/obj/ppc64",
  67         "cmd/internal/obj/riscv",
  68         "cmd/internal/obj/s390x",
  69         "cmd/internal/obj/x86",
  70         "cmd/internal/obj/wasm",
  71         "cmd/internal/pkgpath",
  72         "cmd/internal/src",
  73         "cmd/internal/sys",
  74         "cmd/link",
  75         "cmd/link/internal/amd64",
  76         "cmd/link/internal/arm",
  77         "cmd/link/internal/arm64",
  78         "cmd/link/internal/benchmark",
  79         "cmd/link/internal/ld",
  80         "cmd/link/internal/loadelf",
  81         "cmd/link/internal/loader",
  82         "cmd/link/internal/loadmacho",
  83         "cmd/link/internal/loadpe",
  84         "cmd/link/internal/loadxcoff",
  85         "cmd/link/internal/mips",
  86         "cmd/link/internal/mips64",
  87         "cmd/link/internal/ppc64",
  88         "cmd/link/internal/riscv64",
  89         "cmd/link/internal/s390x",
  90         "cmd/link/internal/sym",
  91         "cmd/link/internal/x86",
  92         "compress/flate",
  93         "compress/zlib",
  94         "cmd/link/internal/wasm",
  95         "container/heap",
  96         "debug/dwarf",
  97         "debug/elf",
  98         "debug/macho",
  99         "debug/pe",
 100         "internal/goversion",
 101         "internal/race",
 102         "internal/unsafeheader",
 103         "internal/xcoff",
 104         "math/big",
 105         "math/bits",
 106         "sort",
 107 }
 108
 109 // File prefixes that are ignored by go/build anyway, and cause
 110 // problems with editor generated temporary files (#18931).
 111 var ignorePrefixes = []string{
 112         ".",
 113         "_",
 114 }
 115
 116 // File suffixes that use build tags introduced since Go 1.4.
 117 // These must not be copied into the bootstrap build directory.
 118 // Also ignore test files.
 119 var ignoreSuffixes = []string{
 120         "_arm64.s",
 121         "_arm64.go",
 122         "_riscv64.s",
 123         "_riscv64.go",
 124         "_wasm.s",
 125         "_wasm.go",
 126         "_test.s",
 127 }
 128
 129 func bootstrapBuildTools() {
 130         goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
 131         if goroot_bootstrap == "" {
 132                 goroot_bootstrap = pathf("%s/go1.4", os.Getenv("HOME"))
 133         }
 134         xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
 135
 136         mkzbootstrap(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
 137
 138         // Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
 139         // We use a subdirectory of $GOROOT/pkg because that's the
 140         // space within $GOROOT where we store all generated objects.
 141         // We could use a temporary directory outside $GOROOT instead,
 142         // but it is easier to debug on failure if the files are in a known location.
 143         workspace := pathf("%s/pkg/bootstrap", goroot)
 144         xremoveall(workspace)
 145         xatexit(func() { xremoveall(workspace) })
 146         base := pathf("%s/src/bootstrap", workspace)
 147         xmkdirall(base)
 148
 149         // Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
 150         writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
 151         for _, dir := range bootstrapDirs {
 152                 src := pathf("%s/src/%s", goroot, dir)
 153                 dst := pathf("%s/%s", base, dir)
 154                 xmkdirall(dst)
 155                 if dir == "cmd/cgo" {
 156                         // Write to src because we need the file both for bootstrap
 157                         // and for later in the main build.
 158                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
 159                 }
 160         Dir:
 161                 for _, name := range xreaddirfiles(src) {
 162                         for _, pre := range ignorePrefixes {
 163                                 if strings.HasPrefix(name, pre) {
 164                                         continue Dir
 165                                 }
 166                         }
 167                         for _, suf := range ignoreSuffixes {
 168                                 if strings.HasSuffix(name, suf) {
 169                                         continue Dir
 170                                 }
 171                         }
 172                         srcFile := pathf("%s/%s", src, name)
 173                         dstFile := pathf("%s/%s", dst, name)
 174                         text := bootstrapRewriteFile(srcFile)
 175                         writefile(text, dstFile, 0)
 176                 }
 177         }
 178
 179         // Set up environment for invoking Go 1.4 go command.
 180         // GOROOT points at Go 1.4 GOROOT,
 181         // GOPATH points at our bootstrap workspace,
 182         // GOBIN is empty, so that binaries are installed to GOPATH/bin,
 183         // and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
 184         // so that Go 1.4 builds whatever kind of binary it knows how to build.
 185         // Restore GOROOT, GOPATH, and GOBIN when done.
 186         // Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
 187         // because setup will take care of those when bootstrapBuildTools returns.
 188
 189         defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
 190         os.Setenv("GOROOT", goroot_bootstrap)
 191
 192         defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
 193         os.Setenv("GOPATH", workspace)
 194
 195         defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
 196         os.Setenv("GOBIN", "")
 197
 198         os.Setenv("GOOS", "")
 199         os.Setenv("GOHOSTOS", "")
 200         os.Setenv("GOARCH", "")
 201         os.Setenv("GOHOSTARCH", "")
 202
 203         // Run Go 1.4 to build binaries. Use -gcflags=-l to disable inlining to
 204         // workaround bugs in Go 1.4's compiler. See discussion thread:
 205         // https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ
 206         // Use the math_big_pure_go build tag to disable the assembly in math/big
 207         // which may contain unsupported instructions.
 208         // Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l
 209         // only applies to the final cmd/go binary, but that's OK: if this is Go 1.10
 210         // or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler.
 211         cmd := []string{
 212                 pathf("%s/bin/go", goroot_bootstrap),
 213                 "install",
 214                 "-gcflags=-l",
 215                 "-tags=math_big_pure_go compiler_bootstrap",
 216         }
 217         if vflag > 0 {
 218                 cmd = append(cmd, "-v")
 219         }
 220         if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
 221                 cmd = append(cmd, "-toolexec="+tool)
 222         }
 223         cmd = append(cmd, "bootstrap/cmd/...")
 224         run(base, ShowOutput|CheckExit, cmd...)
 225
 226         // Copy binaries into tool binary directory.
 227         for _, name := range bootstrapDirs {
 228                 if !strings.HasPrefix(name, "cmd/") {
 229                         continue
 230                 }
 231                 name = name[len("cmd/"):]
 232                 if !strings.Contains(name, "/") {
 233                         copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
 234                 }
 235         }
 236
 237         if vflag > 0 {
 238                 xprintf("\n")
 239         }
 240 }
 241
 242 var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
 243
 244 // isUnneededSSARewriteFile reports whether srcFile is a
 245 // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
 246 // architecture that isn't for the current runtime.GOARCH.
 247 //
 248 // When unneeded is true archCaps is the rewrite base filename without
 249 // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
 250 func isUnneededSSARewriteFile(srcFile string) (archCaps string, unneeded bool) {
 251         if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
 252                 return "", false
 253         }
 254         fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
 255         if fileArch == "" {
 256                 return "", false
 257         }
 258         b := fileArch[0]
 259         if b == '_' || ('a' <= b && b <= 'z') {
 260                 return "", false
 261         }
 262         archCaps = fileArch
 263         fileArch = strings.ToLower(fileArch)
 264         fileArch = strings.TrimSuffix(fileArch, "splitload")
 265         if fileArch == os.Getenv("GOHOSTARCH") {
 266                 return "", false
 267         }
 268         if fileArch == strings.TrimSuffix(runtime.GOARCH, "le") {
 269                 return "", false
 270         }
 271         if fileArch == strings.TrimSuffix(os.Getenv("GOARCH"), "le") {
 272                 return "", false
 273         }
 274         return archCaps, true
 275 }
 276
 277 func bootstrapRewriteFile(srcFile string) string {
 278         // During bootstrap, generate dummy rewrite files for
 279         // irrelevant architectures. We only need to build a bootstrap
 280         // binary that works for the current runtime.GOARCH.
 281         // This saves 6+ seconds of bootstrap.
 282         if archCaps, ok := isUnneededSSARewriteFile(srcFile); ok {
 283                 return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.
 284
 285 package ssa
 286
 287 func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
 288 func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
 289 `, archCaps, archCaps)
 290         }
 291
 292         return bootstrapFixImports(srcFile)
 293 }
 294
 295 func bootstrapFixImports(srcFile string) string {
 296         lines := strings.SplitAfter(readfile(srcFile), "\n")
 297         inBlock := false
 298         for i, line := range lines {
 299                 if strings.HasPrefix(line, "import (") {
 300                         inBlock = true
 301                         continue
 302                 }
 303                 if inBlock && strings.HasPrefix(line, ")") {
 304                         inBlock = false
 305                         continue
 306                 }
 307                 if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
 308                         inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"")) {
 309                         line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
 310                         // During bootstrap, must use plain os/exec.
 311                         line = strings.Replace(line, `exec "internal/execabs"`, `"os/exec"`, -1)
 312                         for _, dir := range bootstrapDirs {
 313                                 if strings.HasPrefix(dir, "cmd/") {
 314                                         continue
 315                                 }
 316                                 line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
 317                         }
 318                         lines[i] = line
 319                 }
 320         }
 321
 322         lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
 323
 324         return strings.Join(lines, "")
 325 }