src/cmd/dist/buildtool.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // Build toolchain using Go 1.4.
   6 //
   7 // The general strategy is to copy the source files we need into
   8 // a new GOPATH workspace, adjust import paths appropriately,
   9 // invoke the Go 1.4 go command to build those sources,
  10 // and then copy the binaries back.
  11
  12 package main
  13
  14 import (
  15         "fmt"
  16         "os"
  17         "path/filepath"
  18         "runtime"
  19         "strings"
  20 )
  21
  22 // bootstrapDirs is a list of directories holding code that must be
  23 // compiled with a Go 1.4 toolchain to produce the bootstrapTargets.
  24 // All directories in this list are relative to and must be below $GOROOT/src.
  25 //
  26 // The list has have two kinds of entries: names beginning with cmd/ with
  27 // no other slashes, which are commands, and other paths, which are packages
  28 // supporting the commands. Packages in the standard library can be listed
  29 // if a newer copy needs to be substituted for the Go 1.4 copy when used
  30 // by the command packages.
  31 // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
  32 var bootstrapDirs = []string{
  33         "cmd/asm",
  34         "cmd/asm/internal/arch",
  35         "cmd/asm/internal/asm",
  36         "cmd/asm/internal/flags",
  37         "cmd/asm/internal/lex",
  38         "cmd/cgo",
  39         "cmd/compile",
  40         "cmd/compile/internal/amd64",
  41         "cmd/compile/internal/arm",
  42         "cmd/compile/internal/arm64",
  43         "cmd/compile/internal/gc",
  44         "cmd/compile/internal/logopt",
  45         "cmd/compile/internal/mips",
  46         "cmd/compile/internal/mips64",
  47         "cmd/compile/internal/ppc64",
  48         "cmd/compile/internal/riscv64",
  49         "cmd/compile/internal/s390x",
  50         "cmd/compile/internal/ssa",
  51         "cmd/compile/internal/syntax",
  52         "cmd/compile/internal/types",
  53         "cmd/compile/internal/x86",
  54         "cmd/compile/internal/wasm",
  55         "cmd/internal/bio",
  56         "cmd/internal/gcprog",
  57         "cmd/internal/dwarf",
  58         "cmd/internal/edit",
  59         "cmd/internal/goobj",
  60         "cmd/internal/objabi",
  61         "cmd/internal/obj",
  62         "cmd/internal/obj/arm",
  63         "cmd/internal/obj/arm64",
  64         "cmd/internal/obj/mips",
  65         "cmd/internal/obj/ppc64",
  66         "cmd/internal/obj/riscv",
  67         "cmd/internal/obj/s390x",
  68         "cmd/internal/obj/x86",
  69         "cmd/internal/obj/wasm",
  70         "cmd/internal/pkgpath",
  71         "cmd/internal/src",
  72         "cmd/internal/sys",
  73         "cmd/link",
  74         "cmd/link/internal/amd64",
  75         "cmd/link/internal/arm",
  76         "cmd/link/internal/arm64",
  77         "cmd/link/internal/benchmark",
  78         "cmd/link/internal/ld",
  79         "cmd/link/internal/loadelf",
  80         "cmd/link/internal/loader",
  81         "cmd/link/internal/loadmacho",
  82         "cmd/link/internal/loadpe",
  83         "cmd/link/internal/loadxcoff",
  84         "cmd/link/internal/mips",
  85         "cmd/link/internal/mips64",
  86         "cmd/link/internal/ppc64",
  87         "cmd/link/internal/riscv64",
  88         "cmd/link/internal/s390x",
  89         "cmd/link/internal/sym",
  90         "cmd/link/internal/x86",
  91         "compress/flate",
  92         "compress/zlib",
  93         "cmd/link/internal/wasm",
  94         "container/heap",
  95         "debug/dwarf",
  96         "debug/elf",
  97         "debug/macho",
  98         "debug/pe",
  99         "internal/goversion",
 100         "internal/race",
 101         "internal/unsafeheader",
 102         "internal/xcoff",
 103         "math/big",
 104         "math/bits",
 105         "sort",
 106 }
 107
 108 // File prefixes that are ignored by go/build anyway, and cause
 109 // problems with editor generated temporary files (#18931).
 110 var ignorePrefixes = []string{
 111         ".",
 112         "_",
 113 }
 114
 115 // File suffixes that use build tags introduced since Go 1.4.
 116 // These must not be copied into the bootstrap build directory.
 117 // Also ignore test files.
 118 var ignoreSuffixes = []string{
 119         "_arm64.s",
 120         "_arm64.go",
 121         "_riscv64.s",
 122         "_riscv64.go",
 123         "_wasm.s",
 124         "_wasm.go",
 125         "_test.s",
 126 }
 127
 128 func bootstrapBuildTools() {
 129         goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
 130         if goroot_bootstrap == "" {
 131                 goroot_bootstrap = pathf("%s/go1.4", os.Getenv("HOME"))
 132         }
 133         xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
 134
 135         mkzbootstrap(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
 136
 137         // Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
 138         // We use a subdirectory of $GOROOT/pkg because that's the
 139         // space within $GOROOT where we store all generated objects.
 140         // We could use a temporary directory outside $GOROOT instead,
 141         // but it is easier to debug on failure if the files are in a known location.
 142         workspace := pathf("%s/pkg/bootstrap", goroot)
 143         xremoveall(workspace)
 144         xatexit(func() { xremoveall(workspace) })
 145         base := pathf("%s/src/bootstrap", workspace)
 146         xmkdirall(base)
 147
 148         // Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
 149         writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
 150         for _, dir := range bootstrapDirs {
 151                 src := pathf("%s/src/%s", goroot, dir)
 152                 dst := pathf("%s/%s", base, dir)
 153                 xmkdirall(dst)
 154                 if dir == "cmd/cgo" {
 155                         // Write to src because we need the file both for bootstrap
 156                         // and for later in the main build.
 157                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
 158                 }
 159         Dir:
 160                 for _, name := range xreaddirfiles(src) {
 161                         for _, pre := range ignorePrefixes {
 162                                 if strings.HasPrefix(name, pre) {
 163                                         continue Dir
 164                                 }
 165                         }
 166                         for _, suf := range ignoreSuffixes {
 167                                 if strings.HasSuffix(name, suf) {
 168                                         continue Dir
 169                                 }
 170                         }
 171                         srcFile := pathf("%s/%s", src, name)
 172                         dstFile := pathf("%s/%s", dst, name)
 173                         text := bootstrapRewriteFile(srcFile)
 174                         writefile(text, dstFile, 0)
 175                 }
 176         }
 177
 178         // Set up environment for invoking Go 1.4 go command.
 179         // GOROOT points at Go 1.4 GOROOT,
 180         // GOPATH points at our bootstrap workspace,
 181         // GOBIN is empty, so that binaries are installed to GOPATH/bin,
 182         // and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
 183         // so that Go 1.4 builds whatever kind of binary it knows how to build.
 184         // Restore GOROOT, GOPATH, and GOBIN when done.
 185         // Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
 186         // because setup will take care of those when bootstrapBuildTools returns.
 187
 188         defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
 189         os.Setenv("GOROOT", goroot_bootstrap)
 190
 191         defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
 192         os.Setenv("GOPATH", workspace)
 193
 194         defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
 195         os.Setenv("GOBIN", "")
 196
 197         os.Setenv("GOOS", "")
 198         os.Setenv("GOHOSTOS", "")
 199         os.Setenv("GOARCH", "")
 200         os.Setenv("GOHOSTARCH", "")
 201
 202         // Run Go 1.4 to build binaries. Use -gcflags=-l to disable inlining to
 203         // workaround bugs in Go 1.4's compiler. See discussion thread:
 204         // https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ
 205         // Use the math_big_pure_go build tag to disable the assembly in math/big
 206         // which may contain unsupported instructions.
 207         // Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l
 208         // only applies to the final cmd/go binary, but that's OK: if this is Go 1.10
 209         // or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler.
 210         cmd := []string{
 211                 pathf("%s/bin/go", goroot_bootstrap),
 212                 "install",
 213                 "-gcflags=-l",
 214                 "-tags=math_big_pure_go compiler_bootstrap",
 215         }
 216         if vflag > 0 {
 217                 cmd = append(cmd, "-v")
 218         }
 219         if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
 220                 cmd = append(cmd, "-toolexec="+tool)
 221         }
 222         cmd = append(cmd, "bootstrap/cmd/...")
 223         run(base, ShowOutput|CheckExit, cmd...)
 224
 225         // Copy binaries into tool binary directory.
 226         for _, name := range bootstrapDirs {
 227                 if !strings.HasPrefix(name, "cmd/") {
 228                         continue
 229                 }
 230                 name = name[len("cmd/"):]
 231                 if !strings.Contains(name, "/") {
 232                         copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
 233                 }
 234         }
 235
 236         if vflag > 0 {
 237                 xprintf("\n")
 238         }
 239 }
 240
 241 var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
 242
 243 // isUnneededSSARewriteFile reports whether srcFile is a
 244 // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
 245 // architecture that isn't for the current runtime.GOARCH.
 246 //
 247 // When unneeded is true archCaps is the rewrite base filename without
 248 // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
 249 func isUnneededSSARewriteFile(srcFile string) (archCaps string, unneeded bool) {
 250         if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
 251                 return "", false
 252         }
 253         fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
 254         if fileArch == "" {
 255                 return "", false
 256         }
 257         b := fileArch[0]
 258         if b == '_' || ('a' <= b && b <= 'z') {
 259                 return "", false
 260         }
 261         archCaps = fileArch
 262         fileArch = strings.ToLower(fileArch)
 263         fileArch = strings.TrimSuffix(fileArch, "splitload")
 264         if fileArch == os.Getenv("GOHOSTARCH") {
 265                 return "", false
 266         }
 267         if fileArch == strings.TrimSuffix(runtime.GOARCH, "le") {
 268                 return "", false
 269         }
 270         if fileArch == strings.TrimSuffix(os.Getenv("GOARCH"), "le") {
 271                 return "", false
 272         }
 273         return archCaps, true
 274 }
 275
 276 func bootstrapRewriteFile(srcFile string) string {
 277         // During bootstrap, generate dummy rewrite files for
 278         // irrelevant architectures. We only need to build a bootstrap
 279         // binary that works for the current runtime.GOARCH.
 280         // This saves 6+ seconds of bootstrap.
 281         if archCaps, ok := isUnneededSSARewriteFile(srcFile); ok {
 282                 return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.
 283
 284 package ssa
 285
 286 func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
 287 func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
 288 `, archCaps, archCaps)
 289         }
 290
 291         return bootstrapFixImports(srcFile)
 292 }
 293
 294 func bootstrapFixImports(srcFile string) string {
 295         lines := strings.SplitAfter(readfile(srcFile), "\n")
 296         inBlock := false
 297         for i, line := range lines {
 298                 if strings.HasPrefix(line, "import (") {
 299                         inBlock = true
 300                         continue
 301                 }
 302                 if inBlock && strings.HasPrefix(line, ")") {
 303                         inBlock = false
 304                         continue
 305                 }
 306                 if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
 307                         inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"")) {
 308                         line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
 309                         for _, dir := range bootstrapDirs {
 310                                 if strings.HasPrefix(dir, "cmd/") {
 311                                         continue
 312                                 }
 313                                 line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
 314                         }
 315                         lines[i] = line
 316                 }
 317         }
 318
 319         lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
 320
 321         return strings.Join(lines, "")
 322 }