src/cmd/dist/buildtool.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // Build toolchain using Go 1.4.
   6 //
   7 // The general strategy is to copy the source files we need into
   8 // a new GOPATH workspace, adjust import paths appropriately,
   9 // invoke the Go 1.4 go command to build those sources,
  10 // and then copy the binaries back.
  11
  12 package main
  13
  14 import (
  15         "fmt"
  16         "os"
  17         "path/filepath"
  18         "runtime"
  19         "strings"
  20 )
  21
  22 // bootstrapDirs is a list of directories holding code that must be
  23 // compiled with a Go 1.4 toolchain to produce the bootstrapTargets.
  24 // All directories in this list are relative to and must be below $GOROOT/src.
  25 //
  26 // The list has have two kinds of entries: names beginning with cmd/ with
  27 // no other slashes, which are commands, and other paths, which are packages
  28 // supporting the commands. Packages in the standard library can be listed
  29 // if a newer copy needs to be substituted for the Go 1.4 copy when used
  30 // by the command packages.
  31 // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
  32 var bootstrapDirs = []string{
  33         "cmd/asm",
  34         "cmd/asm/internal/arch",
  35         "cmd/asm/internal/asm",
  36         "cmd/asm/internal/flags",
  37         "cmd/asm/internal/lex",
  38         "cmd/cgo",
  39         "cmd/compile",
  40         "cmd/compile/internal/amd64",
  41         "cmd/compile/internal/arm",
  42         "cmd/compile/internal/arm64",
  43         "cmd/compile/internal/gc",
  44         "cmd/compile/internal/logopt",
  45         "cmd/compile/internal/mips",
  46         "cmd/compile/internal/mips64",
  47         "cmd/compile/internal/ppc64",
  48         "cmd/compile/internal/riscv64",
  49         "cmd/compile/internal/s390x",
  50         "cmd/compile/internal/ssa",
  51         "cmd/compile/internal/syntax",
  52         "cmd/compile/internal/types",
  53         "cmd/compile/internal/x86",
  54         "cmd/compile/internal/wasm",
  55         "cmd/internal/bio",
  56         "cmd/internal/gcprog",
  57         "cmd/internal/dwarf",
  58         "cmd/internal/edit",
  59         "cmd/internal/goobj",
  60         "cmd/internal/objabi",
  61         "cmd/internal/obj",
  62         "cmd/internal/obj/arm",
  63         "cmd/internal/obj/arm64",
  64         "cmd/internal/obj/mips",
  65         "cmd/internal/obj/ppc64",
  66         "cmd/internal/obj/riscv",
  67         "cmd/internal/obj/s390x",
  68         "cmd/internal/obj/x86",
  69         "cmd/internal/obj/wasm",
  70         "cmd/internal/pkgpath",
  71         "cmd/internal/src",
  72         "cmd/internal/sys",
  73         "cmd/link",
  74         "cmd/link/internal/amd64",
  75         "cmd/link/internal/arm",
  76         "cmd/link/internal/arm64",
  77         "cmd/link/internal/benchmark",
  78         "cmd/link/internal/ld",
  79         "cmd/link/internal/loadelf",
  80         "cmd/link/internal/loader",
  81         "cmd/link/internal/loadmacho",
  82         "cmd/link/internal/loadpe",
  83         "cmd/link/internal/loadxcoff",
  84         "cmd/link/internal/mips",
  85         "cmd/link/internal/mips64",
  86         "cmd/link/internal/ppc64",
  87         "cmd/link/internal/riscv64",
  88         "cmd/link/internal/s390x",
  89         "cmd/link/internal/sym",
  90         "cmd/link/internal/x86",
  91         "compress/flate",
  92         "compress/zlib",
  93         "cmd/link/internal/wasm",
  94         "container/heap",
  95         "debug/dwarf",
  96         "debug/elf",
  97         "debug/macho",
  98         "debug/pe",
  99         "go/constant",
 100         "internal/goversion",
 101         "internal/race",
 102         "internal/unsafeheader",
 103         "internal/xcoff",
 104         "math/big",
 105         "math/bits",
 106         "sort",
 107         "strconv",
 108 }
 109
 110 // File prefixes that are ignored by go/build anyway, and cause
 111 // problems with editor generated temporary files (#18931).
 112 var ignorePrefixes = []string{
 113         ".",
 114         "_",
 115 }
 116
 117 // File suffixes that use build tags introduced since Go 1.4.
 118 // These must not be copied into the bootstrap build directory.
 119 // Also ignore test files.
 120 var ignoreSuffixes = []string{
 121         "_arm64.s",
 122         "_arm64.go",
 123         "_riscv64.s",
 124         "_riscv64.go",
 125         "_wasm.s",
 126         "_wasm.go",
 127         "_test.s",
 128 }
 129
 130 func bootstrapBuildTools() {
 131         goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
 132         if goroot_bootstrap == "" {
 133                 goroot_bootstrap = pathf("%s/go1.4", os.Getenv("HOME"))
 134         }
 135         xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
 136
 137         mkzbootstrap(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
 138
 139         // Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
 140         // We use a subdirectory of $GOROOT/pkg because that's the
 141         // space within $GOROOT where we store all generated objects.
 142         // We could use a temporary directory outside $GOROOT instead,
 143         // but it is easier to debug on failure if the files are in a known location.
 144         workspace := pathf("%s/pkg/bootstrap", goroot)
 145         xremoveall(workspace)
 146         xatexit(func() { xremoveall(workspace) })
 147         base := pathf("%s/src/bootstrap", workspace)
 148         xmkdirall(base)
 149
 150         // Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
 151         writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
 152         for _, dir := range bootstrapDirs {
 153                 src := pathf("%s/src/%s", goroot, dir)
 154                 dst := pathf("%s/%s", base, dir)
 155                 xmkdirall(dst)
 156                 if dir == "cmd/cgo" {
 157                         // Write to src because we need the file both for bootstrap
 158                         // and for later in the main build.
 159                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
 160                 }
 161         Dir:
 162                 for _, name := range xreaddirfiles(src) {
 163                         for _, pre := range ignorePrefixes {
 164                                 if strings.HasPrefix(name, pre) {
 165                                         continue Dir
 166                                 }
 167                         }
 168                         for _, suf := range ignoreSuffixes {
 169                                 if strings.HasSuffix(name, suf) {
 170                                         continue Dir
 171                                 }
 172                         }
 173                         srcFile := pathf("%s/%s", src, name)
 174                         dstFile := pathf("%s/%s", dst, name)
 175                         text := bootstrapRewriteFile(srcFile)
 176                         writefile(text, dstFile, 0)
 177                 }
 178         }
 179
 180         // Set up environment for invoking Go 1.4 go command.
 181         // GOROOT points at Go 1.4 GOROOT,
 182         // GOPATH points at our bootstrap workspace,
 183         // GOBIN is empty, so that binaries are installed to GOPATH/bin,
 184         // and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
 185         // so that Go 1.4 builds whatever kind of binary it knows how to build.
 186         // Restore GOROOT, GOPATH, and GOBIN when done.
 187         // Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
 188         // because setup will take care of those when bootstrapBuildTools returns.
 189
 190         defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
 191         os.Setenv("GOROOT", goroot_bootstrap)
 192
 193         defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
 194         os.Setenv("GOPATH", workspace)
 195
 196         defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
 197         os.Setenv("GOBIN", "")
 198
 199         os.Setenv("GOOS", "")
 200         os.Setenv("GOHOSTOS", "")
 201         os.Setenv("GOARCH", "")
 202         os.Setenv("GOHOSTARCH", "")
 203
 204         // Run Go 1.4 to build binaries. Use -gcflags=-l to disable inlining to
 205         // workaround bugs in Go 1.4's compiler. See discussion thread:
 206         // https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ
 207         // Use the math_big_pure_go build tag to disable the assembly in math/big
 208         // which may contain unsupported instructions.
 209         // Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l
 210         // only applies to the final cmd/go binary, but that's OK: if this is Go 1.10
 211         // or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler.
 212         cmd := []string{
 213                 pathf("%s/bin/go", goroot_bootstrap),
 214                 "install",
 215                 "-gcflags=-l",
 216                 "-tags=math_big_pure_go compiler_bootstrap",
 217         }
 218         if vflag > 0 {
 219                 cmd = append(cmd, "-v")
 220         }
 221         if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
 222                 cmd = append(cmd, "-toolexec="+tool)
 223         }
 224         cmd = append(cmd, "bootstrap/cmd/...")
 225         run(base, ShowOutput|CheckExit, cmd...)
 226
 227         // Copy binaries into tool binary directory.
 228         for _, name := range bootstrapDirs {
 229                 if !strings.HasPrefix(name, "cmd/") {
 230                         continue
 231                 }
 232                 name = name[len("cmd/"):]
 233                 if !strings.Contains(name, "/") {
 234                         copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
 235                 }
 236         }
 237
 238         if vflag > 0 {
 239                 xprintf("\n")
 240         }
 241 }
 242
 243 var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
 244
 245 // isUnneededSSARewriteFile reports whether srcFile is a
 246 // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
 247 // architecture that isn't for the current runtime.GOARCH.
 248 //
 249 // When unneeded is true archCaps is the rewrite base filename without
 250 // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
 251 func isUnneededSSARewriteFile(srcFile string) (archCaps string, unneeded bool) {
 252         if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
 253                 return "", false
 254         }
 255         fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
 256         if fileArch == "" {
 257                 return "", false
 258         }
 259         b := fileArch[0]
 260         if b == '_' || ('a' <= b && b <= 'z') {
 261                 return "", false
 262         }
 263         archCaps = fileArch
 264         fileArch = strings.ToLower(fileArch)
 265         fileArch = strings.TrimSuffix(fileArch, "splitload")
 266         if fileArch == os.Getenv("GOHOSTARCH") {
 267                 return "", false
 268         }
 269         if fileArch == strings.TrimSuffix(runtime.GOARCH, "le") {
 270                 return "", false
 271         }
 272         if fileArch == strings.TrimSuffix(os.Getenv("GOARCH"), "le") {
 273                 return "", false
 274         }
 275         return archCaps, true
 276 }
 277
 278 func bootstrapRewriteFile(srcFile string) string {
 279         // During bootstrap, generate dummy rewrite files for
 280         // irrelevant architectures. We only need to build a bootstrap
 281         // binary that works for the current runtime.GOARCH.
 282         // This saves 6+ seconds of bootstrap.
 283         if archCaps, ok := isUnneededSSARewriteFile(srcFile); ok {
 284                 return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.
 285
 286 package ssa
 287
 288 func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
 289 func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
 290 `, archCaps, archCaps)
 291         }
 292
 293         return bootstrapFixImports(srcFile)
 294 }
 295
 296 func bootstrapFixImports(srcFile string) string {
 297         lines := strings.SplitAfter(readfile(srcFile), "\n")
 298         inBlock := false
 299         for i, line := range lines {
 300                 if strings.HasPrefix(line, "import (") {
 301                         inBlock = true
 302                         continue
 303                 }
 304                 if inBlock && strings.HasPrefix(line, ")") {
 305                         inBlock = false
 306                         continue
 307                 }
 308                 if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
 309                         inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"")) {
 310                         line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
 311                         for _, dir := range bootstrapDirs {
 312                                 if strings.HasPrefix(dir, "cmd/") {
 313                                         continue
 314                                 }
 315                                 line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
 316                         }
 317                         lines[i] = line
 318                 }
 319         }
 320
 321         lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
 322
 323         return strings.Join(lines, "")
 324 }