src/cmd/dist/buildtool.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // Build toolchain using Go 1.4.
   6 //
   7 // The general strategy is to copy the source files we need into
   8 // a new GOPATH workspace, adjust import paths appropriately,
   9 // invoke the Go 1.4 go command to build those sources,
  10 // and then copy the binaries back.
  11
  12 package main
  13
  14 import (
  15         "fmt"
  16         "os"
  17         "path/filepath"
  18         "runtime"
  19         "strings"
  20 )
  21
  22 // bootstrapDirs is a list of directories holding code that must be
  23 // compiled with a Go 1.4 toolchain to produce the bootstrapTargets.
  24 // All directories in this list are relative to and must be below $GOROOT/src.
  25 //
  26 // The list has have two kinds of entries: names beginning with cmd/ with
  27 // no other slashes, which are commands, and other paths, which are packages
  28 // supporting the commands. Packages in the standard library can be listed
  29 // if a newer copy needs to be substituted for the Go 1.4 copy when used
  30 // by the command packages.
  31 // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
  32 var bootstrapDirs = []string{
  33         "cmd/asm",
  34         "cmd/asm/internal/arch",
  35         "cmd/asm/internal/asm",
  36         "cmd/asm/internal/flags",
  37         "cmd/asm/internal/lex",
  38         "cmd/cgo",
  39         "cmd/compile",
  40         "cmd/compile/internal/amd64",
  41         "cmd/compile/internal/base",
  42         "cmd/compile/internal/arm",
  43         "cmd/compile/internal/arm64",
  44         "cmd/compile/internal/gc",
  45         "cmd/compile/internal/ir",
  46         "cmd/compile/internal/logopt",
  47         "cmd/compile/internal/mips",
  48         "cmd/compile/internal/mips64",
  49         "cmd/compile/internal/ppc64",
  50         "cmd/compile/internal/riscv64",
  51         "cmd/compile/internal/s390x",
  52         "cmd/compile/internal/ssa",
  53         "cmd/compile/internal/syntax",
  54         "cmd/compile/internal/types",
  55         "cmd/compile/internal/x86",
  56         "cmd/compile/internal/wasm",
  57         "cmd/internal/bio",
  58         "cmd/internal/codesign",
  59         "cmd/internal/gcprog",
  60         "cmd/internal/dwarf",
  61         "cmd/internal/edit",
  62         "cmd/internal/goobj",
  63         "cmd/internal/objabi",
  64         "cmd/internal/obj",
  65         "cmd/internal/obj/arm",
  66         "cmd/internal/obj/arm64",
  67         "cmd/internal/obj/mips",
  68         "cmd/internal/obj/ppc64",
  69         "cmd/internal/obj/riscv",
  70         "cmd/internal/obj/s390x",
  71         "cmd/internal/obj/x86",
  72         "cmd/internal/obj/wasm",
  73         "cmd/internal/pkgpath",
  74         "cmd/internal/src",
  75         "cmd/internal/sys",
  76         "cmd/link",
  77         "cmd/link/internal/amd64",
  78         "cmd/compile/internal/base",
  79         "cmd/link/internal/arm",
  80         "cmd/link/internal/arm64",
  81         "cmd/link/internal/benchmark",
  82         "cmd/link/internal/ld",
  83         "cmd/link/internal/loadelf",
  84         "cmd/link/internal/loader",
  85         "cmd/link/internal/loadmacho",
  86         "cmd/link/internal/loadpe",
  87         "cmd/link/internal/loadxcoff",
  88         "cmd/link/internal/mips",
  89         "cmd/link/internal/mips64",
  90         "cmd/link/internal/ppc64",
  91         "cmd/link/internal/riscv64",
  92         "cmd/link/internal/s390x",
  93         "cmd/link/internal/sym",
  94         "cmd/link/internal/x86",
  95         "compress/flate",
  96         "compress/zlib",
  97         "cmd/link/internal/wasm",
  98         "container/heap",
  99         "debug/dwarf",
 100         "debug/elf",
 101         "debug/macho",
 102         "debug/pe",
 103         "go/constant",
 104         "internal/goversion",
 105         "internal/race",
 106         "internal/unsafeheader",
 107         "internal/xcoff",
 108         "math/big",
 109         "math/bits",
 110         "sort",
 111         "strconv",
 112 }
 113
 114 // File prefixes that are ignored by go/build anyway, and cause
 115 // problems with editor generated temporary files (#18931).
 116 var ignorePrefixes = []string{
 117         ".",
 118         "_",
 119 }
 120
 121 // File suffixes that use build tags introduced since Go 1.4.
 122 // These must not be copied into the bootstrap build directory.
 123 // Also ignore test files.
 124 var ignoreSuffixes = []string{
 125         "_arm64.s",
 126         "_arm64.go",
 127         "_riscv64.s",
 128         "_riscv64.go",
 129         "_wasm.s",
 130         "_wasm.go",
 131         "_test.s",
 132 }
 133
 134 func bootstrapBuildTools() {
 135         goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
 136         if goroot_bootstrap == "" {
 137                 goroot_bootstrap = pathf("%s/go1.4", os.Getenv("HOME"))
 138         }
 139         xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
 140
 141         mkzbootstrap(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
 142
 143         // Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
 144         // We use a subdirectory of $GOROOT/pkg because that's the
 145         // space within $GOROOT where we store all generated objects.
 146         // We could use a temporary directory outside $GOROOT instead,
 147         // but it is easier to debug on failure if the files are in a known location.
 148         workspace := pathf("%s/pkg/bootstrap", goroot)
 149         xremoveall(workspace)
 150         xatexit(func() { xremoveall(workspace) })
 151         base := pathf("%s/src/bootstrap", workspace)
 152         xmkdirall(base)
 153
 154         // Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
 155         writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
 156         for _, dir := range bootstrapDirs {
 157                 src := pathf("%s/src/%s", goroot, dir)
 158                 dst := pathf("%s/%s", base, dir)
 159                 xmkdirall(dst)
 160                 if dir == "cmd/cgo" {
 161                         // Write to src because we need the file both for bootstrap
 162                         // and for later in the main build.
 163                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
 164                 }
 165         Dir:
 166                 for _, name := range xreaddirfiles(src) {
 167                         for _, pre := range ignorePrefixes {
 168                                 if strings.HasPrefix(name, pre) {
 169                                         continue Dir
 170                                 }
 171                         }
 172                         for _, suf := range ignoreSuffixes {
 173                                 if strings.HasSuffix(name, suf) {
 174                                         continue Dir
 175                                 }
 176                         }
 177                         srcFile := pathf("%s/%s", src, name)
 178                         dstFile := pathf("%s/%s", dst, name)
 179                         text := bootstrapRewriteFile(srcFile)
 180                         writefile(text, dstFile, 0)
 181                 }
 182         }
 183
 184         // Set up environment for invoking Go 1.4 go command.
 185         // GOROOT points at Go 1.4 GOROOT,
 186         // GOPATH points at our bootstrap workspace,
 187         // GOBIN is empty, so that binaries are installed to GOPATH/bin,
 188         // and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
 189         // so that Go 1.4 builds whatever kind of binary it knows how to build.
 190         // Restore GOROOT, GOPATH, and GOBIN when done.
 191         // Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
 192         // because setup will take care of those when bootstrapBuildTools returns.
 193
 194         defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
 195         os.Setenv("GOROOT", goroot_bootstrap)
 196
 197         defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
 198         os.Setenv("GOPATH", workspace)
 199
 200         defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
 201         os.Setenv("GOBIN", "")
 202
 203         os.Setenv("GOOS", "")
 204         os.Setenv("GOHOSTOS", "")
 205         os.Setenv("GOARCH", "")
 206         os.Setenv("GOHOSTARCH", "")
 207
 208         // Run Go 1.4 to build binaries. Use -gcflags=-l to disable inlining to
 209         // workaround bugs in Go 1.4's compiler. See discussion thread:
 210         // https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ
 211         // Use the math_big_pure_go build tag to disable the assembly in math/big
 212         // which may contain unsupported instructions.
 213         // Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l
 214         // only applies to the final cmd/go binary, but that's OK: if this is Go 1.10
 215         // or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler.
 216         cmd := []string{
 217                 pathf("%s/bin/go", goroot_bootstrap),
 218                 "install",
 219                 "-gcflags=-l",
 220                 "-tags=math_big_pure_go compiler_bootstrap",
 221         }
 222         if vflag > 0 {
 223                 cmd = append(cmd, "-v")
 224         }
 225         if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
 226                 cmd = append(cmd, "-toolexec="+tool)
 227         }
 228         cmd = append(cmd, "bootstrap/cmd/...")
 229         run(base, ShowOutput|CheckExit, cmd...)
 230
 231         // Copy binaries into tool binary directory.
 232         for _, name := range bootstrapDirs {
 233                 if !strings.HasPrefix(name, "cmd/") {
 234                         continue
 235                 }
 236                 name = name[len("cmd/"):]
 237                 if !strings.Contains(name, "/") {
 238                         copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
 239                 }
 240         }
 241
 242         if vflag > 0 {
 243                 xprintf("\n")
 244         }
 245 }
 246
 247 var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
 248
 249 // isUnneededSSARewriteFile reports whether srcFile is a
 250 // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
 251 // architecture that isn't for the current runtime.GOARCH.
 252 //
 253 // When unneeded is true archCaps is the rewrite base filename without
 254 // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
 255 func isUnneededSSARewriteFile(srcFile string) (archCaps string, unneeded bool) {
 256         if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
 257                 return "", false
 258         }
 259         fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
 260         if fileArch == "" {
 261                 return "", false
 262         }
 263         b := fileArch[0]
 264         if b == '_' || ('a' <= b && b <= 'z') {
 265                 return "", false
 266         }
 267         archCaps = fileArch
 268         fileArch = strings.ToLower(fileArch)
 269         fileArch = strings.TrimSuffix(fileArch, "splitload")
 270         if fileArch == os.Getenv("GOHOSTARCH") {
 271                 return "", false
 272         }
 273         if fileArch == strings.TrimSuffix(runtime.GOARCH, "le") {
 274                 return "", false
 275         }
 276         if fileArch == strings.TrimSuffix(os.Getenv("GOARCH"), "le") {
 277                 return "", false
 278         }
 279         return archCaps, true
 280 }
 281
 282 func bootstrapRewriteFile(srcFile string) string {
 283         // During bootstrap, generate dummy rewrite files for
 284         // irrelevant architectures. We only need to build a bootstrap
 285         // binary that works for the current runtime.GOARCH.
 286         // This saves 6+ seconds of bootstrap.
 287         if archCaps, ok := isUnneededSSARewriteFile(srcFile); ok {
 288                 return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.
 289
 290 package ssa
 291
 292 func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
 293 func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
 294 `, archCaps, archCaps)
 295         }
 296
 297         return bootstrapFixImports(srcFile)
 298 }
 299
 300 func bootstrapFixImports(srcFile string) string {
 301         lines := strings.SplitAfter(readfile(srcFile), "\n")
 302         inBlock := false
 303         for i, line := range lines {
 304                 if strings.HasPrefix(line, "import (") {
 305                         inBlock = true
 306                         continue
 307                 }
 308                 if inBlock && strings.HasPrefix(line, ")") {
 309                         inBlock = false
 310                         continue
 311                 }
 312                 if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
 313                         inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"")) {
 314                         line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
 315                         for _, dir := range bootstrapDirs {
 316                                 if strings.HasPrefix(dir, "cmd/") {
 317                                         continue
 318                                 }
 319                                 line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
 320                         }
 321                         lines[i] = line
 322                 }
 323         }
 324
 325         lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
 326
 327         return strings.Join(lines, "")
 328 }