src/cmd/dist/buildtool.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // Build toolchain using Go 1.4.
   6 //
   7 // The general strategy is to copy the source files we need into
   8 // a new GOPATH workspace, adjust import paths appropriately,
   9 // invoke the Go 1.4 go command to build those sources,
  10 // and then copy the binaries back.
  11
  12 package main
  13
  14 import (
  15         "fmt"
  16         "os"
  17         "path/filepath"
  18         "runtime"
  19         "strings"
  20 )
  21
  22 // bootstrapDirs is a list of directories holding code that must be
  23 // compiled with a Go 1.4 toolchain to produce the bootstrapTargets.
  24 // All directories in this list are relative to and must be below $GOROOT/src.
  25 //
  26 // The list has have two kinds of entries: names beginning with cmd/ with
  27 // no other slashes, which are commands, and other paths, which are packages
  28 // supporting the commands. Packages in the standard library can be listed
  29 // if a newer copy needs to be substituted for the Go 1.4 copy when used
  30 // by the command packages.
  31 // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
  32 var bootstrapDirs = []string{
  33         "cmd/asm",
  34         "cmd/asm/internal/arch",
  35         "cmd/asm/internal/asm",
  36         "cmd/asm/internal/flags",
  37         "cmd/asm/internal/lex",
  38         "cmd/cgo",
  39         "cmd/compile",
  40         "cmd/compile/internal/amd64",
  41         "cmd/compile/internal/base",
  42         "cmd/compile/internal/arm",
  43         "cmd/compile/internal/arm64",
  44         "cmd/compile/internal/gc",
  45         "cmd/compile/internal/ir",
  46         "cmd/compile/internal/logopt",
  47         "cmd/compile/internal/mips",
  48         "cmd/compile/internal/mips64",
  49         "cmd/compile/internal/ppc64",
  50         "cmd/compile/internal/riscv64",
  51         "cmd/compile/internal/s390x",
  52         "cmd/compile/internal/ssa",
  53         "cmd/compile/internal/syntax",
  54         "cmd/compile/internal/types",
  55         "cmd/compile/internal/x86",
  56         "cmd/compile/internal/wasm",
  57         "cmd/internal/bio",
  58         "cmd/internal/gcprog",
  59         "cmd/internal/dwarf",
  60         "cmd/internal/edit",
  61         "cmd/internal/goobj",
  62         "cmd/internal/objabi",
  63         "cmd/internal/obj",
  64         "cmd/internal/obj/arm",
  65         "cmd/internal/obj/arm64",
  66         "cmd/internal/obj/mips",
  67         "cmd/internal/obj/ppc64",
  68         "cmd/internal/obj/riscv",
  69         "cmd/internal/obj/s390x",
  70         "cmd/internal/obj/x86",
  71         "cmd/internal/obj/wasm",
  72         "cmd/internal/pkgpath",
  73         "cmd/internal/src",
  74         "cmd/internal/sys",
  75         "cmd/link",
  76         "cmd/link/internal/amd64",
  77         "cmd/compile/internal/base",
  78         "cmd/link/internal/arm",
  79         "cmd/link/internal/arm64",
  80         "cmd/link/internal/benchmark",
  81         "cmd/link/internal/ld",
  82         "cmd/link/internal/loadelf",
  83         "cmd/link/internal/loader",
  84         "cmd/link/internal/loadmacho",
  85         "cmd/link/internal/loadpe",
  86         "cmd/link/internal/loadxcoff",
  87         "cmd/link/internal/mips",
  88         "cmd/link/internal/mips64",
  89         "cmd/link/internal/ppc64",
  90         "cmd/link/internal/riscv64",
  91         "cmd/link/internal/s390x",
  92         "cmd/link/internal/sym",
  93         "cmd/link/internal/x86",
  94         "compress/flate",
  95         "compress/zlib",
  96         "cmd/link/internal/wasm",
  97         "container/heap",
  98         "debug/dwarf",
  99         "debug/elf",
 100         "debug/macho",
 101         "debug/pe",
 102         "go/constant",
 103         "internal/goversion",
 104         "internal/race",
 105         "internal/unsafeheader",
 106         "internal/xcoff",
 107         "math/big",
 108         "math/bits",
 109         "sort",
 110         "strconv",
 111 }
 112
 113 // File prefixes that are ignored by go/build anyway, and cause
 114 // problems with editor generated temporary files (#18931).
 115 var ignorePrefixes = []string{
 116         ".",
 117         "_",
 118 }
 119
 120 // File suffixes that use build tags introduced since Go 1.4.
 121 // These must not be copied into the bootstrap build directory.
 122 // Also ignore test files.
 123 var ignoreSuffixes = []string{
 124         "_arm64.s",
 125         "_arm64.go",
 126         "_riscv64.s",
 127         "_riscv64.go",
 128         "_wasm.s",
 129         "_wasm.go",
 130         "_test.s",
 131 }
 132
 133 func bootstrapBuildTools() {
 134         goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
 135         if goroot_bootstrap == "" {
 136                 goroot_bootstrap = pathf("%s/go1.4", os.Getenv("HOME"))
 137         }
 138         xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
 139
 140         mkzbootstrap(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
 141
 142         // Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
 143         // We use a subdirectory of $GOROOT/pkg because that's the
 144         // space within $GOROOT where we store all generated objects.
 145         // We could use a temporary directory outside $GOROOT instead,
 146         // but it is easier to debug on failure if the files are in a known location.
 147         workspace := pathf("%s/pkg/bootstrap", goroot)
 148         xremoveall(workspace)
 149         xatexit(func() { xremoveall(workspace) })
 150         base := pathf("%s/src/bootstrap", workspace)
 151         xmkdirall(base)
 152
 153         // Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
 154         writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
 155         for _, dir := range bootstrapDirs {
 156                 src := pathf("%s/src/%s", goroot, dir)
 157                 dst := pathf("%s/%s", base, dir)
 158                 xmkdirall(dst)
 159                 if dir == "cmd/cgo" {
 160                         // Write to src because we need the file both for bootstrap
 161                         // and for later in the main build.
 162                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
 163                 }
 164         Dir:
 165                 for _, name := range xreaddirfiles(src) {
 166                         for _, pre := range ignorePrefixes {
 167                                 if strings.HasPrefix(name, pre) {
 168                                         continue Dir
 169                                 }
 170                         }
 171                         for _, suf := range ignoreSuffixes {
 172                                 if strings.HasSuffix(name, suf) {
 173                                         continue Dir
 174                                 }
 175                         }
 176                         srcFile := pathf("%s/%s", src, name)
 177                         dstFile := pathf("%s/%s", dst, name)
 178                         text := bootstrapRewriteFile(srcFile)
 179                         writefile(text, dstFile, 0)
 180                 }
 181         }
 182
 183         // Set up environment for invoking Go 1.4 go command.
 184         // GOROOT points at Go 1.4 GOROOT,
 185         // GOPATH points at our bootstrap workspace,
 186         // GOBIN is empty, so that binaries are installed to GOPATH/bin,
 187         // and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
 188         // so that Go 1.4 builds whatever kind of binary it knows how to build.
 189         // Restore GOROOT, GOPATH, and GOBIN when done.
 190         // Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
 191         // because setup will take care of those when bootstrapBuildTools returns.
 192
 193         defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
 194         os.Setenv("GOROOT", goroot_bootstrap)
 195
 196         defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
 197         os.Setenv("GOPATH", workspace)
 198
 199         defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
 200         os.Setenv("GOBIN", "")
 201
 202         os.Setenv("GOOS", "")
 203         os.Setenv("GOHOSTOS", "")
 204         os.Setenv("GOARCH", "")
 205         os.Setenv("GOHOSTARCH", "")
 206
 207         // Run Go 1.4 to build binaries. Use -gcflags=-l to disable inlining to
 208         // workaround bugs in Go 1.4's compiler. See discussion thread:
 209         // https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ
 210         // Use the math_big_pure_go build tag to disable the assembly in math/big
 211         // which may contain unsupported instructions.
 212         // Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l
 213         // only applies to the final cmd/go binary, but that's OK: if this is Go 1.10
 214         // or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler.
 215         cmd := []string{
 216                 pathf("%s/bin/go", goroot_bootstrap),
 217                 "install",
 218                 "-gcflags=-l",
 219                 "-tags=math_big_pure_go compiler_bootstrap",
 220         }
 221         if vflag > 0 {
 222                 cmd = append(cmd, "-v")
 223         }
 224         if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
 225                 cmd = append(cmd, "-toolexec="+tool)
 226         }
 227         cmd = append(cmd, "bootstrap/cmd/...")
 228         run(base, ShowOutput|CheckExit, cmd...)
 229
 230         // Copy binaries into tool binary directory.
 231         for _, name := range bootstrapDirs {
 232                 if !strings.HasPrefix(name, "cmd/") {
 233                         continue
 234                 }
 235                 name = name[len("cmd/"):]
 236                 if !strings.Contains(name, "/") {
 237                         copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
 238                 }
 239         }
 240
 241         if vflag > 0 {
 242                 xprintf("\n")
 243         }
 244 }
 245
 246 var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
 247
 248 // isUnneededSSARewriteFile reports whether srcFile is a
 249 // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
 250 // architecture that isn't for the current runtime.GOARCH.
 251 //
 252 // When unneeded is true archCaps is the rewrite base filename without
 253 // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
 254 func isUnneededSSARewriteFile(srcFile string) (archCaps string, unneeded bool) {
 255         if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
 256                 return "", false
 257         }
 258         fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
 259         if fileArch == "" {
 260                 return "", false
 261         }
 262         b := fileArch[0]
 263         if b == '_' || ('a' <= b && b <= 'z') {
 264                 return "", false
 265         }
 266         archCaps = fileArch
 267         fileArch = strings.ToLower(fileArch)
 268         fileArch = strings.TrimSuffix(fileArch, "splitload")
 269         if fileArch == os.Getenv("GOHOSTARCH") {
 270                 return "", false
 271         }
 272         if fileArch == strings.TrimSuffix(runtime.GOARCH, "le") {
 273                 return "", false
 274         }
 275         if fileArch == strings.TrimSuffix(os.Getenv("GOARCH"), "le") {
 276                 return "", false
 277         }
 278         return archCaps, true
 279 }
 280
 281 func bootstrapRewriteFile(srcFile string) string {
 282         // During bootstrap, generate dummy rewrite files for
 283         // irrelevant architectures. We only need to build a bootstrap
 284         // binary that works for the current runtime.GOARCH.
 285         // This saves 6+ seconds of bootstrap.
 286         if archCaps, ok := isUnneededSSARewriteFile(srcFile); ok {
 287                 return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.
 288
 289 package ssa
 290
 291 func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
 292 func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
 293 `, archCaps, archCaps)
 294         }
 295
 296         return bootstrapFixImports(srcFile)
 297 }
 298
 299 func bootstrapFixImports(srcFile string) string {
 300         lines := strings.SplitAfter(readfile(srcFile), "\n")
 301         inBlock := false
 302         for i, line := range lines {
 303                 if strings.HasPrefix(line, "import (") {
 304                         inBlock = true
 305                         continue
 306                 }
 307                 if inBlock && strings.HasPrefix(line, ")") {
 308                         inBlock = false
 309                         continue
 310                 }
 311                 if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
 312                         inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"")) {
 313                         line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
 314                         for _, dir := range bootstrapDirs {
 315                                 if strings.HasPrefix(dir, "cmd/") {
 316                                         continue
 317                                 }
 318                                 line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
 319                         }
 320                         lines[i] = line
 321                 }
 322         }
 323
 324         lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
 325
 326         return strings.Join(lines, "")
 327 }