src/cmd/dist/buildtool.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // Build toolchain using Go 1.4.
   6 //
   7 // The general strategy is to copy the source files we need into
   8 // a new GOPATH workspace, adjust import paths appropriately,
   9 // invoke the Go 1.4 go command to build those sources,
  10 // and then copy the binaries back.
  11
  12 package main
  13
  14 import (
  15         "fmt"
  16         "os"
  17         "path/filepath"
  18         "runtime"
  19         "strings"
  20 )
  21
  22 // bootstrapDirs is a list of directories holding code that must be
  23 // compiled with a Go 1.4 toolchain to produce the bootstrapTargets.
  24 // All directories in this list are relative to and must be below $GOROOT/src.
  25 //
  26 // The list has two kinds of entries: names beginning with cmd/ with
  27 // no other slashes, which are commands, and other paths, which are packages
  28 // supporting the commands. Packages in the standard library can be listed
  29 // if a newer copy needs to be substituted for the Go 1.4 copy when used
  30 // by the command packages. Paths ending with /... automatically
  31 // include all packages within subdirectories as well.
  32 // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
  33 var bootstrapDirs = []string{
  34         "cmd/asm",
  35         "cmd/asm/internal/...",
  36         "cmd/cgo",
  37         "cmd/compile",
  38         "cmd/compile/internal/...",
  39         "cmd/internal/archive",
  40         "cmd/internal/bio",
  41         "cmd/internal/codesign",
  42         "cmd/internal/dwarf",
  43         "cmd/internal/edit",
  44         "cmd/internal/gcprog",
  45         "cmd/internal/goobj",
  46         "cmd/internal/obj/...",
  47         "cmd/internal/objabi",
  48         "cmd/internal/pkgpath",
  49         "cmd/internal/src",
  50         "cmd/internal/sys",
  51         "cmd/link",
  52         "cmd/link/internal/...",
  53         "compress/flate",
  54         "compress/zlib",
  55         "container/heap",
  56         "debug/dwarf",
  57         "debug/elf",
  58         "debug/macho",
  59         "debug/pe",
  60         "go/constant",
  61         "internal/buildcfg",
  62         "internal/goexperiment",
  63         "internal/goversion",
  64         "internal/race",
  65         "internal/unsafeheader",
  66         "internal/xcoff",
  67         "math/big",
  68         "math/bits",
  69         "sort",
  70         "strconv",
  71 }
  72
  73 // File prefixes that are ignored by go/build anyway, and cause
  74 // problems with editor generated temporary files (#18931).
  75 var ignorePrefixes = []string{
  76         ".",
  77         "_",
  78         "#",
  79 }
  80
  81 // File suffixes that use build tags introduced since Go 1.4.
  82 // These must not be copied into the bootstrap build directory.
  83 // Also ignore test files.
  84 var ignoreSuffixes = []string{
  85         "_arm64.s",
  86         "_arm64.go",
  87         "_riscv64.s",
  88         "_riscv64.go",
  89         "_wasm.s",
  90         "_wasm.go",
  91         "_test.s",
  92         "_test.go",
  93 }
  94
  95 func bootstrapBuildTools() {
  96         goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
  97         if goroot_bootstrap == "" {
  98                 goroot_bootstrap = pathf("%s/go1.4", os.Getenv("HOME"))
  99         }
 100         xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
 101
 102         mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
 103         mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
 104
 105         // Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
 106         // We use a subdirectory of $GOROOT/pkg because that's the
 107         // space within $GOROOT where we store all generated objects.
 108         // We could use a temporary directory outside $GOROOT instead,
 109         // but it is easier to debug on failure if the files are in a known location.
 110         workspace := pathf("%s/pkg/bootstrap", goroot)
 111         xremoveall(workspace)
 112         xatexit(func() { xremoveall(workspace) })
 113         base := pathf("%s/src/bootstrap", workspace)
 114         xmkdirall(base)
 115
 116         // Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
 117         writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
 118         for _, dir := range bootstrapDirs {
 119                 recurse := strings.HasSuffix(dir, "/...")
 120                 dir = strings.TrimSuffix(dir, "/...")
 121                 filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 122                         if err != nil {
 123                                 fatalf("walking bootstrap dirs failed: %v: %v", path, err)
 124                         }
 125
 126                         name := filepath.Base(path)
 127                         src := pathf("%s/src/%s", goroot, path)
 128                         dst := pathf("%s/%s", base, path)
 129
 130                         if info.IsDir() {
 131                                 if !recurse && path != dir || name == "testdata" {
 132                                         return filepath.SkipDir
 133                                 }
 134
 135                                 xmkdirall(dst)
 136                                 if path == "cmd/cgo" {
 137                                         // Write to src because we need the file both for bootstrap
 138                                         // and for later in the main build.
 139                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
 140                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
 141                                 }
 142                                 return nil
 143                         }
 144
 145                         for _, pre := range ignorePrefixes {
 146                                 if strings.HasPrefix(name, pre) {
 147                                         return nil
 148                                 }
 149                         }
 150                         for _, suf := range ignoreSuffixes {
 151                                 if strings.HasSuffix(name, suf) {
 152                                         return nil
 153                                 }
 154                         }
 155
 156                         text := bootstrapRewriteFile(src)
 157                         writefile(text, dst, 0)
 158                         return nil
 159                 })
 160         }
 161
 162         // Set up environment for invoking Go 1.4 go command.
 163         // GOROOT points at Go 1.4 GOROOT,
 164         // GOPATH points at our bootstrap workspace,
 165         // GOBIN is empty, so that binaries are installed to GOPATH/bin,
 166         // and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
 167         // so that Go 1.4 builds whatever kind of binary it knows how to build.
 168         // Restore GOROOT, GOPATH, and GOBIN when done.
 169         // Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
 170         // because setup will take care of those when bootstrapBuildTools returns.
 171
 172         defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
 173         os.Setenv("GOROOT", goroot_bootstrap)
 174
 175         defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
 176         os.Setenv("GOPATH", workspace)
 177
 178         defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
 179         os.Setenv("GOBIN", "")
 180
 181         os.Setenv("GOOS", "")
 182         os.Setenv("GOHOSTOS", "")
 183         os.Setenv("GOARCH", "")
 184         os.Setenv("GOHOSTARCH", "")
 185
 186         // Run Go 1.4 to build binaries. Use -gcflags=-l to disable inlining to
 187         // workaround bugs in Go 1.4's compiler. See discussion thread:
 188         // https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ
 189         // Use the math_big_pure_go build tag to disable the assembly in math/big
 190         // which may contain unsupported instructions.
 191         // Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l
 192         // only applies to the final cmd/go binary, but that's OK: if this is Go 1.10
 193         // or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler.
 194         cmd := []string{
 195                 pathf("%s/bin/go", goroot_bootstrap),
 196                 "install",
 197                 "-gcflags=-l",
 198                 "-tags=math_big_pure_go compiler_bootstrap",
 199         }
 200         if vflag > 0 {
 201                 cmd = append(cmd, "-v")
 202         }
 203         if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
 204                 cmd = append(cmd, "-toolexec="+tool)
 205         }
 206         cmd = append(cmd, "bootstrap/cmd/...")
 207         run(base, ShowOutput|CheckExit, cmd...)
 208
 209         // Copy binaries into tool binary directory.
 210         for _, name := range bootstrapDirs {
 211                 if !strings.HasPrefix(name, "cmd/") {
 212                         continue
 213                 }
 214                 name = name[len("cmd/"):]
 215                 if !strings.Contains(name, "/") {
 216                         copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
 217                 }
 218         }
 219
 220         if vflag > 0 {
 221                 xprintf("\n")
 222         }
 223 }
 224
 225 var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
 226
 227 // isUnneededSSARewriteFile reports whether srcFile is a
 228 // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
 229 // architecture that isn't for the current runtime.GOARCH.
 230 //
 231 // When unneeded is true archCaps is the rewrite base filename without
 232 // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
 233 func isUnneededSSARewriteFile(srcFile string) (archCaps string, unneeded bool) {
 234         if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
 235                 return "", false
 236         }
 237         fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
 238         if fileArch == "" {
 239                 return "", false
 240         }
 241         b := fileArch[0]
 242         if b == '_' || ('a' <= b && b <= 'z') {
 243                 return "", false
 244         }
 245         archCaps = fileArch
 246         fileArch = strings.ToLower(fileArch)
 247         fileArch = strings.TrimSuffix(fileArch, "splitload")
 248         if fileArch == os.Getenv("GOHOSTARCH") {
 249                 return "", false
 250         }
 251         if fileArch == strings.TrimSuffix(runtime.GOARCH, "le") {
 252                 return "", false
 253         }
 254         if fileArch == strings.TrimSuffix(os.Getenv("GOARCH"), "le") {
 255                 return "", false
 256         }
 257         return archCaps, true
 258 }
 259
 260 func bootstrapRewriteFile(srcFile string) string {
 261         // During bootstrap, generate dummy rewrite files for
 262         // irrelevant architectures. We only need to build a bootstrap
 263         // binary that works for the current runtime.GOARCH.
 264         // This saves 6+ seconds of bootstrap.
 265         if archCaps, ok := isUnneededSSARewriteFile(srcFile); ok {
 266                 return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.
 267
 268 package ssa
 269
 270 func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
 271 func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
 272 `, archCaps, archCaps)
 273         }
 274
 275         return bootstrapFixImports(srcFile)
 276 }
 277
 278 func bootstrapFixImports(srcFile string) string {
 279         lines := strings.SplitAfter(readfile(srcFile), "\n")
 280         inBlock := false
 281         for i, line := range lines {
 282                 if strings.HasPrefix(line, "import (") {
 283                         inBlock = true
 284                         continue
 285                 }
 286                 if inBlock && strings.HasPrefix(line, ")") {
 287                         inBlock = false
 288                         continue
 289                 }
 290                 if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
 291                         inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"")) {
 292                         line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
 293                         // During bootstrap, must use plain os/exec.
 294                         line = strings.Replace(line, `exec "internal/execabs"`, `"os/exec"`, -1)
 295                         for _, dir := range bootstrapDirs {
 296                                 if strings.HasPrefix(dir, "cmd/") {
 297                                         continue
 298                                 }
 299                                 line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
 300                         }
 301                         lines[i] = line
 302                 }
 303         }
 304
 305         lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
 306
 307         return strings.Join(lines, "")
 308 }