src/cmd/dist/buildtool.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // Build toolchain using Go 1.4.
   6 //
   7 // The general strategy is to copy the source files we need into
   8 // a new GOPATH workspace, adjust import paths appropriately,
   9 // invoke the Go 1.4 go command to build those sources,
  10 // and then copy the binaries back.
  11
  12 package main
  13
  14 import (
  15         "fmt"
  16         "os"
  17         "path/filepath"
  18         "runtime"
  19         "strings"
  20 )
  21
  22 // bootstrapDirs is a list of directories holding code that must be
  23 // compiled with a Go 1.4 toolchain to produce the bootstrapTargets.
  24 // All directories in this list are relative to and must be below $GOROOT/src.
  25 //
  26 // The list has two kinds of entries: names beginning with cmd/ with
  27 // no other slashes, which are commands, and other paths, which are packages
  28 // supporting the commands. Packages in the standard library can be listed
  29 // if a newer copy needs to be substituted for the Go 1.4 copy when used
  30 // by the command packages. Paths ending with /... automatically
  31 // include all packages within subdirectories as well.
  32 // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
  33 var bootstrapDirs = []string{
  34         "cmd/asm",
  35         "cmd/asm/internal/...",
  36         "cmd/cgo",
  37         "cmd/compile",
  38         "cmd/compile/internal/...",
  39         "cmd/internal/archive",
  40         "cmd/internal/bio",
  41         "cmd/internal/codesign",
  42         "cmd/internal/dwarf",
  43         "cmd/internal/edit",
  44         "cmd/internal/gcprog",
  45         "cmd/internal/goobj",
  46         "cmd/internal/obj/...",
  47         "cmd/internal/objabi",
  48         "cmd/internal/pkgpath",
  49         "cmd/internal/quoted",
  50         "cmd/internal/src",
  51         "cmd/internal/sys",
  52         "cmd/link",
  53         "cmd/link/internal/...",
  54         "compress/flate",
  55         "compress/zlib",
  56         "container/heap",
  57         "debug/dwarf",
  58         "debug/elf",
  59         "debug/macho",
  60         "debug/pe",
  61         "go/constant",
  62         "internal/buildcfg",
  63         "internal/goexperiment",
  64         "internal/goversion",
  65         "internal/race",
  66         "internal/unsafeheader",
  67         "internal/xcoff",
  68         "math/big",
  69         "math/bits",
  70         "sort",
  71         "strconv",
  72 }
  73
  74 // File prefixes that are ignored by go/build anyway, and cause
  75 // problems with editor generated temporary files (#18931).
  76 var ignorePrefixes = []string{
  77         ".",
  78         "_",
  79         "#",
  80 }
  81
  82 // File suffixes that use build tags introduced since Go 1.4.
  83 // These must not be copied into the bootstrap build directory.
  84 // Also ignore test files.
  85 var ignoreSuffixes = []string{
  86         "_arm64.s",
  87         "_arm64.go",
  88         "_riscv64.s",
  89         "_riscv64.go",
  90         "_wasm.s",
  91         "_wasm.go",
  92         "_test.s",
  93         "_test.go",
  94 }
  95
  96 func bootstrapBuildTools() {
  97         goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
  98         if goroot_bootstrap == "" {
  99                 goroot_bootstrap = pathf("%s/go1.4", os.Getenv("HOME"))
 100         }
 101         xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
 102
 103         mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
 104         mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
 105
 106         // Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
 107         // We use a subdirectory of $GOROOT/pkg because that's the
 108         // space within $GOROOT where we store all generated objects.
 109         // We could use a temporary directory outside $GOROOT instead,
 110         // but it is easier to debug on failure if the files are in a known location.
 111         workspace := pathf("%s/pkg/bootstrap", goroot)
 112         xremoveall(workspace)
 113         xatexit(func() { xremoveall(workspace) })
 114         base := pathf("%s/src/bootstrap", workspace)
 115         xmkdirall(base)
 116
 117         // Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
 118         writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
 119         for _, dir := range bootstrapDirs {
 120                 recurse := strings.HasSuffix(dir, "/...")
 121                 dir = strings.TrimSuffix(dir, "/...")
 122                 filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 123                         if err != nil {
 124                                 fatalf("walking bootstrap dirs failed: %v: %v", path, err)
 125                         }
 126
 127                         name := filepath.Base(path)
 128                         src := pathf("%s/src/%s", goroot, path)
 129                         dst := pathf("%s/%s", base, path)
 130
 131                         if info.IsDir() {
 132                                 if !recurse && path != dir || name == "testdata" {
 133                                         return filepath.SkipDir
 134                                 }
 135
 136                                 xmkdirall(dst)
 137                                 if path == "cmd/cgo" {
 138                                         // Write to src because we need the file both for bootstrap
 139                                         // and for later in the main build.
 140                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
 141                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
 142                                 }
 143                                 return nil
 144                         }
 145
 146                         for _, pre := range ignorePrefixes {
 147                                 if strings.HasPrefix(name, pre) {
 148                                         return nil
 149                                 }
 150                         }
 151                         for _, suf := range ignoreSuffixes {
 152                                 if strings.HasSuffix(name, suf) {
 153                                         return nil
 154                                 }
 155                         }
 156
 157                         text := bootstrapRewriteFile(src)
 158                         writefile(text, dst, 0)
 159                         return nil
 160                 })
 161         }
 162
 163         // Set up environment for invoking Go 1.4 go command.
 164         // GOROOT points at Go 1.4 GOROOT,
 165         // GOPATH points at our bootstrap workspace,
 166         // GOBIN is empty, so that binaries are installed to GOPATH/bin,
 167         // and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
 168         // so that Go 1.4 builds whatever kind of binary it knows how to build.
 169         // Restore GOROOT, GOPATH, and GOBIN when done.
 170         // Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
 171         // because setup will take care of those when bootstrapBuildTools returns.
 172
 173         defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
 174         os.Setenv("GOROOT", goroot_bootstrap)
 175
 176         defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
 177         os.Setenv("GOPATH", workspace)
 178
 179         defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
 180         os.Setenv("GOBIN", "")
 181
 182         os.Setenv("GOOS", "")
 183         os.Setenv("GOHOSTOS", "")
 184         os.Setenv("GOARCH", "")
 185         os.Setenv("GOHOSTARCH", "")
 186
 187         // Run Go 1.4 to build binaries. Use -gcflags=-l to disable inlining to
 188         // workaround bugs in Go 1.4's compiler. See discussion thread:
 189         // https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ
 190         // Use the math_big_pure_go build tag to disable the assembly in math/big
 191         // which may contain unsupported instructions.
 192         // Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l
 193         // only applies to the final cmd/go binary, but that's OK: if this is Go 1.10
 194         // or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler.
 195         cmd := []string{
 196                 pathf("%s/bin/go", goroot_bootstrap),
 197                 "install",
 198                 "-gcflags=-l",
 199                 "-tags=math_big_pure_go compiler_bootstrap",
 200         }
 201         if vflag > 0 {
 202                 cmd = append(cmd, "-v")
 203         }
 204         if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
 205                 cmd = append(cmd, "-toolexec="+tool)
 206         }
 207         cmd = append(cmd, "bootstrap/cmd/...")
 208         run(base, ShowOutput|CheckExit, cmd...)
 209
 210         // Copy binaries into tool binary directory.
 211         for _, name := range bootstrapDirs {
 212                 if !strings.HasPrefix(name, "cmd/") {
 213                         continue
 214                 }
 215                 name = name[len("cmd/"):]
 216                 if !strings.Contains(name, "/") {
 217                         copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
 218                 }
 219         }
 220
 221         if vflag > 0 {
 222                 xprintf("\n")
 223         }
 224 }
 225
 226 var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
 227
 228 // isUnneededSSARewriteFile reports whether srcFile is a
 229 // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
 230 // architecture that isn't for the current runtime.GOARCH.
 231 //
 232 // When unneeded is true archCaps is the rewrite base filename without
 233 // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
 234 func isUnneededSSARewriteFile(srcFile string) (archCaps string, unneeded bool) {
 235         if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
 236                 return "", false
 237         }
 238         fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
 239         if fileArch == "" {
 240                 return "", false
 241         }
 242         b := fileArch[0]
 243         if b == '_' || ('a' <= b && b <= 'z') {
 244                 return "", false
 245         }
 246         archCaps = fileArch
 247         fileArch = strings.ToLower(fileArch)
 248         fileArch = strings.TrimSuffix(fileArch, "splitload")
 249         if fileArch == os.Getenv("GOHOSTARCH") {
 250                 return "", false
 251         }
 252         if fileArch == strings.TrimSuffix(runtime.GOARCH, "le") {
 253                 return "", false
 254         }
 255         if fileArch == strings.TrimSuffix(os.Getenv("GOARCH"), "le") {
 256                 return "", false
 257         }
 258         return archCaps, true
 259 }
 260
 261 func bootstrapRewriteFile(srcFile string) string {
 262         // During bootstrap, generate dummy rewrite files for
 263         // irrelevant architectures. We only need to build a bootstrap
 264         // binary that works for the current runtime.GOARCH.
 265         // This saves 6+ seconds of bootstrap.
 266         if archCaps, ok := isUnneededSSARewriteFile(srcFile); ok {
 267                 return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.
 268
 269 package ssa
 270
 271 func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
 272 func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
 273 `, archCaps, archCaps)
 274         }
 275
 276         return bootstrapFixImports(srcFile)
 277 }
 278
 279 func bootstrapFixImports(srcFile string) string {
 280         lines := strings.SplitAfter(readfile(srcFile), "\n")
 281         inBlock := false
 282         for i, line := range lines {
 283                 if strings.HasPrefix(line, "import (") {
 284                         inBlock = true
 285                         continue
 286                 }
 287                 if inBlock && strings.HasPrefix(line, ")") {
 288                         inBlock = false
 289                         continue
 290                 }
 291                 if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
 292                         inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"")) {
 293                         line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
 294                         // During bootstrap, must use plain os/exec.
 295                         line = strings.Replace(line, `exec "internal/execabs"`, `"os/exec"`, -1)
 296                         for _, dir := range bootstrapDirs {
 297                                 if strings.HasPrefix(dir, "cmd/") {
 298                                         continue
 299                                 }
 300                                 line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
 301                         }
 302                         lines[i] = line
 303                 }
 304         }
 305
 306         lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
 307
 308         return strings.Join(lines, "")
 309 }