src/cmd/dist/buildtool.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // Build toolchain using Go 1.4.
   6 //
   7 // The general strategy is to copy the source files we need into
   8 // a new GOPATH workspace, adjust import paths appropriately,
   9 // invoke the Go 1.4 go command to build those sources,
  10 // and then copy the binaries back.
  11
  12 package main
  13
  14 import (
  15         "fmt"
  16         "os"
  17         "path/filepath"
  18         "runtime"
  19         "strings"
  20 )
  21
  22 // bootstrapDirs is a list of directories holding code that must be
  23 // compiled with a Go 1.4 toolchain to produce the bootstrapTargets.
  24 // All directories in this list are relative to and must be below $GOROOT/src.
  25 //
  26 // The list has two kinds of entries: names beginning with cmd/ with
  27 // no other slashes, which are commands, and other paths, which are packages
  28 // supporting the commands. Packages in the standard library can be listed
  29 // if a newer copy needs to be substituted for the Go 1.4 copy when used
  30 // by the command packages. Paths ending with /... automatically
  31 // include all packages within subdirectories as well.
  32 // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
  33 var bootstrapDirs = []string{
  34         "cmd/asm",
  35         "cmd/asm/internal/...",
  36         "cmd/cgo",
  37         "cmd/compile",
  38         "cmd/compile/internal/...",
  39         "cmd/internal/archive",
  40         "cmd/internal/bio",
  41         "cmd/internal/codesign",
  42         "cmd/internal/dwarf",
  43         "cmd/internal/edit",
  44         "cmd/internal/gcprog",
  45         "cmd/internal/goobj",
  46         "cmd/internal/obj/...",
  47         "cmd/internal/objabi",
  48         "cmd/internal/pkgpath",
  49         "cmd/internal/src",
  50         "cmd/internal/sys",
  51         "cmd/link",
  52         "cmd/link/internal/...",
  53         "compress/flate",
  54         "compress/zlib",
  55         "container/heap",
  56         "debug/dwarf",
  57         "debug/elf",
  58         "debug/macho",
  59         "debug/pe",
  60         "go/constant",
  61         "internal/goversion",
  62         "internal/race",
  63         "internal/unsafeheader",
  64         "internal/xcoff",
  65         "math/big",
  66         "math/bits",
  67         "sort",
  68         "strconv",
  69 }
  70
  71 // File prefixes that are ignored by go/build anyway, and cause
  72 // problems with editor generated temporary files (#18931).
  73 var ignorePrefixes = []string{
  74         ".",
  75         "_",
  76         "#",
  77 }
  78
  79 // File suffixes that use build tags introduced since Go 1.4.
  80 // These must not be copied into the bootstrap build directory.
  81 // Also ignore test files.
  82 var ignoreSuffixes = []string{
  83         "_arm64.s",
  84         "_arm64.go",
  85         "_riscv64.s",
  86         "_riscv64.go",
  87         "_wasm.s",
  88         "_wasm.go",
  89         "_test.s",
  90         "_test.go",
  91 }
  92
  93 func bootstrapBuildTools() {
  94         goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
  95         if goroot_bootstrap == "" {
  96                 goroot_bootstrap = pathf("%s/go1.4", os.Getenv("HOME"))
  97         }
  98         xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
  99
 100         mkzbootstrap(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
 101
 102         // Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
 103         // We use a subdirectory of $GOROOT/pkg because that's the
 104         // space within $GOROOT where we store all generated objects.
 105         // We could use a temporary directory outside $GOROOT instead,
 106         // but it is easier to debug on failure if the files are in a known location.
 107         workspace := pathf("%s/pkg/bootstrap", goroot)
 108         xremoveall(workspace)
 109         xatexit(func() { xremoveall(workspace) })
 110         base := pathf("%s/src/bootstrap", workspace)
 111         xmkdirall(base)
 112
 113         // Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
 114         writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
 115         for _, dir := range bootstrapDirs {
 116                 recurse := strings.HasSuffix(dir, "/...")
 117                 dir = strings.TrimSuffix(dir, "/...")
 118                 filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 119                         if err != nil {
 120                                 fatalf("walking bootstrap dirs failed: %v: %v", path, err)
 121                         }
 122
 123                         name := filepath.Base(path)
 124                         src := pathf("%s/src/%s", goroot, path)
 125                         dst := pathf("%s/%s", base, path)
 126
 127                         if info.IsDir() {
 128                                 if !recurse && path != dir || name == "testdata" {
 129                                         return filepath.SkipDir
 130                                 }
 131
 132                                 xmkdirall(dst)
 133                                 if path == "cmd/cgo" {
 134                                         // Write to src because we need the file both for bootstrap
 135                                         // and for later in the main build.
 136                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
 137                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
 138                                 }
 139                                 return nil
 140                         }
 141
 142                         for _, pre := range ignorePrefixes {
 143                                 if strings.HasPrefix(name, pre) {
 144                                         return nil
 145                                 }
 146                         }
 147                         for _, suf := range ignoreSuffixes {
 148                                 if strings.HasSuffix(name, suf) {
 149                                         return nil
 150                                 }
 151                         }
 152
 153                         text := bootstrapRewriteFile(src)
 154                         writefile(text, dst, 0)
 155                         return nil
 156                 })
 157         }
 158
 159         // Set up environment for invoking Go 1.4 go command.
 160         // GOROOT points at Go 1.4 GOROOT,
 161         // GOPATH points at our bootstrap workspace,
 162         // GOBIN is empty, so that binaries are installed to GOPATH/bin,
 163         // and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
 164         // so that Go 1.4 builds whatever kind of binary it knows how to build.
 165         // Restore GOROOT, GOPATH, and GOBIN when done.
 166         // Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
 167         // because setup will take care of those when bootstrapBuildTools returns.
 168
 169         defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
 170         os.Setenv("GOROOT", goroot_bootstrap)
 171
 172         defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
 173         os.Setenv("GOPATH", workspace)
 174
 175         defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
 176         os.Setenv("GOBIN", "")
 177
 178         os.Setenv("GOOS", "")
 179         os.Setenv("GOHOSTOS", "")
 180         os.Setenv("GOARCH", "")
 181         os.Setenv("GOHOSTARCH", "")
 182
 183         // Run Go 1.4 to build binaries. Use -gcflags=-l to disable inlining to
 184         // workaround bugs in Go 1.4's compiler. See discussion thread:
 185         // https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ
 186         // Use the math_big_pure_go build tag to disable the assembly in math/big
 187         // which may contain unsupported instructions.
 188         // Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l
 189         // only applies to the final cmd/go binary, but that's OK: if this is Go 1.10
 190         // or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler.
 191         cmd := []string{
 192                 pathf("%s/bin/go", goroot_bootstrap),
 193                 "install",
 194                 "-gcflags=-l",
 195                 "-tags=math_big_pure_go compiler_bootstrap",
 196         }
 197         if vflag > 0 {
 198                 cmd = append(cmd, "-v")
 199         }
 200         if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
 201                 cmd = append(cmd, "-toolexec="+tool)
 202         }
 203         cmd = append(cmd, "bootstrap/cmd/...")
 204         run(base, ShowOutput|CheckExit, cmd...)
 205
 206         // Copy binaries into tool binary directory.
 207         for _, name := range bootstrapDirs {
 208                 if !strings.HasPrefix(name, "cmd/") {
 209                         continue
 210                 }
 211                 name = name[len("cmd/"):]
 212                 if !strings.Contains(name, "/") {
 213                         copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
 214                 }
 215         }
 216
 217         if vflag > 0 {
 218                 xprintf("\n")
 219         }
 220 }
 221
 222 var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
 223
 224 // isUnneededSSARewriteFile reports whether srcFile is a
 225 // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
 226 // architecture that isn't for the current runtime.GOARCH.
 227 //
 228 // When unneeded is true archCaps is the rewrite base filename without
 229 // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
 230 func isUnneededSSARewriteFile(srcFile string) (archCaps string, unneeded bool) {
 231         if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
 232                 return "", false
 233         }
 234         fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
 235         if fileArch == "" {
 236                 return "", false
 237         }
 238         b := fileArch[0]
 239         if b == '_' || ('a' <= b && b <= 'z') {
 240                 return "", false
 241         }
 242         archCaps = fileArch
 243         fileArch = strings.ToLower(fileArch)
 244         fileArch = strings.TrimSuffix(fileArch, "splitload")
 245         if fileArch == os.Getenv("GOHOSTARCH") {
 246                 return "", false
 247         }
 248         if fileArch == strings.TrimSuffix(runtime.GOARCH, "le") {
 249                 return "", false
 250         }
 251         if fileArch == strings.TrimSuffix(os.Getenv("GOARCH"), "le") {
 252                 return "", false
 253         }
 254         return archCaps, true
 255 }
 256
 257 func bootstrapRewriteFile(srcFile string) string {
 258         // During bootstrap, generate dummy rewrite files for
 259         // irrelevant architectures. We only need to build a bootstrap
 260         // binary that works for the current runtime.GOARCH.
 261         // This saves 6+ seconds of bootstrap.
 262         if archCaps, ok := isUnneededSSARewriteFile(srcFile); ok {
 263                 return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.
 264
 265 package ssa
 266
 267 func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
 268 func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
 269 `, archCaps, archCaps)
 270         }
 271
 272         return bootstrapFixImports(srcFile)
 273 }
 274
 275 func bootstrapFixImports(srcFile string) string {
 276         lines := strings.SplitAfter(readfile(srcFile), "\n")
 277         inBlock := false
 278         for i, line := range lines {
 279                 if strings.HasPrefix(line, "import (") {
 280                         inBlock = true
 281                         continue
 282                 }
 283                 if inBlock && strings.HasPrefix(line, ")") {
 284                         inBlock = false
 285                         continue
 286                 }
 287                 if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
 288                         inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"")) {
 289                         line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
 290                         // During bootstrap, must use plain os/exec.
 291                         line = strings.Replace(line, `exec "internal/execabs"`, `"os/exec"`, -1)
 292                         for _, dir := range bootstrapDirs {
 293                                 if strings.HasPrefix(dir, "cmd/") {
 294                                         continue
 295                                 }
 296                                 line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
 297                         }
 298                         lines[i] = line
 299                 }
 300         }
 301
 302         lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
 303
 304         return strings.Join(lines, "")
 305 }