src/cmd/dist/buildtool.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // Build toolchain using Go bootstrap version.
   6 //
   7 // The general strategy is to copy the source files we need into
   8 // a new GOPATH workspace, adjust import paths appropriately,
   9 // invoke the Go bootstrap toolchains go command to build those sources,
  10 // and then copy the binaries back.
  11
  12 package main
  13
  14 import (
  15         "fmt"
  16         "os"
  17         "path/filepath"
  18         "regexp"
  19         "strings"
  20 )
  21
  22 // bootstrapDirs is a list of directories holding code that must be
  23 // compiled with the Go bootstrap toolchain to produce the bootstrapTargets.
  24 // All directories in this list are relative to and must be below $GOROOT/src.
  25 //
  26 // The list has two kinds of entries: names beginning with cmd/ with
  27 // no other slashes, which are commands, and other paths, which are packages
  28 // supporting the commands. Packages in the standard library can be listed
  29 // if a newer copy needs to be substituted for the Go bootstrap copy when used
  30 // by the command packages. Paths ending with /... automatically
  31 // include all packages within subdirectories as well.
  32 // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
  33 var bootstrapDirs = []string{
  34         "cmd/asm",
  35         "cmd/asm/internal/...",
  36         "cmd/cgo",
  37         "cmd/compile",
  38         "cmd/compile/internal/...",
  39         "cmd/internal/archive",
  40         "cmd/internal/bio",
  41         "cmd/internal/codesign",
  42         "cmd/internal/dwarf",
  43         "cmd/internal/edit",
  44         "cmd/internal/gcprog",
  45         "cmd/internal/goobj",
  46         "cmd/internal/notsha256",
  47         "cmd/internal/obj/...",
  48         "cmd/internal/objabi",
  49         "cmd/internal/pkgpath",
  50         "cmd/internal/quoted",
  51         "cmd/internal/src",
  52         "cmd/internal/sys",
  53         "cmd/link",
  54         "cmd/link/internal/...",
  55         "compress/flate",
  56         "compress/zlib",
  57         "container/heap",
  58         "debug/dwarf",
  59         "debug/elf",
  60         "debug/macho",
  61         "debug/pe",
  62         "go/build/constraint",
  63         "go/constant",
  64         "internal/abi",
  65         "internal/coverage",
  66         "internal/buildcfg",
  67         "internal/goarch",
  68         "internal/godebugs",
  69         "internal/goexperiment",
  70         "internal/goroot",
  71         "internal/goversion",
  72         "internal/pkgbits",
  73         "internal/platform",
  74         "internal/profile",
  75         "internal/race",
  76         "internal/saferio",
  77         "internal/syscall/unix",
  78         "internal/types/errors",
  79         "internal/unsafeheader",
  80         "internal/xcoff",
  81         "internal/zstd",
  82         "math/big",
  83         "math/bits",
  84         "sort",
  85         "strconv",
  86 }
  87
  88 // File prefixes that are ignored by go/build anyway, and cause
  89 // problems with editor generated temporary files (#18931).
  90 var ignorePrefixes = []string{
  91         ".",
  92         "_",
  93         "#",
  94 }
  95
  96 // File suffixes that use build tags introduced since Go 1.17.
  97 // These must not be copied into the bootstrap build directory.
  98 // Also ignore test files.
  99 var ignoreSuffixes = []string{
 100         "_test.s",
 101         "_test.go",
 102 }
 103
 104 var tryDirs = []string{
 105         "sdk/go1.17",
 106         "go1.17",
 107 }
 108
 109 func bootstrapBuildTools() {
 110         goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
 111         if goroot_bootstrap == "" {
 112                 home := os.Getenv("HOME")
 113                 goroot_bootstrap = pathf("%s/go1.4", home)
 114                 for _, d := range tryDirs {
 115                         if p := pathf("%s/%s", home, d); isdir(p) {
 116                                 goroot_bootstrap = p
 117                         }
 118                 }
 119         }
 120         xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
 121
 122         mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
 123         mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
 124
 125         // Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
 126         // We use a subdirectory of $GOROOT/pkg because that's the
 127         // space within $GOROOT where we store all generated objects.
 128         // We could use a temporary directory outside $GOROOT instead,
 129         // but it is easier to debug on failure if the files are in a known location.
 130         workspace := pathf("%s/pkg/bootstrap", goroot)
 131         xremoveall(workspace)
 132         xatexit(func() { xremoveall(workspace) })
 133         base := pathf("%s/src/bootstrap", workspace)
 134         xmkdirall(base)
 135
 136         // Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
 137         writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
 138         for _, dir := range bootstrapDirs {
 139                 recurse := strings.HasSuffix(dir, "/...")
 140                 dir = strings.TrimSuffix(dir, "/...")
 141                 filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 142                         if err != nil {
 143                                 fatalf("walking bootstrap dirs failed: %v: %v", path, err)
 144                         }
 145
 146                         name := filepath.Base(path)
 147                         src := pathf("%s/src/%s", goroot, path)
 148                         dst := pathf("%s/%s", base, path)
 149
 150                         if info.IsDir() {
 151                                 if !recurse && path != dir || name == "testdata" {
 152                                         return filepath.SkipDir
 153                                 }
 154
 155                                 xmkdirall(dst)
 156                                 if path == "cmd/cgo" {
 157                                         // Write to src because we need the file both for bootstrap
 158                                         // and for later in the main build.
 159                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
 160                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
 161                                 }
 162                                 return nil
 163                         }
 164
 165                         for _, pre := range ignorePrefixes {
 166                                 if strings.HasPrefix(name, pre) {
 167                                         return nil
 168                                 }
 169                         }
 170                         for _, suf := range ignoreSuffixes {
 171                                 if strings.HasSuffix(name, suf) {
 172                                         return nil
 173                                 }
 174                         }
 175
 176                         text := bootstrapRewriteFile(src)
 177                         writefile(text, dst, 0)
 178                         return nil
 179                 })
 180         }
 181
 182         // Set up environment for invoking Go bootstrap toolchains go command.
 183         // GOROOT points at Go bootstrap GOROOT,
 184         // GOPATH points at our bootstrap workspace,
 185         // GOBIN is empty, so that binaries are installed to GOPATH/bin,
 186         // and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
 187         // so that Go bootstrap toolchain builds whatever kind of binary it knows how to build.
 188         // Restore GOROOT, GOPATH, and GOBIN when done.
 189         // Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
 190         // because setup will take care of those when bootstrapBuildTools returns.
 191
 192         defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
 193         os.Setenv("GOROOT", goroot_bootstrap)
 194
 195         defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
 196         os.Setenv("GOPATH", workspace)
 197
 198         defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
 199         os.Setenv("GOBIN", "")
 200
 201         os.Setenv("GOOS", "")
 202         os.Setenv("GOHOSTOS", "")
 203         os.Setenv("GOARCH", "")
 204         os.Setenv("GOHOSTARCH", "")
 205
 206         // Run Go bootstrap to build binaries.
 207         // Use the math_big_pure_go build tag to disable the assembly in math/big
 208         // which may contain unsupported instructions.
 209         // Use the purego build tag to disable other assembly code,
 210         // such as in cmd/internal/notsha256.
 211         cmd := []string{
 212                 pathf("%s/bin/go", goroot_bootstrap),
 213                 "install",
 214                 "-tags=math_big_pure_go compiler_bootstrap purego",
 215         }
 216         if vflag > 0 {
 217                 cmd = append(cmd, "-v")
 218         }
 219         if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
 220                 cmd = append(cmd, "-toolexec="+tool)
 221         }
 222         cmd = append(cmd, "bootstrap/cmd/...")
 223         run(base, ShowOutput|CheckExit, cmd...)
 224
 225         // Copy binaries into tool binary directory.
 226         for _, name := range bootstrapDirs {
 227                 if !strings.HasPrefix(name, "cmd/") {
 228                         continue
 229                 }
 230                 name = name[len("cmd/"):]
 231                 if !strings.Contains(name, "/") {
 232                         copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
 233                 }
 234         }
 235
 236         if vflag > 0 {
 237                 xprintf("\n")
 238         }
 239 }
 240
 241 var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
 242
 243 // isUnneededSSARewriteFile reports whether srcFile is a
 244 // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
 245 // architecture that isn't for the given GOARCH.
 246 //
 247 // When unneeded is true archCaps is the rewrite base filename without
 248 // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
 249 func isUnneededSSARewriteFile(srcFile, goArch string) (archCaps string, unneeded bool) {
 250         if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
 251                 return "", false
 252         }
 253         fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
 254         if fileArch == "" {
 255                 return "", false
 256         }
 257         b := fileArch[0]
 258         if b == '_' || ('a' <= b && b <= 'z') {
 259                 return "", false
 260         }
 261         archCaps = fileArch
 262         fileArch = strings.ToLower(fileArch)
 263         fileArch = strings.TrimSuffix(fileArch, "splitload")
 264         fileArch = strings.TrimSuffix(fileArch, "latelower")
 265         if fileArch == goArch {
 266                 return "", false
 267         }
 268         if fileArch == strings.TrimSuffix(goArch, "le") {
 269                 return "", false
 270         }
 271         return archCaps, true
 272 }
 273
 274 func bootstrapRewriteFile(srcFile string) string {
 275         // During bootstrap, generate dummy rewrite files for
 276         // irrelevant architectures. We only need to build a bootstrap
 277         // binary that works for the current gohostarch.
 278         // This saves 6+ seconds of bootstrap.
 279         if archCaps, ok := isUnneededSSARewriteFile(srcFile, gohostarch); ok {
 280                 return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.
 281
 282 package ssa
 283
 284 func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
 285 func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
 286 `, archCaps, archCaps)
 287         }
 288
 289         return bootstrapFixImports(srcFile)
 290 }
 291
 292 func bootstrapFixImports(srcFile string) string {
 293         text := readfile(srcFile)
 294         if !strings.Contains(srcFile, "/cmd/") && !strings.Contains(srcFile, `\cmd\`) {
 295                 text = regexp.MustCompile(`\bany\b`).ReplaceAllString(text, "interface{}")
 296         }
 297         lines := strings.SplitAfter(text, "\n")
 298         inBlock := false
 299         for i, line := range lines {
 300                 if strings.HasPrefix(line, "import (") {
 301                         inBlock = true
 302                         continue
 303                 }
 304                 if inBlock && strings.HasPrefix(line, ")") {
 305                         inBlock = false
 306                         continue
 307                 }
 308                 if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
 309                         inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"")) {
 310                         line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
 311                         for _, dir := range bootstrapDirs {
 312                                 if strings.HasPrefix(dir, "cmd/") {
 313                                         continue
 314                                 }
 315                                 line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
 316                         }
 317                         lines[i] = line
 318                 }
 319         }
 320
 321         lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
 322
 323         return strings.Join(lines, "")
 324 }