src/cmd/dist/buildtool.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // Build toolchain using Go bootstrap version.
   6 //
   7 // The general strategy is to copy the source files we need into
   8 // a new GOPATH workspace, adjust import paths appropriately,
   9 // invoke the Go bootstrap toolchains go command to build those sources,
  10 // and then copy the binaries back.
  11
  12 package main
  13
  14 import (
  15         "fmt"
  16         "os"
  17         "path/filepath"
  18         "regexp"
  19         "strings"
  20 )
  21
  22 // bootstrapDirs is a list of directories holding code that must be
  23 // compiled with the Go bootstrap toolchain to produce the bootstrapTargets.
  24 // All directories in this list are relative to and must be below $GOROOT/src.
  25 //
  26 // The list has two kinds of entries: names beginning with cmd/ with
  27 // no other slashes, which are commands, and other paths, which are packages
  28 // supporting the commands. Packages in the standard library can be listed
  29 // if a newer copy needs to be substituted for the Go bootstrap copy when used
  30 // by the command packages. Paths ending with /... automatically
  31 // include all packages within subdirectories as well.
  32 // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
  33 var bootstrapDirs = []string{
  34         "cmd/asm",
  35         "cmd/asm/internal/...",
  36         "cmd/cgo",
  37         "cmd/compile",
  38         "cmd/compile/internal/...",
  39         "cmd/internal/archive",
  40         "cmd/internal/bio",
  41         "cmd/internal/codesign",
  42         "cmd/internal/dwarf",
  43         "cmd/internal/edit",
  44         "cmd/internal/gcprog",
  45         "cmd/internal/goobj",
  46         "cmd/internal/notsha256",
  47         "cmd/internal/obj/...",
  48         "cmd/internal/objabi",
  49         "cmd/internal/pkgpath",
  50         "cmd/internal/quoted",
  51         "cmd/internal/src",
  52         "cmd/internal/sys",
  53         "cmd/link",
  54         "cmd/link/internal/...",
  55         "compress/flate",
  56         "compress/zlib",
  57         "container/heap",
  58         "debug/dwarf",
  59         "debug/elf",
  60         "debug/macho",
  61         "debug/pe",
  62         "go/build/constraint",
  63         "go/constant",
  64         "internal/abi",
  65         "internal/coverage",
  66         "internal/bisect",
  67         "internal/buildcfg",
  68         "internal/goarch",
  69         "internal/godebugs",
  70         "internal/goexperiment",
  71         "internal/goroot",
  72         "internal/goversion",
  73         "internal/pkgbits",
  74         "internal/platform",
  75         "internal/profile",
  76         "internal/race",
  77         "internal/saferio",
  78         "internal/syscall/unix",
  79         "internal/types/errors",
  80         "internal/unsafeheader",
  81         "internal/xcoff",
  82         "internal/zstd",
  83         "math/big",
  84         "math/bits",
  85         "sort",
  86         "strconv",
  87 }
  88
  89 // File prefixes that are ignored by go/build anyway, and cause
  90 // problems with editor generated temporary files (#18931).
  91 var ignorePrefixes = []string{
  92         ".",
  93         "_",
  94         "#",
  95 }
  96
  97 // File suffixes that use build tags introduced since Go 1.17.
  98 // These must not be copied into the bootstrap build directory.
  99 // Also ignore test files.
 100 var ignoreSuffixes = []string{
 101         "_test.s",
 102         "_test.go",
 103         // Skip PGO profile. No need to build toolchain1 compiler
 104         // with PGO. And as it is not a text file the import path
 105         // rewrite will break it.
 106         ".pgo",
 107 }
 108
 109 var tryDirs = []string{
 110         "sdk/go1.17",
 111         "go1.17",
 112 }
 113
 114 func bootstrapBuildTools() {
 115         goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
 116         if goroot_bootstrap == "" {
 117                 home := os.Getenv("HOME")
 118                 goroot_bootstrap = pathf("%s/go1.4", home)
 119                 for _, d := range tryDirs {
 120                         if p := pathf("%s/%s", home, d); isdir(p) {
 121                                 goroot_bootstrap = p
 122                         }
 123                 }
 124         }
 125         xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
 126
 127         mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
 128         mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
 129
 130         // Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
 131         // We use a subdirectory of $GOROOT/pkg because that's the
 132         // space within $GOROOT where we store all generated objects.
 133         // We could use a temporary directory outside $GOROOT instead,
 134         // but it is easier to debug on failure if the files are in a known location.
 135         workspace := pathf("%s/pkg/bootstrap", goroot)
 136         xremoveall(workspace)
 137         xatexit(func() { xremoveall(workspace) })
 138         base := pathf("%s/src/bootstrap", workspace)
 139         xmkdirall(base)
 140
 141         // Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
 142         writefile("module bootstrap\ngo 1.20\n", pathf("%s/%s", base, "go.mod"), 0)
 143         for _, dir := range bootstrapDirs {
 144                 recurse := strings.HasSuffix(dir, "/...")
 145                 dir = strings.TrimSuffix(dir, "/...")
 146                 filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 147                         if err != nil {
 148                                 fatalf("walking bootstrap dirs failed: %v: %v", path, err)
 149                         }
 150
 151                         name := filepath.Base(path)
 152                         src := pathf("%s/src/%s", goroot, path)
 153                         dst := pathf("%s/%s", base, path)
 154
 155                         if info.IsDir() {
 156                                 if !recurse && path != dir || name == "testdata" {
 157                                         return filepath.SkipDir
 158                                 }
 159
 160                                 xmkdirall(dst)
 161                                 if path == "cmd/cgo" {
 162                                         // Write to src because we need the file both for bootstrap
 163                                         // and for later in the main build.
 164                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
 165                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
 166                                 }
 167                                 return nil
 168                         }
 169
 170                         for _, pre := range ignorePrefixes {
 171                                 if strings.HasPrefix(name, pre) {
 172                                         return nil
 173                                 }
 174                         }
 175                         for _, suf := range ignoreSuffixes {
 176                                 if strings.HasSuffix(name, suf) {
 177                                         return nil
 178                                 }
 179                         }
 180
 181                         text := bootstrapRewriteFile(src)
 182                         writefile(text, dst, 0)
 183                         return nil
 184                 })
 185         }
 186
 187         // Set up environment for invoking Go bootstrap toolchains go command.
 188         // GOROOT points at Go bootstrap GOROOT,
 189         // GOPATH points at our bootstrap workspace,
 190         // GOBIN is empty, so that binaries are installed to GOPATH/bin,
 191         // and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
 192         // so that Go bootstrap toolchain builds whatever kind of binary it knows how to build.
 193         // Restore GOROOT, GOPATH, and GOBIN when done.
 194         // Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
 195         // because setup will take care of those when bootstrapBuildTools returns.
 196
 197         defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
 198         os.Setenv("GOROOT", goroot_bootstrap)
 199
 200         defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
 201         os.Setenv("GOPATH", workspace)
 202
 203         defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
 204         os.Setenv("GOBIN", "")
 205
 206         os.Setenv("GOOS", "")
 207         os.Setenv("GOHOSTOS", "")
 208         os.Setenv("GOARCH", "")
 209         os.Setenv("GOHOSTARCH", "")
 210
 211         // Run Go bootstrap to build binaries.
 212         // Use the math_big_pure_go build tag to disable the assembly in math/big
 213         // which may contain unsupported instructions.
 214         // Use the purego build tag to disable other assembly code,
 215         // such as in cmd/internal/notsha256.
 216         cmd := []string{
 217                 pathf("%s/bin/go", goroot_bootstrap),
 218                 "install",
 219                 "-tags=math_big_pure_go compiler_bootstrap purego",
 220         }
 221         if vflag > 0 {
 222                 cmd = append(cmd, "-v")
 223         }
 224         if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
 225                 cmd = append(cmd, "-toolexec="+tool)
 226         }
 227         cmd = append(cmd, "bootstrap/cmd/...")
 228         run(base, ShowOutput|CheckExit, cmd...)
 229
 230         // Copy binaries into tool binary directory.
 231         for _, name := range bootstrapDirs {
 232                 if !strings.HasPrefix(name, "cmd/") {
 233                         continue
 234                 }
 235                 name = name[len("cmd/"):]
 236                 if !strings.Contains(name, "/") {
 237                         copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
 238                 }
 239         }
 240
 241         if vflag > 0 {
 242                 xprintf("\n")
 243         }
 244 }
 245
 246 var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
 247
 248 // isUnneededSSARewriteFile reports whether srcFile is a
 249 // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
 250 // architecture that isn't for the given GOARCH.
 251 //
 252 // When unneeded is true archCaps is the rewrite base filename without
 253 // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
 254 func isUnneededSSARewriteFile(srcFile, goArch string) (archCaps string, unneeded bool) {
 255         if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
 256                 return "", false
 257         }
 258         fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
 259         if fileArch == "" {
 260                 return "", false
 261         }
 262         b := fileArch[0]
 263         if b == '_' || ('a' <= b && b <= 'z') {
 264                 return "", false
 265         }
 266         archCaps = fileArch
 267         fileArch = strings.ToLower(fileArch)
 268         fileArch = strings.TrimSuffix(fileArch, "splitload")
 269         fileArch = strings.TrimSuffix(fileArch, "latelower")
 270         if fileArch == goArch {
 271                 return "", false
 272         }
 273         if fileArch == strings.TrimSuffix(goArch, "le") {
 274                 return "", false
 275         }
 276         return archCaps, true
 277 }
 278
 279 func bootstrapRewriteFile(srcFile string) string {
 280         // During bootstrap, generate dummy rewrite files for
 281         // irrelevant architectures. We only need to build a bootstrap
 282         // binary that works for the current gohostarch.
 283         // This saves 6+ seconds of bootstrap.
 284         if archCaps, ok := isUnneededSSARewriteFile(srcFile, gohostarch); ok {
 285                 return fmt.Sprintf(`%spackage ssa
 286
 287 func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
 288 func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
 289 `, generatedHeader, archCaps, archCaps)
 290         }
 291
 292         return bootstrapFixImports(srcFile)
 293 }
 294
 295 func bootstrapFixImports(srcFile string) string {
 296         text := readfile(srcFile)
 297         if !strings.Contains(srcFile, "/cmd/") && !strings.Contains(srcFile, `\cmd\`) {
 298                 text = regexp.MustCompile(`\bany\b`).ReplaceAllString(text, "interface{}")
 299         }
 300         lines := strings.SplitAfter(text, "\n")
 301         inBlock := false
 302         for i, line := range lines {
 303                 if strings.HasPrefix(line, "import (") {
 304                         inBlock = true
 305                         continue
 306                 }
 307                 if inBlock && strings.HasPrefix(line, ")") {
 308                         inBlock = false
 309                         continue
 310                 }
 311                 if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
 312                         inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"") || strings.HasPrefix(line, "\trtabi \"")) {
 313                         line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
 314                         for _, dir := range bootstrapDirs {
 315                                 if strings.HasPrefix(dir, "cmd/") {
 316                                         continue
 317                                 }
 318                                 line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
 319                         }
 320                         lines[i] = line
 321                 }
 322         }
 323
 324         lines[0] = generatedHeader + "// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
 325
 326         return strings.Join(lines, "")
 327 }