src/cmd/dist/buildtool.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // Build toolchain using Go bootstrap version.
   6 //
   7 // The general strategy is to copy the source files we need into
   8 // a new GOPATH workspace, adjust import paths appropriately,
   9 // invoke the Go bootstrap toolchains go command to build those sources,
  10 // and then copy the binaries back.
  11
  12 package main
  13
  14 import (
  15         "fmt"
  16         "os"
  17         "path/filepath"
  18         "regexp"
  19         "strings"
  20 )
  21
  22 // bootstrapDirs is a list of directories holding code that must be
  23 // compiled with the Go bootstrap toolchain to produce the bootstrapTargets.
  24 // All directories in this list are relative to and must be below $GOROOT/src.
  25 //
  26 // The list has two kinds of entries: names beginning with cmd/ with
  27 // no other slashes, which are commands, and other paths, which are packages
  28 // supporting the commands. Packages in the standard library can be listed
  29 // if a newer copy needs to be substituted for the Go bootstrap copy when used
  30 // by the command packages. Paths ending with /... automatically
  31 // include all packages within subdirectories as well.
  32 // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
  33 var bootstrapDirs = []string{
  34         "cmd/asm",
  35         "cmd/asm/internal/...",
  36         "cmd/cgo",
  37         "cmd/compile",
  38         "cmd/compile/internal/...",
  39         "cmd/internal/archive",
  40         "cmd/internal/bio",
  41         "cmd/internal/codesign",
  42         "cmd/internal/dwarf",
  43         "cmd/internal/edit",
  44         "cmd/internal/gcprog",
  45         "cmd/internal/goobj",
  46         "cmd/internal/notsha256",
  47         "cmd/internal/obj/...",
  48         "cmd/internal/objabi",
  49         "cmd/internal/pkgpath",
  50         "cmd/internal/quoted",
  51         "cmd/internal/src",
  52         "cmd/internal/sys",
  53         "cmd/link",
  54         "cmd/link/internal/...",
  55         "compress/flate",
  56         "compress/zlib",
  57         "container/heap",
  58         "debug/dwarf",
  59         "debug/elf",
  60         "debug/macho",
  61         "debug/pe",
  62         "go/build/constraint",
  63         "go/constant",
  64         "internal/abi",
  65         "internal/coverage",
  66         "cmd/internal/cov/covcmd",
  67         "internal/bisect",
  68         "internal/buildcfg",
  69         "internal/goarch",
  70         "internal/godebugs",
  71         "internal/goexperiment",
  72         "internal/goroot",
  73         "internal/goversion",
  74         // internal/lazyregexp is provided by Go 1.17, which permits it to
  75         // be imported by other packages in this list, but is not provided
  76         // by the Go 1.17 version of gccgo. It's on this list only to
  77         // support gccgo, and can be removed if we require gccgo 14 or later.
  78         "internal/lazyregexp",
  79         "internal/pkgbits",
  80         "internal/platform",
  81         "internal/profile",
  82         "internal/race",
  83         "internal/saferio",
  84         "internal/syscall/unix",
  85         "internal/types/errors",
  86         "internal/unsafeheader",
  87         "internal/xcoff",
  88         "internal/zstd",
  89         "math/bits",
  90         "sort",
  91 }
  92
  93 // File prefixes that are ignored by go/build anyway, and cause
  94 // problems with editor generated temporary files (#18931).
  95 var ignorePrefixes = []string{
  96         ".",
  97         "_",
  98         "#",
  99 }
 100
 101 // File suffixes that use build tags introduced since Go 1.17.
 102 // These must not be copied into the bootstrap build directory.
 103 // Also ignore test files.
 104 var ignoreSuffixes = []string{
 105         "_test.s",
 106         "_test.go",
 107         // Skip PGO profile. No need to build toolchain1 compiler
 108         // with PGO. And as it is not a text file the import path
 109         // rewrite will break it.
 110         ".pgo",
 111 }
 112
 113 var tryDirs = []string{
 114         "sdk/go1.17",
 115         "go1.17",
 116 }
 117
 118 func bootstrapBuildTools() {
 119         goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
 120         if goroot_bootstrap == "" {
 121                 home := os.Getenv("HOME")
 122                 goroot_bootstrap = pathf("%s/go1.4", home)
 123                 for _, d := range tryDirs {
 124                         if p := pathf("%s/%s", home, d); isdir(p) {
 125                                 goroot_bootstrap = p
 126                         }
 127                 }
 128         }
 129         xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
 130
 131         mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
 132         mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
 133
 134         // Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
 135         // We use a subdirectory of $GOROOT/pkg because that's the
 136         // space within $GOROOT where we store all generated objects.
 137         // We could use a temporary directory outside $GOROOT instead,
 138         // but it is easier to debug on failure if the files are in a known location.
 139         workspace := pathf("%s/pkg/bootstrap", goroot)
 140         xremoveall(workspace)
 141         xatexit(func() { xremoveall(workspace) })
 142         base := pathf("%s/src/bootstrap", workspace)
 143         xmkdirall(base)
 144
 145         // Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
 146         writefile("module bootstrap\ngo 1.20\n", pathf("%s/%s", base, "go.mod"), 0)
 147         for _, dir := range bootstrapDirs {
 148                 recurse := strings.HasSuffix(dir, "/...")
 149                 dir = strings.TrimSuffix(dir, "/...")
 150                 filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 151                         if err != nil {
 152                                 fatalf("walking bootstrap dirs failed: %v: %v", path, err)
 153                         }
 154
 155                         name := filepath.Base(path)
 156                         src := pathf("%s/src/%s", goroot, path)
 157                         dst := pathf("%s/%s", base, path)
 158
 159                         if info.IsDir() {
 160                                 if !recurse && path != dir || name == "testdata" {
 161                                         return filepath.SkipDir
 162                                 }
 163
 164                                 xmkdirall(dst)
 165                                 if path == "cmd/cgo" {
 166                                         // Write to src because we need the file both for bootstrap
 167                                         // and for later in the main build.
 168                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
 169                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
 170                                 }
 171                                 return nil
 172                         }
 173
 174                         for _, pre := range ignorePrefixes {
 175                                 if strings.HasPrefix(name, pre) {
 176                                         return nil
 177                                 }
 178                         }
 179                         for _, suf := range ignoreSuffixes {
 180                                 if strings.HasSuffix(name, suf) {
 181                                         return nil
 182                                 }
 183                         }
 184
 185                         text := bootstrapRewriteFile(src)
 186                         writefile(text, dst, 0)
 187                         return nil
 188                 })
 189         }
 190
 191         // Set up environment for invoking Go bootstrap toolchains go command.
 192         // GOROOT points at Go bootstrap GOROOT,
 193         // GOPATH points at our bootstrap workspace,
 194         // GOBIN is empty, so that binaries are installed to GOPATH/bin,
 195         // and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
 196         // so that Go bootstrap toolchain builds whatever kind of binary it knows how to build.
 197         // Restore GOROOT, GOPATH, and GOBIN when done.
 198         // Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
 199         // because setup will take care of those when bootstrapBuildTools returns.
 200
 201         defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
 202         os.Setenv("GOROOT", goroot_bootstrap)
 203
 204         defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
 205         os.Setenv("GOPATH", workspace)
 206
 207         defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
 208         os.Setenv("GOBIN", "")
 209
 210         os.Setenv("GOOS", "")
 211         os.Setenv("GOHOSTOS", "")
 212         os.Setenv("GOARCH", "")
 213         os.Setenv("GOHOSTARCH", "")
 214
 215         // Run Go bootstrap to build binaries.
 216         // Use the math_big_pure_go build tag to disable the assembly in math/big
 217         // which may contain unsupported instructions.
 218         // Use the purego build tag to disable other assembly code,
 219         // such as in cmd/internal/notsha256.
 220         cmd := []string{
 221                 pathf("%s/bin/go", goroot_bootstrap),
 222                 "install",
 223                 "-tags=math_big_pure_go compiler_bootstrap purego",
 224         }
 225         if vflag > 0 {
 226                 cmd = append(cmd, "-v")
 227         }
 228         if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
 229                 cmd = append(cmd, "-toolexec="+tool)
 230         }
 231         cmd = append(cmd, "bootstrap/cmd/...")
 232         run(base, ShowOutput|CheckExit, cmd...)
 233
 234         // Copy binaries into tool binary directory.
 235         for _, name := range bootstrapDirs {
 236                 if !strings.HasPrefix(name, "cmd/") {
 237                         continue
 238                 }
 239                 name = name[len("cmd/"):]
 240                 if !strings.Contains(name, "/") {
 241                         copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
 242                 }
 243         }
 244
 245         if vflag > 0 {
 246                 xprintf("\n")
 247         }
 248 }
 249
 250 var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
 251
 252 // isUnneededSSARewriteFile reports whether srcFile is a
 253 // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
 254 // architecture that isn't for the given GOARCH.
 255 //
 256 // When unneeded is true archCaps is the rewrite base filename without
 257 // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
 258 func isUnneededSSARewriteFile(srcFile, goArch string) (archCaps string, unneeded bool) {
 259         if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
 260                 return "", false
 261         }
 262         fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
 263         if fileArch == "" {
 264                 return "", false
 265         }
 266         b := fileArch[0]
 267         if b == '_' || ('a' <= b && b <= 'z') {
 268                 return "", false
 269         }
 270         archCaps = fileArch
 271         fileArch = strings.ToLower(fileArch)
 272         fileArch = strings.TrimSuffix(fileArch, "splitload")
 273         fileArch = strings.TrimSuffix(fileArch, "latelower")
 274         if fileArch == goArch {
 275                 return "", false
 276         }
 277         if fileArch == strings.TrimSuffix(goArch, "le") {
 278                 return "", false
 279         }
 280         return archCaps, true
 281 }
 282
 283 func bootstrapRewriteFile(srcFile string) string {
 284         // During bootstrap, generate dummy rewrite files for
 285         // irrelevant architectures. We only need to build a bootstrap
 286         // binary that works for the current gohostarch.
 287         // This saves 6+ seconds of bootstrap.
 288         if archCaps, ok := isUnneededSSARewriteFile(srcFile, gohostarch); ok {
 289                 return fmt.Sprintf(`%spackage ssa
 290
 291 func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
 292 func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
 293 `, generatedHeader, archCaps, archCaps)
 294         }
 295
 296         return bootstrapFixImports(srcFile)
 297 }
 298
 299 func bootstrapFixImports(srcFile string) string {
 300         text := readfile(srcFile)
 301         if !strings.Contains(srcFile, "/cmd/") && !strings.Contains(srcFile, `\cmd\`) {
 302                 text = regexp.MustCompile(`\bany\b`).ReplaceAllString(text, "interface{}")
 303         }
 304         lines := strings.SplitAfter(text, "\n")
 305         inBlock := false
 306         for i, line := range lines {
 307                 if strings.HasPrefix(line, "import (") {
 308                         inBlock = true
 309                         continue
 310                 }
 311                 if inBlock && strings.HasPrefix(line, ")") {
 312                         inBlock = false
 313                         continue
 314                 }
 315                 if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
 316                         inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"") || strings.HasPrefix(line, "\trtabi \"")) {
 317                         line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
 318                         for _, dir := range bootstrapDirs {
 319                                 if strings.HasPrefix(dir, "cmd/") {
 320                                         continue
 321                                 }
 322                                 line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
 323                         }
 324                         lines[i] = line
 325                 }
 326         }
 327
 328         lines[0] = generatedHeader + "// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
 329
 330         return strings.Join(lines, "")
 331 }