src/cmd/dist/buildtool.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // Build toolchain using Go bootstrap version.
   6 //
   7 // The general strategy is to copy the source files we need into
   8 // a new GOPATH workspace, adjust import paths appropriately,
   9 // invoke the Go bootstrap toolchains go command to build those sources,
  10 // and then copy the binaries back.
  11
  12 package main
  13
  14 import (
  15         "fmt"
  16         "os"
  17         "path/filepath"
  18         "regexp"
  19         "strings"
  20 )
  21
  22 // bootstrapDirs is a list of directories holding code that must be
  23 // compiled with the Go bootstrap toolchain to produce the bootstrapTargets.
  24 // All directories in this list are relative to and must be below $GOROOT/src.
  25 //
  26 // The list has two kinds of entries: names beginning with cmd/ with
  27 // no other slashes, which are commands, and other paths, which are packages
  28 // supporting the commands. Packages in the standard library can be listed
  29 // if a newer copy needs to be substituted for the Go bootstrap copy when used
  30 // by the command packages. Paths ending with /... automatically
  31 // include all packages within subdirectories as well.
  32 // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
  33 var bootstrapDirs = []string{
  34         "cmd/asm",
  35         "cmd/asm/internal/...",
  36         "cmd/cgo",
  37         "cmd/compile",
  38         "cmd/compile/internal/...",
  39         "cmd/internal/archive",
  40         "cmd/internal/bio",
  41         "cmd/internal/codesign",
  42         "cmd/internal/dwarf",
  43         "cmd/internal/edit",
  44         "cmd/internal/gcprog",
  45         "cmd/internal/goobj",
  46         "cmd/internal/notsha256",
  47         "cmd/internal/obj/...",
  48         "cmd/internal/objabi",
  49         "cmd/internal/pkgpath",
  50         "cmd/internal/quoted",
  51         "cmd/internal/src",
  52         "cmd/internal/sys",
  53         "cmd/link",
  54         "cmd/link/internal/...",
  55         "compress/flate",
  56         "compress/zlib",
  57         "container/heap",
  58         "debug/dwarf",
  59         "debug/elf",
  60         "debug/macho",
  61         "debug/pe",
  62         "go/build/constraint",
  63         "go/constant",
  64         "internal/abi",
  65         "internal/coverage",
  66         "internal/bisect",
  67         "internal/buildcfg",
  68         "internal/goarch",
  69         "internal/godebugs",
  70         "internal/goexperiment",
  71         "internal/goroot",
  72         "internal/goversion",
  73         // internal/lazyregexp is provided by Go 1.17, which permits it to
  74         // be imported by other packages in this list, but is not provided
  75         // by the Go 1.17 version of gccgo. It's on this list only to
  76         // support gccgo, and can be removed if we require gccgo 14 or later.
  77         "internal/lazyregexp",
  78         "internal/pkgbits",
  79         "internal/platform",
  80         "internal/profile",
  81         "internal/race",
  82         "internal/saferio",
  83         "internal/syscall/unix",
  84         "internal/types/errors",
  85         "internal/unsafeheader",
  86         "internal/xcoff",
  87         "internal/zstd",
  88         "math/big",
  89         "math/bits",
  90         "sort",
  91         "strconv",
  92 }
  93
  94 // File prefixes that are ignored by go/build anyway, and cause
  95 // problems with editor generated temporary files (#18931).
  96 var ignorePrefixes = []string{
  97         ".",
  98         "_",
  99         "#",
 100 }
 101
 102 // File suffixes that use build tags introduced since Go 1.17.
 103 // These must not be copied into the bootstrap build directory.
 104 // Also ignore test files.
 105 var ignoreSuffixes = []string{
 106         "_test.s",
 107         "_test.go",
 108         // Skip PGO profile. No need to build toolchain1 compiler
 109         // with PGO. And as it is not a text file the import path
 110         // rewrite will break it.
 111         ".pgo",
 112 }
 113
 114 var tryDirs = []string{
 115         "sdk/go1.17",
 116         "go1.17",
 117 }
 118
 119 func bootstrapBuildTools() {
 120         goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
 121         if goroot_bootstrap == "" {
 122                 home := os.Getenv("HOME")
 123                 goroot_bootstrap = pathf("%s/go1.4", home)
 124                 for _, d := range tryDirs {
 125                         if p := pathf("%s/%s", home, d); isdir(p) {
 126                                 goroot_bootstrap = p
 127                         }
 128                 }
 129         }
 130         xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
 131
 132         mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
 133         mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
 134
 135         // Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
 136         // We use a subdirectory of $GOROOT/pkg because that's the
 137         // space within $GOROOT where we store all generated objects.
 138         // We could use a temporary directory outside $GOROOT instead,
 139         // but it is easier to debug on failure if the files are in a known location.
 140         workspace := pathf("%s/pkg/bootstrap", goroot)
 141         xremoveall(workspace)
 142         xatexit(func() { xremoveall(workspace) })
 143         base := pathf("%s/src/bootstrap", workspace)
 144         xmkdirall(base)
 145
 146         // Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
 147         writefile("module bootstrap\ngo 1.20\n", pathf("%s/%s", base, "go.mod"), 0)
 148         for _, dir := range bootstrapDirs {
 149                 recurse := strings.HasSuffix(dir, "/...")
 150                 dir = strings.TrimSuffix(dir, "/...")
 151                 filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 152                         if err != nil {
 153                                 fatalf("walking bootstrap dirs failed: %v: %v", path, err)
 154                         }
 155
 156                         name := filepath.Base(path)
 157                         src := pathf("%s/src/%s", goroot, path)
 158                         dst := pathf("%s/%s", base, path)
 159
 160                         if info.IsDir() {
 161                                 if !recurse && path != dir || name == "testdata" {
 162                                         return filepath.SkipDir
 163                                 }
 164
 165                                 xmkdirall(dst)
 166                                 if path == "cmd/cgo" {
 167                                         // Write to src because we need the file both for bootstrap
 168                                         // and for later in the main build.
 169                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
 170                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
 171                                 }
 172                                 return nil
 173                         }
 174
 175                         for _, pre := range ignorePrefixes {
 176                                 if strings.HasPrefix(name, pre) {
 177                                         return nil
 178                                 }
 179                         }
 180                         for _, suf := range ignoreSuffixes {
 181                                 if strings.HasSuffix(name, suf) {
 182                                         return nil
 183                                 }
 184                         }
 185
 186                         text := bootstrapRewriteFile(src)
 187                         writefile(text, dst, 0)
 188                         return nil
 189                 })
 190         }
 191
 192         // Set up environment for invoking Go bootstrap toolchains go command.
 193         // GOROOT points at Go bootstrap GOROOT,
 194         // GOPATH points at our bootstrap workspace,
 195         // GOBIN is empty, so that binaries are installed to GOPATH/bin,
 196         // and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
 197         // so that Go bootstrap toolchain builds whatever kind of binary it knows how to build.
 198         // Restore GOROOT, GOPATH, and GOBIN when done.
 199         // Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
 200         // because setup will take care of those when bootstrapBuildTools returns.
 201
 202         defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
 203         os.Setenv("GOROOT", goroot_bootstrap)
 204
 205         defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
 206         os.Setenv("GOPATH", workspace)
 207
 208         defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
 209         os.Setenv("GOBIN", "")
 210
 211         os.Setenv("GOOS", "")
 212         os.Setenv("GOHOSTOS", "")
 213         os.Setenv("GOARCH", "")
 214         os.Setenv("GOHOSTARCH", "")
 215
 216         // Run Go bootstrap to build binaries.
 217         // Use the math_big_pure_go build tag to disable the assembly in math/big
 218         // which may contain unsupported instructions.
 219         // Use the purego build tag to disable other assembly code,
 220         // such as in cmd/internal/notsha256.
 221         cmd := []string{
 222                 pathf("%s/bin/go", goroot_bootstrap),
 223                 "install",
 224                 "-tags=math_big_pure_go compiler_bootstrap purego",
 225         }
 226         if vflag > 0 {
 227                 cmd = append(cmd, "-v")
 228         }
 229         if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
 230                 cmd = append(cmd, "-toolexec="+tool)
 231         }
 232         cmd = append(cmd, "bootstrap/cmd/...")
 233         run(base, ShowOutput|CheckExit, cmd...)
 234
 235         // Copy binaries into tool binary directory.
 236         for _, name := range bootstrapDirs {
 237                 if !strings.HasPrefix(name, "cmd/") {
 238                         continue
 239                 }
 240                 name = name[len("cmd/"):]
 241                 if !strings.Contains(name, "/") {
 242                         copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
 243                 }
 244         }
 245
 246         if vflag > 0 {
 247                 xprintf("\n")
 248         }
 249 }
 250
 251 var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
 252
 253 // isUnneededSSARewriteFile reports whether srcFile is a
 254 // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
 255 // architecture that isn't for the given GOARCH.
 256 //
 257 // When unneeded is true archCaps is the rewrite base filename without
 258 // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
 259 func isUnneededSSARewriteFile(srcFile, goArch string) (archCaps string, unneeded bool) {
 260         if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
 261                 return "", false
 262         }
 263         fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
 264         if fileArch == "" {
 265                 return "", false
 266         }
 267         b := fileArch[0]
 268         if b == '_' || ('a' <= b && b <= 'z') {
 269                 return "", false
 270         }
 271         archCaps = fileArch
 272         fileArch = strings.ToLower(fileArch)
 273         fileArch = strings.TrimSuffix(fileArch, "splitload")
 274         fileArch = strings.TrimSuffix(fileArch, "latelower")
 275         if fileArch == goArch {
 276                 return "", false
 277         }
 278         if fileArch == strings.TrimSuffix(goArch, "le") {
 279                 return "", false
 280         }
 281         return archCaps, true
 282 }
 283
 284 func bootstrapRewriteFile(srcFile string) string {
 285         // During bootstrap, generate dummy rewrite files for
 286         // irrelevant architectures. We only need to build a bootstrap
 287         // binary that works for the current gohostarch.
 288         // This saves 6+ seconds of bootstrap.
 289         if archCaps, ok := isUnneededSSARewriteFile(srcFile, gohostarch); ok {
 290                 return fmt.Sprintf(`%spackage ssa
 291
 292 func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
 293 func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
 294 `, generatedHeader, archCaps, archCaps)
 295         }
 296
 297         return bootstrapFixImports(srcFile)
 298 }
 299
 300 func bootstrapFixImports(srcFile string) string {
 301         text := readfile(srcFile)
 302         if !strings.Contains(srcFile, "/cmd/") && !strings.Contains(srcFile, `\cmd\`) {
 303                 text = regexp.MustCompile(`\bany\b`).ReplaceAllString(text, "interface{}")
 304         }
 305         lines := strings.SplitAfter(text, "\n")
 306         inBlock := false
 307         for i, line := range lines {
 308                 if strings.HasPrefix(line, "import (") {
 309                         inBlock = true
 310                         continue
 311                 }
 312                 if inBlock && strings.HasPrefix(line, ")") {
 313                         inBlock = false
 314                         continue
 315                 }
 316                 if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
 317                         inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"") || strings.HasPrefix(line, "\trtabi \"")) {
 318                         line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
 319                         for _, dir := range bootstrapDirs {
 320                                 if strings.HasPrefix(dir, "cmd/") {
 321                                         continue
 322                                 }
 323                                 line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
 324                         }
 325                         lines[i] = line
 326                 }
 327         }
 328
 329         lines[0] = generatedHeader + "// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
 330
 331         return strings.Join(lines, "")
 332 }