src/cmd/dist/buildtool.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // Build toolchain using Go bootstrap version.
   6 //
   7 // The general strategy is to copy the source files we need into
   8 // a new GOPATH workspace, adjust import paths appropriately,
   9 // invoke the Go bootstrap toolchains go command to build those sources,
  10 // and then copy the binaries back.
  11
  12 package main
  13
  14 import (
  15         "fmt"
  16         "os"
  17         "path/filepath"
  18         "regexp"
  19         "strings"
  20 )
  21
  22 // bootstrapDirs is a list of directories holding code that must be
  23 // compiled with the Go bootstrap toolchain to produce the bootstrapTargets.
  24 // All directories in this list are relative to and must be below $GOROOT/src.
  25 //
  26 // The list has two kinds of entries: names beginning with cmd/ with
  27 // no other slashes, which are commands, and other paths, which are packages
  28 // supporting the commands. Packages in the standard library can be listed
  29 // if a newer copy needs to be substituted for the Go bootstrap copy when used
  30 // by the command packages. Paths ending with /... automatically
  31 // include all packages within subdirectories as well.
  32 // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
  33 var bootstrapDirs = []string{
  34         "cmd/asm",
  35         "cmd/asm/internal/...",
  36         "cmd/cgo",
  37         "cmd/compile",
  38         "cmd/compile/internal/...",
  39         "cmd/internal/archive",
  40         "cmd/internal/bio",
  41         "cmd/internal/codesign",
  42         "cmd/internal/dwarf",
  43         "cmd/internal/edit",
  44         "cmd/internal/gcprog",
  45         "cmd/internal/goobj",
  46         "cmd/internal/notsha256",
  47         "cmd/internal/obj/...",
  48         "cmd/internal/objabi",
  49         "cmd/internal/pkgpath",
  50         "cmd/internal/quoted",
  51         "cmd/internal/src",
  52         "cmd/internal/sys",
  53         "cmd/link",
  54         "cmd/link/internal/...",
  55         "compress/flate",
  56         "compress/zlib",
  57         "container/heap",
  58         "debug/dwarf",
  59         "debug/elf",
  60         "debug/macho",
  61         "debug/pe",
  62         "go/build/constraint",
  63         "go/constant",
  64         "internal/abi",
  65         "internal/coverage",
  66         "internal/bisect",
  67         "internal/buildcfg",
  68         "internal/goarch",
  69         "internal/godebugs",
  70         "internal/goexperiment",
  71         "internal/goroot",
  72         "internal/goversion",
  73         "internal/pkgbits",
  74         "internal/platform",
  75         "internal/profile",
  76         "internal/race",
  77         "internal/saferio",
  78         "internal/syscall/unix",
  79         "internal/types/errors",
  80         "internal/unsafeheader",
  81         "internal/xcoff",
  82         "internal/zstd",
  83         "math/big",
  84         "math/bits",
  85         "sort",
  86         "strconv",
  87 }
  88
  89 // File prefixes that are ignored by go/build anyway, and cause
  90 // problems with editor generated temporary files (#18931).
  91 var ignorePrefixes = []string{
  92         ".",
  93         "_",
  94         "#",
  95 }
  96
  97 // File suffixes that use build tags introduced since Go 1.17.
  98 // These must not be copied into the bootstrap build directory.
  99 // Also ignore test files.
 100 var ignoreSuffixes = []string{
 101         "_test.s",
 102         "_test.go",
 103         // Skip PGO profile. No need to build toolchain1 compiler
 104         // with PGO. And as it is not a text file the import path
 105         // rewrite will break it.
 106         ".pgo",
 107 }
 108
 109 var tryDirs = []string{
 110         "sdk/go1.17",
 111         "go1.17",
 112 }
 113
 114 func bootstrapBuildTools() {
 115         goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
 116         if goroot_bootstrap == "" {
 117                 home := os.Getenv("HOME")
 118                 goroot_bootstrap = pathf("%s/go1.4", home)
 119                 for _, d := range tryDirs {
 120                         if p := pathf("%s/%s", home, d); isdir(p) {
 121                                 goroot_bootstrap = p
 122                         }
 123                 }
 124         }
 125         xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
 126
 127         mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
 128         mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
 129         mkzosarch("", pathf("%s/src/internal/platform/zosarch.go", goroot))
 130
 131         // Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
 132         // We use a subdirectory of $GOROOT/pkg because that's the
 133         // space within $GOROOT where we store all generated objects.
 134         // We could use a temporary directory outside $GOROOT instead,
 135         // but it is easier to debug on failure if the files are in a known location.
 136         workspace := pathf("%s/pkg/bootstrap", goroot)
 137         xremoveall(workspace)
 138         xatexit(func() { xremoveall(workspace) })
 139         base := pathf("%s/src/bootstrap", workspace)
 140         xmkdirall(base)
 141
 142         // Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
 143         writefile("module bootstrap\ngo 1.20\n", pathf("%s/%s", base, "go.mod"), 0)
 144         for _, dir := range bootstrapDirs {
 145                 recurse := strings.HasSuffix(dir, "/...")
 146                 dir = strings.TrimSuffix(dir, "/...")
 147                 filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 148                         if err != nil {
 149                                 fatalf("walking bootstrap dirs failed: %v: %v", path, err)
 150                         }
 151
 152                         name := filepath.Base(path)
 153                         src := pathf("%s/src/%s", goroot, path)
 154                         dst := pathf("%s/%s", base, path)
 155
 156                         if info.IsDir() {
 157                                 if !recurse && path != dir || name == "testdata" {
 158                                         return filepath.SkipDir
 159                                 }
 160
 161                                 xmkdirall(dst)
 162                                 if path == "cmd/cgo" {
 163                                         // Write to src because we need the file both for bootstrap
 164                                         // and for later in the main build.
 165                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
 166                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
 167                                 }
 168                                 return nil
 169                         }
 170
 171                         for _, pre := range ignorePrefixes {
 172                                 if strings.HasPrefix(name, pre) {
 173                                         return nil
 174                                 }
 175                         }
 176                         for _, suf := range ignoreSuffixes {
 177                                 if strings.HasSuffix(name, suf) {
 178                                         return nil
 179                                 }
 180                         }
 181
 182                         text := bootstrapRewriteFile(src)
 183                         writefile(text, dst, 0)
 184                         return nil
 185                 })
 186         }
 187
 188         // Set up environment for invoking Go bootstrap toolchains go command.
 189         // GOROOT points at Go bootstrap GOROOT,
 190         // GOPATH points at our bootstrap workspace,
 191         // GOBIN is empty, so that binaries are installed to GOPATH/bin,
 192         // and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
 193         // so that Go bootstrap toolchain builds whatever kind of binary it knows how to build.
 194         // Restore GOROOT, GOPATH, and GOBIN when done.
 195         // Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
 196         // because setup will take care of those when bootstrapBuildTools returns.
 197
 198         defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
 199         os.Setenv("GOROOT", goroot_bootstrap)
 200
 201         defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
 202         os.Setenv("GOPATH", workspace)
 203
 204         defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
 205         os.Setenv("GOBIN", "")
 206
 207         os.Setenv("GOOS", "")
 208         os.Setenv("GOHOSTOS", "")
 209         os.Setenv("GOARCH", "")
 210         os.Setenv("GOHOSTARCH", "")
 211
 212         // Run Go bootstrap to build binaries.
 213         // Use the math_big_pure_go build tag to disable the assembly in math/big
 214         // which may contain unsupported instructions.
 215         // Use the purego build tag to disable other assembly code,
 216         // such as in cmd/internal/notsha256.
 217         cmd := []string{
 218                 pathf("%s/bin/go", goroot_bootstrap),
 219                 "install",
 220                 "-tags=math_big_pure_go compiler_bootstrap purego",
 221         }
 222         if vflag > 0 {
 223                 cmd = append(cmd, "-v")
 224         }
 225         if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
 226                 cmd = append(cmd, "-toolexec="+tool)
 227         }
 228         cmd = append(cmd, "bootstrap/cmd/...")
 229         run(base, ShowOutput|CheckExit, cmd...)
 230
 231         // Copy binaries into tool binary directory.
 232         for _, name := range bootstrapDirs {
 233                 if !strings.HasPrefix(name, "cmd/") {
 234                         continue
 235                 }
 236                 name = name[len("cmd/"):]
 237                 if !strings.Contains(name, "/") {
 238                         copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
 239                 }
 240         }
 241
 242         if vflag > 0 {
 243                 xprintf("\n")
 244         }
 245 }
 246
 247 var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
 248
 249 // isUnneededSSARewriteFile reports whether srcFile is a
 250 // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
 251 // architecture that isn't for the given GOARCH.
 252 //
 253 // When unneeded is true archCaps is the rewrite base filename without
 254 // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
 255 func isUnneededSSARewriteFile(srcFile, goArch string) (archCaps string, unneeded bool) {
 256         if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
 257                 return "", false
 258         }
 259         fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
 260         if fileArch == "" {
 261                 return "", false
 262         }
 263         b := fileArch[0]
 264         if b == '_' || ('a' <= b && b <= 'z') {
 265                 return "", false
 266         }
 267         archCaps = fileArch
 268         fileArch = strings.ToLower(fileArch)
 269         fileArch = strings.TrimSuffix(fileArch, "splitload")
 270         fileArch = strings.TrimSuffix(fileArch, "latelower")
 271         if fileArch == goArch {
 272                 return "", false
 273         }
 274         if fileArch == strings.TrimSuffix(goArch, "le") {
 275                 return "", false
 276         }
 277         return archCaps, true
 278 }
 279
 280 func bootstrapRewriteFile(srcFile string) string {
 281         // During bootstrap, generate dummy rewrite files for
 282         // irrelevant architectures. We only need to build a bootstrap
 283         // binary that works for the current gohostarch.
 284         // This saves 6+ seconds of bootstrap.
 285         if archCaps, ok := isUnneededSSARewriteFile(srcFile, gohostarch); ok {
 286                 return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.
 287
 288 package ssa
 289
 290 func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
 291 func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
 292 `, archCaps, archCaps)
 293         }
 294
 295         return bootstrapFixImports(srcFile)
 296 }
 297
 298 func bootstrapFixImports(srcFile string) string {
 299         text := readfile(srcFile)
 300         if !strings.Contains(srcFile, "/cmd/") && !strings.Contains(srcFile, `\cmd\`) {
 301                 text = regexp.MustCompile(`\bany\b`).ReplaceAllString(text, "interface{}")
 302         }
 303         lines := strings.SplitAfter(text, "\n")
 304         inBlock := false
 305         for i, line := range lines {
 306                 if strings.HasPrefix(line, "import (") {
 307                         inBlock = true
 308                         continue
 309                 }
 310                 if inBlock && strings.HasPrefix(line, ")") {
 311                         inBlock = false
 312                         continue
 313                 }
 314                 if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
 315                         inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"") || strings.HasPrefix(line, "\trtabi \"")) {
 316                         line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
 317                         for _, dir := range bootstrapDirs {
 318                                 if strings.HasPrefix(dir, "cmd/") {
 319                                         continue
 320                                 }
 321                                 line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
 322                         }
 323                         lines[i] = line
 324                 }
 325         }
 326
 327         lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
 328
 329         return strings.Join(lines, "")
 330 }