src/cmd/dist/buildtool.go

   1 // Copyright 2015 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 // Build toolchain using Go 1.4.
   6 //
   7 // The general strategy is to copy the source files we need into
   8 // a new GOPATH workspace, adjust import paths appropriately,
   9 // invoke the Go 1.4 go command to build those sources,
  10 // and then copy the binaries back.
  11
  12 package main
  13
  14 import (
  15         "fmt"
  16         "os"
  17         "path/filepath"
  18         "regexp"
  19         "strings"
  20 )
  21
  22 // bootstrapDirs is a list of directories holding code that must be
  23 // compiled with a Go 1.4 toolchain to produce the bootstrapTargets.
  24 // All directories in this list are relative to and must be below $GOROOT/src.
  25 //
  26 // The list has two kinds of entries: names beginning with cmd/ with
  27 // no other slashes, which are commands, and other paths, which are packages
  28 // supporting the commands. Packages in the standard library can be listed
  29 // if a newer copy needs to be substituted for the Go 1.4 copy when used
  30 // by the command packages. Paths ending with /... automatically
  31 // include all packages within subdirectories as well.
  32 // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
  33 var bootstrapDirs = []string{
  34         "cmd/asm",
  35         "cmd/asm/internal/...",
  36         "cmd/cgo",
  37         "cmd/compile",
  38         "cmd/compile/internal/...",
  39         "cmd/internal/archive",
  40         "cmd/internal/bio",
  41         "cmd/internal/codesign",
  42         "cmd/internal/dwarf",
  43         "cmd/internal/edit",
  44         "cmd/internal/gcprog",
  45         "cmd/internal/goobj",
  46         "cmd/internal/notsha256",
  47         "cmd/internal/obj/...",
  48         "cmd/internal/objabi",
  49         "cmd/internal/pkgpath",
  50         "cmd/internal/quoted",
  51         "cmd/internal/src",
  52         "cmd/internal/sys",
  53         "cmd/link",
  54         "cmd/link/internal/...",
  55         "compress/flate",
  56         "compress/zlib",
  57         "container/heap",
  58         "debug/dwarf",
  59         "debug/elf",
  60         "debug/macho",
  61         "debug/pe",
  62         "go/constant",
  63         "internal/buildcfg",
  64         "internal/goexperiment",
  65         "internal/goversion",
  66         "internal/pkgbits",
  67         "internal/race",
  68         "internal/unsafeheader",
  69         "internal/xcoff",
  70         "math/big",
  71         "math/bits",
  72         "sort",
  73         "strconv",
  74 }
  75
  76 // File prefixes that are ignored by go/build anyway, and cause
  77 // problems with editor generated temporary files (#18931).
  78 var ignorePrefixes = []string{
  79         ".",
  80         "_",
  81         "#",
  82 }
  83
  84 // File suffixes that use build tags introduced since Go 1.4.
  85 // These must not be copied into the bootstrap build directory.
  86 // Also ignore test files.
  87 var ignoreSuffixes = []string{
  88         "_arm64.s",
  89         "_arm64.go",
  90         "_loong64.s",
  91         "_loong64.go",
  92         "_riscv64.s",
  93         "_riscv64.go",
  94         "_wasm.s",
  95         "_wasm.go",
  96         "_test.s",
  97         "_test.go",
  98 }
  99
 100 var tryDirs = []string{
 101         "sdk/go1.17",
 102         "go1.17",
 103 }
 104
 105 func bootstrapBuildTools() {
 106         goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
 107         if goroot_bootstrap == "" {
 108                 home := os.Getenv("HOME")
 109                 goroot_bootstrap = pathf("%s/go1.4", home)
 110                 for _, d := range tryDirs {
 111                         if p := pathf("%s/%s", home, d); isdir(p) {
 112                                 goroot_bootstrap = p
 113                         }
 114                 }
 115         }
 116         xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
 117
 118         mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
 119         mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
 120
 121         // Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
 122         // We use a subdirectory of $GOROOT/pkg because that's the
 123         // space within $GOROOT where we store all generated objects.
 124         // We could use a temporary directory outside $GOROOT instead,
 125         // but it is easier to debug on failure if the files are in a known location.
 126         workspace := pathf("%s/pkg/bootstrap", goroot)
 127         xremoveall(workspace)
 128         xatexit(func() { xremoveall(workspace) })
 129         base := pathf("%s/src/bootstrap", workspace)
 130         xmkdirall(base)
 131
 132         // Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
 133         writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
 134         for _, dir := range bootstrapDirs {
 135                 recurse := strings.HasSuffix(dir, "/...")
 136                 dir = strings.TrimSuffix(dir, "/...")
 137                 filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 138                         if err != nil {
 139                                 fatalf("walking bootstrap dirs failed: %v: %v", path, err)
 140                         }
 141
 142                         name := filepath.Base(path)
 143                         src := pathf("%s/src/%s", goroot, path)
 144                         dst := pathf("%s/%s", base, path)
 145
 146                         if info.IsDir() {
 147                                 if !recurse && path != dir || name == "testdata" {
 148                                         return filepath.SkipDir
 149                                 }
 150
 151                                 xmkdirall(dst)
 152                                 if path == "cmd/cgo" {
 153                                         // Write to src because we need the file both for bootstrap
 154                                         // and for later in the main build.
 155                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
 156                                         mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
 157                                 }
 158                                 return nil
 159                         }
 160
 161                         for _, pre := range ignorePrefixes {
 162                                 if strings.HasPrefix(name, pre) {
 163                                         return nil
 164                                 }
 165                         }
 166                         for _, suf := range ignoreSuffixes {
 167                                 if strings.HasSuffix(name, suf) {
 168                                         return nil
 169                                 }
 170                         }
 171
 172                         text := bootstrapRewriteFile(src)
 173                         writefile(text, dst, 0)
 174                         return nil
 175                 })
 176         }
 177
 178         // Set up environment for invoking Go 1.4 go command.
 179         // GOROOT points at Go 1.4 GOROOT,
 180         // GOPATH points at our bootstrap workspace,
 181         // GOBIN is empty, so that binaries are installed to GOPATH/bin,
 182         // and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
 183         // so that Go 1.4 builds whatever kind of binary it knows how to build.
 184         // Restore GOROOT, GOPATH, and GOBIN when done.
 185         // Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
 186         // because setup will take care of those when bootstrapBuildTools returns.
 187
 188         defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
 189         os.Setenv("GOROOT", goroot_bootstrap)
 190
 191         defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
 192         os.Setenv("GOPATH", workspace)
 193
 194         defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
 195         os.Setenv("GOBIN", "")
 196
 197         os.Setenv("GOOS", "")
 198         os.Setenv("GOHOSTOS", "")
 199         os.Setenv("GOARCH", "")
 200         os.Setenv("GOHOSTARCH", "")
 201
 202         // Run Go 1.4 to build binaries. Use -gcflags=-l to disable inlining to
 203         // workaround bugs in Go 1.4's compiler. See discussion thread:
 204         // https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ
 205         // Use the math_big_pure_go build tag to disable the assembly in math/big
 206         // which may contain unsupported instructions.
 207         // Use the purego build tag to disable other assembly code,
 208         // such as in cmd/internal/notsha256.
 209         // Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l
 210         // only applies to the final cmd/go binary, but that's OK: if this is Go 1.10
 211         // or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler.
 212         cmd := []string{
 213                 pathf("%s/bin/go", goroot_bootstrap),
 214                 "install",
 215                 "-gcflags=-l",
 216                 "-tags=math_big_pure_go compiler_bootstrap purego",
 217         }
 218         if vflag > 0 {
 219                 cmd = append(cmd, "-v")
 220         }
 221         if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
 222                 cmd = append(cmd, "-toolexec="+tool)
 223         }
 224         cmd = append(cmd, "bootstrap/cmd/...")
 225         run(base, ShowOutput|CheckExit, cmd...)
 226
 227         // Copy binaries into tool binary directory.
 228         for _, name := range bootstrapDirs {
 229                 if !strings.HasPrefix(name, "cmd/") {
 230                         continue
 231                 }
 232                 name = name[len("cmd/"):]
 233                 if !strings.Contains(name, "/") {
 234                         copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
 235                 }
 236         }
 237
 238         if vflag > 0 {
 239                 xprintf("\n")
 240         }
 241 }
 242
 243 var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
 244
 245 // isUnneededSSARewriteFile reports whether srcFile is a
 246 // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
 247 // architecture that isn't for the given GOARCH.
 248 //
 249 // When unneeded is true archCaps is the rewrite base filename without
 250 // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
 251 func isUnneededSSARewriteFile(srcFile, goArch string) (archCaps string, unneeded bool) {
 252         if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
 253                 return "", false
 254         }
 255         fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
 256         if fileArch == "" {
 257                 return "", false
 258         }
 259         b := fileArch[0]
 260         if b == '_' || ('a' <= b && b <= 'z') {
 261                 return "", false
 262         }
 263         archCaps = fileArch
 264         fileArch = strings.ToLower(fileArch)
 265         fileArch = strings.TrimSuffix(fileArch, "splitload")
 266         if fileArch == goArch {
 267                 return "", false
 268         }
 269         if fileArch == strings.TrimSuffix(goArch, "le") {
 270                 return "", false
 271         }
 272         return archCaps, true
 273 }
 274
 275 func bootstrapRewriteFile(srcFile string) string {
 276         // During bootstrap, generate dummy rewrite files for
 277         // irrelevant architectures. We only need to build a bootstrap
 278         // binary that works for the current gohostarch.
 279         // This saves 6+ seconds of bootstrap.
 280         if archCaps, ok := isUnneededSSARewriteFile(srcFile, gohostarch); ok {
 281                 return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.
 282
 283 package ssa
 284
 285 func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
 286 func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
 287 `, archCaps, archCaps)
 288         }
 289
 290         return bootstrapFixImports(srcFile)
 291 }
 292
 293 func bootstrapFixImports(srcFile string) string {
 294         text := readfile(srcFile)
 295         if !strings.Contains(srcFile, "/cmd/") && !strings.Contains(srcFile, `\cmd\`) {
 296                 text = regexp.MustCompile(`\bany\b`).ReplaceAllString(text, "interface{}")
 297         }
 298         lines := strings.SplitAfter(text, "\n")
 299         inBlock := false
 300         for i, line := range lines {
 301                 if strings.HasPrefix(line, "import (") {
 302                         inBlock = true
 303                         continue
 304                 }
 305                 if inBlock && strings.HasPrefix(line, ")") {
 306                         inBlock = false
 307                         continue
 308                 }
 309                 if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
 310                         inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"")) {
 311                         line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
 312                         for _, dir := range bootstrapDirs {
 313                                 if strings.HasPrefix(dir, "cmd/") {
 314                                         continue
 315                                 }
 316                                 line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
 317                         }
 318                         lines[i] = line
 319                 }
 320         }
 321
 322         lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
 323
 324         return strings.Join(lines, "")
 325 }