1 // Copyright 2011 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
18 "cmd/go/internal/base"
20 "cmd/go/internal/fsys"
21 "cmd/go/internal/load"
28 // The 'path' used for GOROOT_FINAL when -trimpath is specified
29 const trimPathGoRootFinal = "go"
33 type gcToolchain struct{}
35 func (gcToolchain) compiler() string {
36 return base.Tool("compile")
39 func (gcToolchain) linker() string {
40 return base.Tool("link")
43 func pkgPath(a *Action) string {
46 if cfg.BuildBuildmode == "plugin" {
48 } else if p.Name == "main" && !p.Internal.ForceLibrary {
54 func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg, embedcfg []byte, symabis string, asmhdr bool, gofiles []string) (ofile string, output []byte, err error) {
65 gcargs := []string{"-p", pkgpath}
67 v := p.Module.GoVersion
69 // We started adding a 'go' directive to the go.mod file unconditionally
70 // as of Go 1.12, so any module that still lacks such a directive must
71 // either have been authored before then, or have a hand-edited go.mod
72 // file that hasn't been updated by cmd/go since that edit.
74 // Unfortunately, through at least Go 1.16 we didn't add versions to
75 // vendor/modules.txt. So this could also be a vendored 1.16 dependency.
77 // Fortunately, there were no breaking changes to the language between Go
78 // 1.11 and 1.16, so if we assume Go 1.16 semantics we will not introduce
79 // any spurious errors — we will only mask errors, and not particularly
80 // important ones at that.
83 if allowedVersion(v) {
84 gcargs = append(gcargs, "-lang=go"+v)
88 gcargs = append(gcargs, "-std")
90 compilingRuntime := p.Standard && (p.ImportPath == "runtime" || strings.HasPrefix(p.ImportPath, "runtime/internal"))
91 // The runtime package imports a couple of general internal packages.
92 if p.Standard && (p.ImportPath == "internal/cpu" || p.ImportPath == "internal/bytealg" || p.ImportPath == "internal/abi") {
93 compilingRuntime = true
96 // runtime compiles with a special gc flag to check for
97 // memory allocations that are invalid in the runtime package,
98 // and to implement some special compiler pragmas.
99 gcargs = append(gcargs, "-+")
102 // If we're giving the compiler the entire package (no C etc files), tell it that,
103 // so that it can give good error messages about forward declarations.
104 // Exceptions: a few standard packages have forward declarations for
105 // pieces supplied behind-the-scenes by package runtime.
106 extFiles := len(p.CgoFiles) + len(p.CFiles) + len(p.CXXFiles) + len(p.MFiles) + len(p.FFiles) + len(p.SFiles) + len(p.SysoFiles) + len(p.SwigFiles) + len(p.SwigCXXFiles)
108 switch p.ImportPath {
109 case "bytes", "internal/poll", "net", "os":
111 case "runtime/metrics", "runtime/pprof", "runtime/trace":
113 case "sync", "syscall", "time":
118 gcargs = append(gcargs, "-complete")
120 if cfg.BuildContext.InstallSuffix != "" {
121 gcargs = append(gcargs, "-installsuffix", cfg.BuildContext.InstallSuffix)
124 gcargs = append(gcargs, "-buildid", a.buildID)
126 if p.Internal.OmitDebug || cfg.Goos == "plan9" || cfg.Goarch == "wasm" {
127 gcargs = append(gcargs, "-dwarf=false")
129 if strings.HasPrefix(runtimeVersion, "go1") && !strings.Contains(os.Args[0], "go_bootstrap") {
130 gcargs = append(gcargs, "-goversion", runtimeVersion)
133 gcargs = append(gcargs, "-symabis", symabis)
136 gcflags := str.StringList(forcedGcflags, p.Internal.Gcflags)
137 if compilingRuntime {
138 // Remove -N, if present.
139 // It is not possible to build the runtime with no optimizations,
140 // because the compiler cannot eliminate enough write barriers.
141 for i := 0; i < len(gcflags); i++ {
142 if gcflags[i] == "-N" {
143 copy(gcflags[i:], gcflags[i+1:])
144 gcflags = gcflags[:len(gcflags)-1]
150 args := []interface{}{cfg.BuildToolexec, base.Tool("compile"), "-o", ofile, "-trimpath", a.trimpath(), gcflags, gcargs}
151 if p.Internal.LocalPrefix != "" {
152 // Workaround #43883.
153 args = append(args, "-D", p.Internal.LocalPrefix)
155 if importcfg != nil {
156 if err := b.writeFile(objdir+"importcfg", importcfg); err != nil {
159 args = append(args, "-importcfg", objdir+"importcfg")
162 if err := b.writeFile(objdir+"embedcfg", embedcfg); err != nil {
165 args = append(args, "-embedcfg", objdir+"embedcfg")
167 if ofile == archive {
168 args = append(args, "-pack")
171 args = append(args, "-asmhdr", objdir+"go_asm.h")
174 // Add -c=N to use concurrent backend compilation, if possible.
175 if c := gcBackendConcurrency(gcflags); c > 1 {
176 args = append(args, fmt.Sprintf("-c=%d", c))
179 for _, f := range gofiles {
182 // Handle overlays. Convert path names using OverlayPath
183 // so these paths can be handed directly to tools.
184 // Deleted files won't show up in when scanning directories earlier,
185 // so OverlayPath will never return "" (meaning a deleted file) here.
186 // TODO(#39958): Handle cases where the package directory
187 // doesn't exist on disk (this can happen when all the package's
188 // files are in an overlay): the code expects the package directory
189 // to exist and runs some tools in that directory.
190 // TODO(#39958): Process the overlays when the
191 // gofiles, cgofiles, cfiles, sfiles, and cxxfiles variables are
192 // created in (*Builder).build. Doing that requires rewriting the
193 // code that uses those values to expect absolute paths.
194 f, _ = fsys.OverlayPath(f)
196 args = append(args, f)
199 output, err = b.runOut(a, base.Cwd, nil, args...)
200 return ofile, output, err
203 // gcBackendConcurrency returns the backend compiler concurrency level for a package compilation.
204 func gcBackendConcurrency(gcflags []string) int {
205 // First, check whether we can use -c at all for this compilation.
206 canDashC := concurrentGCBackendCompilationEnabledByDefault
208 switch e := os.Getenv("GO19CONCURRENTCOMPILATION"); e {
214 // Not set. Use default.
216 log.Fatalf("GO19CONCURRENTCOMPILATION must be 0, 1, or unset, got %q", e)
220 for _, flag := range gcflags {
221 // Concurrent compilation is presumed incompatible with any gcflags,
222 // except for known commonly used flags.
223 // If the user knows better, they can manually add their own -c to the gcflags.
225 case "-N", "-l", "-S", "-B", "-C", "-I":
233 // TODO: Test and delete these conditions.
234 if objabi.Experiment.FieldTrack || objabi.Experiment.PreemptibleLoops {
242 // Decide how many concurrent backend compilations to allow.
244 // If we allow too many, in theory we might end up with p concurrent processes,
245 // each with c concurrent backend compiles, all fighting over the same resources.
246 // However, in practice, that seems not to happen too much.
247 // Most build graphs are surprisingly serial, so p==1 for much of the build.
248 // Furthermore, concurrent backend compilation is only enabled for a part
249 // of the overall compiler execution, so c==1 for much of the build.
250 // So don't worry too much about that interaction for now.
252 // However, in practice, setting c above 4 tends not to help very much.
253 // See the analysis in CL 41192.
255 // TODO(josharian): attempt to detect whether this particular compilation
256 // is likely to be a bottleneck, e.g. when:
257 // - it has no successor packages to compile (usually package main)
258 // - all paths through the build graph pass through it
259 // - critical path scheduling says it is high priority
260 // and in such a case, set c to runtime.GOMAXPROCS(0).
261 // By default this is the same as runtime.NumCPU.
262 // We do this now when p==1.
263 // To limit parallelism, set GOMAXPROCS below numCPU; this may be useful
264 // on a low-memory builder, or if a deterministic build order is required.
265 c := runtime.GOMAXPROCS(0)
267 // No process parallelism, do not cap compiler parallelism.
270 // Some process parallelism. Set c to min(4, maxprocs).
277 // trimpath returns the -trimpath argument to use
278 // when compiling the action.
279 func (a *Action) trimpath() string {
280 // Keep in sync with Builder.ccompile
281 // The trimmed paths are a little different, but we need to trim in the
284 // Strip the object directory entirely.
286 if len(objdir) > 1 && objdir[len(objdir)-1] == filepath.Separator {
287 objdir = objdir[:len(objdir)-1]
291 rewriteDir := a.Package.Dir
292 if cfg.BuildTrimpath {
293 importPath := a.Package.Internal.OrigImportPath
294 if m := a.Package.Module; m != nil && m.Version != "" {
295 rewriteDir = m.Path + "@" + m.Version + strings.TrimPrefix(importPath, m.Path)
297 rewriteDir = importPath
299 rewrite += a.Package.Dir + "=>" + rewriteDir + ";"
302 // Add rewrites for overlays. The 'from' and 'to' paths in overlays don't need to have
303 // same basename, so go from the overlay contents file path (passed to the compiler)
304 // to the path the disk path would be rewritten to.
306 cgoFiles := make(map[string]bool)
307 for _, f := range a.Package.CgoFiles {
311 // TODO(matloob): Higher up in the stack, when the logic for deciding when to make copies
312 // of c/c++/m/f/hfiles is consolidated, use the same logic that Build uses to determine
313 // whether to create the copies in objdir to decide whether to rewrite objdir to the
314 // package directory here.
315 var overlayNonGoRewrites string // rewrites for non-go files
316 hasCgoOverlay := false
317 if fsys.OverlayFile != "" {
318 for _, filename := range a.Package.AllFiles() {
320 if !filepath.IsAbs(path) {
321 path = filepath.Join(a.Package.Dir, path)
323 base := filepath.Base(path)
324 isGo := strings.HasSuffix(filename, ".go") || strings.HasSuffix(filename, ".s")
325 isCgo := cgoFiles[filename] || !isGo
326 overlayPath, isOverlay := fsys.OverlayPath(path)
327 if isCgo && isOverlay {
330 if !isCgo && isOverlay {
331 rewrite += overlayPath + "=>" + filepath.Join(rewriteDir, base) + ";"
333 // Generate rewrites for non-Go files copied to files in objdir.
334 if filepath.Dir(path) == a.Package.Dir {
335 // This is a file copied to objdir.
336 overlayNonGoRewrites += filepath.Join(objdir, base) + "=>" + filepath.Join(rewriteDir, base) + ";"
339 // Non-overlay Go files are covered by the a.Package.Dir rewrite rule above.
344 rewrite += overlayNonGoRewrites
346 rewrite += objdir + "=>"
351 func asmArgs(a *Action, p *load.Package) []interface{} {
352 // Add -I pkg/GOOS_GOARCH so #include "textflag.h" works in .s files.
353 inc := filepath.Join(cfg.GOROOT, "pkg", "include")
354 pkgpath := pkgPath(a)
355 args := []interface{}{cfg.BuildToolexec, base.Tool("asm"), "-p", pkgpath, "-trimpath", a.trimpath(), "-I", a.Objdir, "-I", inc, "-D", "GOOS_" + cfg.Goos, "-D", "GOARCH_" + cfg.Goarch, forcedAsmflags, p.Internal.Asmflags}
356 if p.ImportPath == "runtime" && cfg.Goarch == "386" {
357 for _, arg := range forcedAsmflags {
358 if arg == "-dynlink" {
359 args = append(args, "-D=GOBUILDMODE_shared=1")
363 if objabi.IsRuntimePackagePath(pkgpath) {
364 args = append(args, "-compiling-runtime")
367 if cfg.Goarch == "mips" || cfg.Goarch == "mipsle" {
368 // Define GOMIPS_value from cfg.GOMIPS.
369 args = append(args, "-D", "GOMIPS_"+cfg.GOMIPS)
372 if cfg.Goarch == "mips64" || cfg.Goarch == "mips64le" {
373 // Define GOMIPS64_value from cfg.GOMIPS64.
374 args = append(args, "-D", "GOMIPS64_"+cfg.GOMIPS64)
380 func (gcToolchain) asm(b *Builder, a *Action, sfiles []string) ([]string, error) {
382 args := asmArgs(a, p)
385 for _, sfile := range sfiles {
386 overlayPath, _ := fsys.OverlayPath(mkAbs(p.Dir, sfile))
387 ofile := a.Objdir + sfile[:len(sfile)-len(".s")] + ".o"
388 ofiles = append(ofiles, ofile)
389 args1 := append(args, "-o", ofile, overlayPath)
390 if err := b.run(a, p.Dir, p.ImportPath, nil, args1...); err != nil {
397 func (gcToolchain) symabis(b *Builder, a *Action, sfiles []string) (string, error) {
398 mkSymabis := func(p *load.Package, sfiles []string, path string) error {
399 args := asmArgs(a, p)
400 args = append(args, "-gensymabis", "-o", path)
401 for _, sfile := range sfiles {
402 if p.ImportPath == "runtime/cgo" && strings.HasPrefix(sfile, "gcc_") {
405 op, _ := fsys.OverlayPath(mkAbs(p.Dir, sfile))
406 args = append(args, op)
409 // Supply an empty go_asm.h as if the compiler had been run.
410 // -gensymabis parsing is lax enough that we don't need the
411 // actual definitions that would appear in go_asm.h.
412 if err := b.writeFile(a.Objdir+"go_asm.h", nil); err != nil {
416 return b.run(a, p.Dir, p.ImportPath, nil, args...)
419 var symabis string // Only set if we actually create the file
421 if len(sfiles) != 0 {
422 symabis = a.Objdir + "symabis"
423 if err := mkSymabis(p, sfiles, symabis); err != nil {
431 // toolVerify checks that the command line args writes the same output file
432 // if run using newTool instead.
433 // Unused now but kept around for future use.
434 func toolVerify(a *Action, b *Builder, p *load.Package, newTool string, ofile string, args []interface{}) error {
435 newArgs := make([]interface{}, len(args))
437 newArgs[1] = base.Tool(newTool)
438 newArgs[3] = ofile + ".new" // x.6 becomes x.6.new
439 if err := b.run(a, p.Dir, p.ImportPath, nil, newArgs...); err != nil {
442 data1, err := os.ReadFile(ofile)
446 data2, err := os.ReadFile(ofile + ".new")
450 if !bytes.Equal(data1, data2) {
451 return fmt.Errorf("%s and %s produced different output files:\n%s\n%s", filepath.Base(args[1].(string)), newTool, strings.Join(str.StringList(args...), " "), strings.Join(str.StringList(newArgs...), " "))
453 os.Remove(ofile + ".new")
457 func (gcToolchain) pack(b *Builder, a *Action, afile string, ofiles []string) error {
458 var absOfiles []string
459 for _, f := range ofiles {
460 absOfiles = append(absOfiles, mkAbs(a.Objdir, f))
462 absAfile := mkAbs(a.Objdir, afile)
464 // The archive file should have been created by the compiler.
465 // Since it used to not work that way, verify.
467 if _, err := os.Stat(absAfile); err != nil {
468 base.Fatalf("os.Stat of archive file failed: %v", err)
473 if cfg.BuildN || cfg.BuildX {
474 cmdline := str.StringList(base.Tool("pack"), "r", absAfile, absOfiles)
475 b.Showcmd(p.Dir, "%s # internal", joinUnambiguously(cmdline))
480 if err := packInternal(absAfile, absOfiles); err != nil {
481 b.showOutput(a, p.Dir, p.Desc(), err.Error()+"\n")
482 return errPrintedOutput
487 func packInternal(afile string, ofiles []string) error {
488 dst, err := os.OpenFile(afile, os.O_WRONLY|os.O_APPEND, 0)
492 defer dst.Close() // only for error returns or panics
493 w := bufio.NewWriter(dst)
495 for _, ofile := range ofiles {
496 src, err := os.Open(ofile)
500 fi, err := src.Stat()
505 // Note: Not using %-16.16s format because we care
506 // about bytes, not runes.
511 name += strings.Repeat(" ", 16-len(name))
514 fmt.Fprintf(w, "%s%-12d%-6d%-6d%-8o%-10d`\n",
515 name, 0, 0, 0, 0644, size)
516 n, err := io.Copy(w, src)
518 if err == nil && n < size {
519 err = io.ErrUnexpectedEOF
520 } else if err == nil && n > size {
521 err = fmt.Errorf("file larger than size reported by stat")
524 return fmt.Errorf("copying %s to %s: %v", ofile, afile, err)
531 if err := w.Flush(); err != nil {
537 // setextld sets the appropriate linker flags for the specified compiler.
538 func setextld(ldflags []string, compiler []string) []string {
539 for _, f := range ldflags {
540 if f == "-extld" || strings.HasPrefix(f, "-extld=") {
541 // don't override -extld if supplied
545 ldflags = append(ldflags, "-extld="+compiler[0])
546 if len(compiler) > 1 {
548 add := strings.Join(compiler[1:], " ")
549 for i, f := range ldflags {
550 if f == "-extldflags" && i+1 < len(ldflags) {
551 ldflags[i+1] = add + " " + ldflags[i+1]
554 } else if strings.HasPrefix(f, "-extldflags=") {
555 ldflags[i] = "-extldflags=" + add + " " + ldflags[i][len("-extldflags="):]
561 ldflags = append(ldflags, "-extldflags="+add)
567 // pluginPath computes the package path for a plugin main package.
569 // This is typically the import path of the main package p, unless the
570 // plugin is being built directly from source files. In that case we
571 // combine the package build ID with the contents of the main package
572 // source files. This allows us to identify two different plugins
573 // built from two source files with the same name.
574 func pluginPath(a *Action) string {
576 if p.ImportPath != "command-line-arguments" {
581 if a.Mode == "link" {
582 // For linking, use the main package's build ID instead of
583 // the binary's build ID, so it is the same hash used in
584 // compiling and linking.
585 // When compiling, we use actionID/actionID (instead of
586 // actionID/contentID) as a temporary build ID to compute
587 // the hash. Do the same here. (See buildid.go:useCache)
588 // The build ID matters because it affects the overall hash
589 // in the plugin's pseudo-import path returned below.
590 // We need to use the same import path when compiling and linking.
591 id := strings.Split(buildID, buildIDSeparator)
592 buildID = id[1] + buildIDSeparator + id[1]
594 fmt.Fprintf(h, "build ID: %s\n", buildID)
595 for _, file := range str.StringList(p.GoFiles, p.CgoFiles, p.SFiles) {
596 data, err := os.ReadFile(filepath.Join(p.Dir, file))
598 base.Fatalf("go: %s", err)
602 return fmt.Sprintf("plugin/unnamed-%x", h.Sum(nil))
605 func (gcToolchain) ld(b *Builder, root *Action, out, importcfg, mainpkg string) error {
606 cxx := len(root.Package.CXXFiles) > 0 || len(root.Package.SwigCXXFiles) > 0
607 for _, a := range root.Deps {
608 if a.Package != nil && (len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0) {
613 if cfg.BuildContext.InstallSuffix != "" {
614 ldflags = append(ldflags, "-installsuffix", cfg.BuildContext.InstallSuffix)
616 if root.Package.Internal.OmitDebug {
617 ldflags = append(ldflags, "-s", "-w")
619 if cfg.BuildBuildmode == "plugin" {
620 ldflags = append(ldflags, "-pluginpath", pluginPath(root))
623 // Store BuildID inside toolchain binaries as a unique identifier of the
624 // tool being run, for use by content-based staleness determination.
625 if root.Package.Goroot && strings.HasPrefix(root.Package.ImportPath, "cmd/") {
626 // External linking will include our build id in the external
627 // linker's build id, which will cause our build id to not
628 // match the next time the tool is built.
629 // Rely on the external build id instead.
630 if !sys.MustLinkExternal(cfg.Goos, cfg.Goarch) {
631 ldflags = append(ldflags, "-X=cmd/internal/objabi.buildID="+root.buildID)
635 // If the user has not specified the -extld option, then specify the
636 // appropriate linker. In case of C++ code, use the compiler named
637 // by the CXX environment variable or defaultCXX if CXX is not set.
638 // Else, use the CC environment variable and defaultCC as fallback.
639 var compiler []string
641 compiler = envList("CXX", cfg.DefaultCXX(cfg.Goos, cfg.Goarch))
643 compiler = envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch))
645 ldflags = append(ldflags, "-buildmode="+ldBuildmode)
646 if root.buildID != "" {
647 ldflags = append(ldflags, "-buildid="+root.buildID)
649 ldflags = append(ldflags, forcedLdflags...)
650 ldflags = append(ldflags, root.Package.Internal.Ldflags...)
651 ldflags = setextld(ldflags, compiler)
653 // On OS X when using external linking to build a shared library,
654 // the argument passed here to -o ends up recorded in the final
655 // shared library in the LC_ID_DYLIB load command.
656 // To avoid putting the temporary output directory name there
657 // (and making the resulting shared library useless),
658 // run the link in the output directory so that -o can name
659 // just the final path element.
660 // On Windows, DLL file name is recorded in PE file
661 // export section, so do like on OS X.
663 if (cfg.Goos == "darwin" || cfg.Goos == "windows") && cfg.BuildBuildmode == "c-shared" {
664 dir, out = filepath.Split(out)
668 if cfg.BuildTrimpath {
669 env = append(env, "GOROOT_FINAL="+trimPathGoRootFinal)
671 return b.run(root, dir, root.Package.ImportPath, env, cfg.BuildToolexec, base.Tool("link"), "-o", out, "-importcfg", importcfg, ldflags, mainpkg)
674 func (gcToolchain) ldShared(b *Builder, root *Action, toplevelactions []*Action, out, importcfg string, allactions []*Action) error {
675 ldflags := []string{"-installsuffix", cfg.BuildContext.InstallSuffix}
676 ldflags = append(ldflags, "-buildmode=shared")
677 ldflags = append(ldflags, forcedLdflags...)
678 ldflags = append(ldflags, root.Package.Internal.Ldflags...)
680 for _, a := range allactions {
681 if a.Package != nil && (len(a.Package.CXXFiles) > 0 || len(a.Package.SwigCXXFiles) > 0) {
685 // If the user has not specified the -extld option, then specify the
686 // appropriate linker. In case of C++ code, use the compiler named
687 // by the CXX environment variable or defaultCXX if CXX is not set.
688 // Else, use the CC environment variable and defaultCC as fallback.
689 var compiler []string
691 compiler = envList("CXX", cfg.DefaultCXX(cfg.Goos, cfg.Goarch))
693 compiler = envList("CC", cfg.DefaultCC(cfg.Goos, cfg.Goarch))
695 ldflags = setextld(ldflags, compiler)
696 for _, d := range toplevelactions {
697 if !strings.HasSuffix(d.Target, ".a") { // omit unsafe etc and actions for other shared libraries
700 ldflags = append(ldflags, d.Package.ImportPath+"="+d.Target)
702 return b.run(root, ".", out, nil, cfg.BuildToolexec, base.Tool("link"), "-o", out, "-importcfg", importcfg, ldflags)
705 func (gcToolchain) cc(b *Builder, a *Action, ofile, cfile string) error {
706 return fmt.Errorf("%s: C source files not supported without cgo", mkAbs(a.Package.Dir, cfile))