]> Cypherpunks.ru repositories - gostls13.git/commitdiff
cmd/dist: add -distpack flag to build distribution archives
authorRuss Cox <rsc@golang.org>
Thu, 19 Jan 2023 22:04:07 +0000 (17:04 -0500)
committerRuss Cox <rsc@golang.org>
Tue, 11 Apr 2023 21:29:13 +0000 (21:29 +0000)
We want to enable others to reproduce the exact distribution archives
we are serving on go.dev/dl. Today the code for building those
archives lives in golang.org/x/build, which is fundamentally tied to
running on the Go team build infrastructure and not easy for others to
run. This CL adds a new flag -distpack to cmd/dist, usually invoked as
make.bash -distpack, to build the distribution archives using code in
the main repository that anyone can run. Starting in Go 1.21,
the Go team build infrastructure will run this instead of its current
custom code to build those archives.

The current builds are not reproducible even given identical
infrastructure, because the archives are stamped with the current
time. It is helpful to have a timestamp in the archives indicating
when the code is from, but that time needs to be reproducible.
To ensure this, the new -distpack flag extends the VERSION file to
include a time stamp, which it uses as the modification time for all
files in the archive.

The new -distpack flag is implemented by a separate program,
cmd/distpack, instead of being in cmd/dist, so that it can be compiled
by the toolchain being distributed and not the bootstrap toolchain.
Otherwise details like the exact compression algorithms might vary
from one bootstrap toolchain to another and produce non-reproducible
builds. So there is a new 'go tool distpack', but it's omitted from
the distributions themselves, just as 'go tool dist' is.

make.bash already accepts any flags for cmd/dist, including -distpack.
make.bat is less sophisticated and looks for each known flag, so this
CL adds an update to look for -distpack. The CL also changes make.bat
to accept the idiomatic Go -flagname in addition to the non-idiomatic
(for Go) --flagname. Previously it insisted on the --flag form.

I have confirmed that using make.bash -distpack produces the
identical distribution archives for windows/amd64, linux/amd64,
darwin/amd64, and darwin/arm64 whether it is run on
windows/amd64, linux/amd64, or darwin/amd64 hosts.

For #24904.

Change-Id: Ie6d69365ee3d7294d05b4f96ffb9159b41918074
Reviewed-on: https://go-review.googlesource.com/c/go/+/470676
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Heschi Kreinick <heschi@google.com>
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: Carlos Amedee <amedee@google.com>
src/cmd/dist/build.go
src/cmd/distpack/archive.go [new file with mode: 0644]
src/cmd/distpack/archive_test.go [new file with mode: 0644]
src/cmd/distpack/pack.go [new file with mode: 0644]
src/cmd/distpack/test.go [new file with mode: 0644]
src/make.bat

index d82aaa3be2426f6b6f23d0ff37fbc4a5810b43aa..a76c312709111a47159f7f607fb75964b1776c3f 100644 (file)
@@ -351,12 +351,40 @@ func chomp(s string) string {
 }
 
 // findgoversion determines the Go version to use in the version string.
+// It also parses any other metadata found in the version file.
 func findgoversion() string {
        // The $GOROOT/VERSION file takes priority, for distributions
        // without the source repo.
        path := pathf("%s/VERSION", goroot)
        if isfile(path) {
                b := chomp(readfile(path))
+
+               // Starting in Go 1.21 the VERSION file starts with the
+               // version on a line by itself but then can contain other
+               // metadata about the release, one item per line.
+               if i := strings.Index(b, "\n"); i >= 0 {
+                       rest := b[i+1:]
+                       b = chomp(b[:i])
+                       for _, line := range strings.Split(rest, "\n") {
+                               f := strings.Fields(line)
+                               if len(f) == 0 {
+                                       continue
+                               }
+                               switch f[0] {
+                               default:
+                                       fatalf("VERSION: unexpected line: %s", line)
+                               case "time":
+                                       if len(f) != 2 {
+                                               fatalf("VERSION: unexpected time line: %s", line)
+                                       }
+                                       _, err := time.Parse(time.RFC3339, f[1])
+                                       if err != nil {
+                                               fatalf("VERSION: bad time: %s", err)
+                                       }
+                               }
+                       }
+               }
+
                // Commands such as "dist version > VERSION" will cause
                // the shell to create an empty VERSION file and set dist's
                // stdout to its fd. dist in turn looks at VERSION and uses
@@ -591,6 +619,7 @@ func mustLinkExternal(goos, goarch string, cgoEnabled bool) bool {
 // exclude files with that prefix.
 // Note that this table applies only to the build of cmd/go,
 // after the main compiler bootstrap.
+// Files listed here should also be listed in ../distpack/pack.go's srcArch.Remove list.
 var deptab = []struct {
        prefix string   // prefix of target
        dep    []string // dependency tweaks for targets with that prefix
@@ -1206,6 +1235,9 @@ func clean() {
 
                // Remove cached version info.
                xremove(pathf("%s/VERSION.cache", goroot))
+
+               // Remove distribution packages.
+               xremoveall(pathf("%s/pkg/distpack", goroot))
        }
 }
 
@@ -1347,9 +1379,10 @@ func cmdbootstrap() {
        timelog("start", "dist bootstrap")
        defer timelog("end", "dist bootstrap")
 
-       var debug, force, noBanner, noClean bool
+       var debug, distpack, force, noBanner, noClean bool
        flag.BoolVar(&rebuildall, "a", rebuildall, "rebuild all")
        flag.BoolVar(&debug, "d", debug, "enable debugging of bootstrap process")
+       flag.BoolVar(&distpack, "distpack", distpack, "write distribution files to pkg/distpack")
        flag.BoolVar(&force, "force", force, "build even if the port is marked as broken")
        flag.BoolVar(&noBanner, "no-banner", noBanner, "do not print banner")
        flag.BoolVar(&noClean, "no-clean", noClean, "print deprecation warning")
@@ -1592,6 +1625,11 @@ func cmdbootstrap() {
                os.Setenv("CC", oldcc)
        }
 
+       if distpack {
+               xprintf("Packaging archives for %s/%s.\n", goos, goarch)
+               run("", ShowOutput|CheckExit, pathf("%s/distpack", tooldir))
+       }
+
        // Print trailing banner unless instructed otherwise.
        if !noBanner {
                banner()
diff --git a/src/cmd/distpack/archive.go b/src/cmd/distpack/archive.go
new file mode 100644 (file)
index 0000000..2fdc006
--- /dev/null
@@ -0,0 +1,197 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+       "io/fs"
+       "log"
+       "os"
+       "path"
+       "path/filepath"
+       "sort"
+       "strings"
+       "time"
+)
+
+// An Archive describes an archive to write: a collection of files.
+// Directories are implied by the files and not explicitly listed.
+type Archive struct {
+       Files []File
+}
+
+// A File describes a single file to write to an archive.
+type File struct {
+       Name string    // name in archive
+       Time time.Time // modification time
+       Mode fs.FileMode
+       Size int64
+       Src  string // source file in OS file system
+}
+
+// Info returns a FileInfo about the file, for use with tar.FileInfoHeader
+// and zip.FileInfoHeader.
+func (f *File) Info() fs.FileInfo {
+       return fileInfo{f}
+}
+
+// A fileInfo is an implementation of fs.FileInfo describing a File.
+type fileInfo struct {
+       f *File
+}
+
+func (i fileInfo) Name() string       { return path.Base(i.f.Name) }
+func (i fileInfo) ModTime() time.Time { return i.f.Time }
+func (i fileInfo) Mode() fs.FileMode  { return i.f.Mode }
+func (i fileInfo) IsDir() bool        { return false }
+func (i fileInfo) Size() int64        { return i.f.Size }
+func (i fileInfo) Sys() any           { return nil }
+
+// NewArchive returns a new Archive containing all the files in the directory dir.
+// The archive can be amended afterward using methods like Add and Filter.
+func NewArchive(dir string) (*Archive, error) {
+       a := new(Archive)
+       err := fs.WalkDir(os.DirFS(dir), ".", func(name string, d fs.DirEntry, err error) error {
+               if err != nil {
+                       return err
+               }
+               if d.IsDir() {
+                       return nil
+               }
+               info, err := d.Info()
+               if err != nil {
+                       return err
+               }
+               a.Add(name, filepath.Join(dir, name), info)
+               return nil
+       })
+       if err != nil {
+               return nil, err
+       }
+       a.Sort()
+       return a, nil
+}
+
+// Add adds a file with the given name and info to the archive.
+// The content of the file comes from the operating system file src.
+// After a sequence of one or more calls to Add,
+// the caller should invoke Sort to re-sort the archive's files.
+func (a *Archive) Add(name, src string, info fs.FileInfo) {
+       a.Files = append(a.Files, File{
+               Name: name,
+               Time: info.ModTime(),
+               Mode: info.Mode(),
+               Size: info.Size(),
+               Src:  src,
+       })
+}
+
+// Sort sorts the files in the archive.
+// It is only necessary to call Sort after calling Add.
+// ArchiveDir returns a sorted archive, and the other methods
+// preserve the sorting of the archive.
+func (a *Archive) Sort() {
+       sort.Slice(a.Files, func(i, j int) bool {
+               return a.Files[i].Name < a.Files[j].Name
+       })
+}
+
+// Clone returns a copy of the Archive.
+// Method calls like Add and Filter invoked on the copy do not affect the original,
+// nor do calls on the original affect the copy.
+func (a *Archive) Clone() *Archive {
+       b := &Archive{
+               Files: make([]File, len(a.Files)),
+       }
+       copy(b.Files, a.Files)
+       return b
+}
+
+// AddPrefix adds a prefix to all file names in the archive.
+func (a *Archive) AddPrefix(prefix string) {
+       for i := range a.Files {
+               a.Files[i].Name = path.Join(prefix, a.Files[i].Name)
+       }
+}
+
+// Filter removes files from the archive for which keep(name) returns false.
+func (a *Archive) Filter(keep func(name string) bool) {
+       files := a.Files[:0]
+       for _, f := range a.Files {
+               if keep(f.Name) {
+                       files = append(files, f)
+               }
+       }
+       a.Files = files
+}
+
+// SetMode changes the mode of every file in the archive
+// to be mode(name, m), where m is the file's current mode.
+func (a *Archive) SetMode(mode func(name string, m fs.FileMode) fs.FileMode) {
+       for i := range a.Files {
+               a.Files[i].Mode = mode(a.Files[i].Name, a.Files[i].Mode)
+       }
+}
+
+// Remove removes files matching any of the patterns from the archive.
+// The patterns use the syntax of path.Match, with an extension of allowing
+// a leading **/ or trailing /**, which match any number of path elements
+// (including no path elements) before or after the main match.
+func (a *Archive) Remove(patterns ...string) {
+       a.Filter(func(name string) bool {
+               for _, pattern := range patterns {
+                       match, err := amatch(pattern, name)
+                       if err != nil {
+                               log.Fatalf("archive remove: %v", err)
+                       }
+                       if match {
+                               return false
+                       }
+               }
+               return true
+       })
+}
+
+// SetTime sets the modification time of all files in the archive to t.
+func (a *Archive) SetTime(t time.Time) {
+       for i := range a.Files {
+               a.Files[i].Time = t
+       }
+}
+
+func amatch(pattern, name string) (bool, error) {
+       // firstN returns the prefix of name corresponding to the first n path elements.
+       // If n <= 0, firstN returns the entire name.
+       firstN := func(name string, n int) string {
+               for i := 0; i < len(name); i++ {
+                       if name[i] == '/' {
+                               if n--; n == 0 {
+                                       return name[:i]
+                               }
+                       }
+               }
+               return name
+       }
+
+       // lastN returns the suffix of name corresponding to the last n path elements.
+       // If n <= 0, lastN returns the entire name.
+       lastN := func(name string, n int) string {
+               for i := len(name) - 1; i >= 0; i-- {
+                       if name[i] == '/' {
+                               if n--; n == 0 {
+                                       return name[i+1:]
+                               }
+                       }
+               }
+               return name
+       }
+
+       if p, ok := strings.CutPrefix(pattern, "**/"); ok {
+               return path.Match(p, lastN(name, 1+strings.Count(p, "/")))
+       }
+       if p, ok := strings.CutSuffix(pattern, "/**"); ok {
+               return path.Match(p, firstN(name, 1+strings.Count(p, "/")))
+       }
+       return path.Match(pattern, name)
+}
diff --git a/src/cmd/distpack/archive_test.go b/src/cmd/distpack/archive_test.go
new file mode 100644 (file)
index 0000000..620b970
--- /dev/null
@@ -0,0 +1,39 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import "testing"
+
+var amatchTests = []struct {
+       pattern string
+       name    string
+       ok      bool
+}{
+       {"a", "a", true},
+       {"a", "b", false},
+       {"a/**", "a", true},
+       {"a/**", "b", false},
+       {"a/**", "a/b", true},
+       {"a/**", "b/b", false},
+       {"a/**", "a/b/c/d/e/f", true},
+       {"a/**", "z/a/b/c/d/e/f", false},
+       {"**/a", "a", true},
+       {"**/a", "b", false},
+       {"**/a", "x/a", true},
+       {"**/a", "x/a/b", false},
+       {"**/a", "x/y/z/a", true},
+       {"**/a", "x/y/z/a/b", false},
+
+       {"go/pkg/tool/*/compile", "go/pkg/tool/darwin_amd64/compile", true},
+}
+
+func TestAmatch(t *testing.T) {
+       for _, tt := range amatchTests {
+               ok, err := amatch(tt.pattern, tt.name)
+               if ok != tt.ok || err != nil {
+                       t.Errorf("amatch(%q, %q) = %v, %v, want %v, nil", tt.pattern, tt.name, ok, err, tt.ok)
+               }
+       }
+}
diff --git a/src/cmd/distpack/pack.go b/src/cmd/distpack/pack.go
new file mode 100644 (file)
index 0000000..ffeb4a1
--- /dev/null
@@ -0,0 +1,376 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Distpack creates the tgz and zip files for a Go distribution.
+// It writes into GOROOT/pkg/distpack:
+//
+//     - a binary distribution (tgz or zip) for the current GOOS and GOARCH
+//     - a source distribution that is independent of GOOS/GOARCH
+//     - the module mod, info, and zip files for a distribution in module form
+//       (as used by GOTOOLCHAIN support in the go command).
+//
+// Distpack is typically invoked by the -distpack flag to make.bash.
+// A cross-compiled distribution for goos/goarch can be built using:
+//
+//     GOOS=goos GOARCH=goarch ./make.bash -distpack
+//
+package main
+
+import (
+       "archive/tar"
+       "archive/zip"
+       "compress/flate"
+       "compress/gzip"
+       "crypto/sha256"
+       "flag"
+       "fmt"
+       "io"
+       "io/fs"
+       "log"
+       "os"
+       "path"
+       "path/filepath"
+       "runtime"
+       "strings"
+       "time"
+)
+
+func usage() {
+       fmt.Fprintf(os.Stderr, "usage: distpack\n")
+       os.Exit(2)
+}
+
+const (
+       modPath          = "golang.org/toolchain"
+       modVersionPrefix = "v0.0.1"
+)
+
+var (
+       goroot     string
+       gohostos   string
+       gohostarch string
+       goos       string
+       goarch     string
+)
+
+func main() {
+       log.SetPrefix("distpack: ")
+       log.SetFlags(0)
+       flag.Usage = usage
+       flag.Parse()
+       if flag.NArg() != 0 {
+               usage()
+       }
+
+       // Load context.
+       goroot = runtime.GOROOT()
+       if goroot == "" {
+               log.Fatalf("missing $GOROOT")
+       }
+       gohostos = runtime.GOOS
+       gohostarch = runtime.GOARCH
+       goos = os.Getenv("GOOS")
+       if goos == "" {
+               goos = gohostos
+       }
+       goarch = os.Getenv("GOARCH")
+       if goarch == "" {
+               goarch = gohostarch
+       }
+       goosUnderGoarch := goos + "_" + goarch
+       goosDashGoarch := goos + "-" + goarch
+       exe := ""
+       if goos == "windows" {
+               exe = ".exe"
+       }
+       version, versionTime := readVERSION(goroot)
+
+       // Start with files from GOROOT, filtering out non-distribution files.
+       base, err := NewArchive(goroot)
+       if err != nil {
+               log.Fatal(err)
+       }
+       base.SetTime(versionTime)
+       base.SetMode(mode)
+       base.Remove(
+               ".git/**",
+               ".gitattributes",
+               ".github/**",
+               ".gitignore",
+               "VERSION.cache",
+               "misc/cgo/*/_obj/**",
+               "**/.DS_Store",
+               "**/*.exe~", // go.dev/issue/23894
+               // Generated during make.bat/make.bash.
+               "src/cmd/dist/dist",
+               "src/cmd/dist/dist.exe",
+       )
+
+       // The source distribution removes files generated during the release build.
+       // See ../dist/build.go's deptab.
+       srcArch := base.Clone()
+       srcArch.Remove(
+               "bin/**",
+               "pkg/**",
+               // Generated during cmd/dist. See ../dist/build.go:/deptab.
+               "src/cmd/cgo/zdefaultcc.go",
+               "src/cmd/go/internal/cfg/zdefaultcc.go",
+               "src/cmd/go/internal/cfg/zosarch.go",
+               "src/cmd/internal/objabi/zbootstrap.go",
+               "src/go/build/zcgo.go",
+               "src/internal/buildcfg/zbootstrap.go",
+               "src/runtime/internal/sys/zversion.go",
+               "src/time/tzdata/zzipdata.go",
+       )
+       srcArch.AddPrefix("go")
+       testSrc(srcArch)
+
+       // The binary distribution includes only a subset of bin and pkg.
+       binArch := base.Clone()
+       binArch.Filter(func(name string) bool {
+               // Discard bin/ for now, will add back later.
+               if strings.HasPrefix(name, "bin/") {
+                       return false
+               }
+               // Discard most of pkg.
+               if strings.HasPrefix(name, "pkg/") {
+                       // Keep pkg/include.
+                       if strings.HasPrefix(name, "pkg/include/") {
+                               return true
+                       }
+                       // Discard other pkg except pkg/tool.
+                       if !strings.HasPrefix(name, "pkg/tool/") {
+                               return false
+                       }
+                       // Inside pkg/tool, keep only $GOOS_$GOARCH.
+                       if !strings.HasPrefix(name, "pkg/tool/"+goosUnderGoarch+"/") {
+                               return false
+                       }
+                       // Inside pkg/tool/$GOOS_$GOARCH, discard helper tools.
+                       switch strings.TrimSuffix(path.Base(name), ".exe") {
+                       case "api", "dist", "distpack", "metadata":
+                               return false
+                       }
+               }
+               return true
+       })
+
+       // Add go and gofmt to bin, using cross-compiled binaries
+       // if this is a cross-compiled distribution.
+       binExes := []string{
+               "go",
+               "gofmt",
+       }
+       crossBin := "bin"
+       if goos != gohostos || goarch != gohostarch {
+               crossBin = "bin/" + goosUnderGoarch
+       }
+       for _, b := range binExes {
+               name := "bin/" + b + exe
+               src := filepath.Join(goroot, crossBin, b+exe)
+               info, err := os.Stat(src)
+               if err != nil {
+                       log.Fatal(err)
+               }
+               binArch.Add(name, src, info)
+       }
+       binArch.Sort()
+       binArch.SetTime(versionTime) // fix added files
+       binArch.SetMode(mode)        // fix added files
+
+       zipArch := binArch.Clone()
+       zipArch.AddPrefix("go")
+       testZip(zipArch)
+
+       // The module distribution is the binary distribution with unnecessary files removed
+       // and file names using the necessary prefix for the module.
+       modArch := binArch.Clone()
+       modArch.Remove(
+               "api/**",
+               "doc/**",
+               "misc/**",
+               "test/**",
+       )
+       modVers := modVersionPrefix + "-" + version + "." + goosDashGoarch
+       modArch.AddPrefix(modPath + "@" + modVers)
+       testMod(modArch)
+
+       // distpack returns the full path to name in the distpack directory.
+       distpack := func(name string) string {
+               return filepath.Join(goroot, "pkg/distpack", name)
+       }
+       if err := os.MkdirAll(filepath.Join(goroot, "pkg/distpack"), 0777); err != nil {
+               log.Fatal(err)
+       }
+
+       writeTgz(distpack(version+".src.tar.gz"), srcArch)
+
+       if goos == "windows" {
+               writeZip(distpack(version+"."+goos+"-"+goarch+".zip"), zipArch)
+       } else {
+               writeTgz(distpack(version+"."+goos+"-"+goarch+".tar.gz"), zipArch)
+       }
+
+       writeZip(distpack(modVers+".zip"), modArch)
+       writeFile(distpack(modVers+".mod"),
+               []byte(fmt.Sprintf("module %s\n", modPath)))
+       writeFile(distpack(modVers+".info"),
+               []byte(fmt.Sprintf("{%q:%q, %q:%q}\n",
+                       "Version", modVers,
+                       "Time", versionTime.Format(time.RFC3339))))
+}
+
+// mode computes the mode for the given file name.
+func mode(name string, _ fs.FileMode) fs.FileMode {
+       if strings.HasPrefix(name, "bin/") ||
+               strings.HasPrefix(name, "pkg/tool/") ||
+               strings.HasSuffix(name, ".bash") ||
+               strings.HasSuffix(name, ".sh") ||
+               strings.HasSuffix(name, ".pl") ||
+               strings.HasSuffix(name, ".rc") {
+               return 0o755
+       }
+       return 0o644
+}
+
+// readVERSION reads the VERSION file.
+// The first line of the file is the Go version.
+// Additional lines are 'key value' pairs setting other data.
+// The only valid key at the moment is 'time', which sets the modification time for file archives.
+func readVERSION(goroot string) (version string, t time.Time) {
+       data, err := os.ReadFile(filepath.Join(goroot, "VERSION"))
+       if err != nil {
+               log.Fatal(err)
+       }
+       version, rest, _ := strings.Cut(string(data), "\n")
+       for _, line := range strings.Split(rest, "\n") {
+               f := strings.Fields(line)
+               if len(f) == 0 {
+                       continue
+               }
+               switch f[0] {
+               default:
+                       log.Fatalf("VERSION: unexpected line: %s", line)
+               case "time":
+                       if len(f) != 2 {
+                               log.Fatalf("VERSION: unexpected time line: %s", line)
+                       }
+                       t, err = time.ParseInLocation(time.RFC3339, f[1], time.UTC)
+                       if err != nil {
+                               log.Fatalf("VERSION: bad time: %s", err)
+                       }
+               }
+       }
+       return version, t
+}
+
+// writeFile writes a file with the given name and data or fatals.
+func writeFile(name string, data []byte) {
+       if err := os.WriteFile(name, data, 0666); err != nil {
+               log.Fatal(err)
+       }
+       reportHash(name)
+}
+
+// check panics if err is not nil. Otherwise it returns x.
+// It is only meant to be used in a function that has deferred
+// a function to recover appropriately from the panic.
+func check[T any](x T, err error) T {
+       check1(err)
+       return x
+}
+
+// check1 panics if err is not nil.
+// It is only meant to be used in a function that has deferred
+// a function to recover appropriately from the panic.
+func check1(err error) {
+       if err != nil {
+               panic(err)
+       }
+}
+
+// writeTgz writes the archive in tgz form to the file named name.
+func writeTgz(name string, a *Archive) {
+       out, err := os.Create(name)
+       if err != nil {
+               log.Fatal(err)
+       }
+
+       var f File
+       defer func() {
+               if err := recover(); err != nil {
+                       extra := ""
+                       if f.Name != "" {
+                               extra = " " + f.Name
+                       }
+                       log.Fatalf("writing %s%s: %v", name, extra, err)
+               }
+       }()
+
+       zw := check(gzip.NewWriterLevel(out, gzip.BestCompression))
+       tw := tar.NewWriter(zw)
+       for _, f = range a.Files {
+               h := check(tar.FileInfoHeader(f.Info(), ""))
+               h.Name = f.Name
+               if err := tw.WriteHeader(h); err != nil {
+                       panic(err)
+               }
+               r := check(os.Open(f.Src))
+               check(io.Copy(tw, r))
+               check1(r.Close())
+       }
+       f.Name = ""
+       check1(tw.Close())
+       check1(zw.Close())
+       check1(out.Close())
+       reportHash(name)
+}
+
+// writeZip writes the archive in zip form to the file named name.
+func writeZip(name string, a *Archive) {
+       out, err := os.Create(name)
+       if err != nil {
+               log.Fatal(err)
+       }
+
+       var f File
+       defer func() {
+               if err := recover(); err != nil {
+                       extra := ""
+                       if f.Name != "" {
+                               extra = " " + f.Name
+                       }
+                       log.Fatalf("writing %s%s: %v", name, extra, err)
+               }
+       }()
+
+       zw := zip.NewWriter(out)
+       zw.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
+               return flate.NewWriter(out, flate.BestCompression)
+       })
+       for _, f = range a.Files {
+               h := check(zip.FileInfoHeader(f.Info()))
+               h.Name = f.Name
+               h.Method = zip.Deflate
+               w := check(zw.CreateHeader(h))
+               r := check(os.Open(f.Src))
+               check(io.Copy(w, r))
+               check1(r.Close())
+       }
+       f.Name = ""
+       check1(zw.Close())
+       check1(out.Close())
+       reportHash(name)
+}
+
+func reportHash(name string) {
+       f, err := os.Open(name)
+       if err != nil {
+               log.Fatal(err)
+       }
+       h := sha256.New()
+       io.Copy(h, f)
+       f.Close()
+       fmt.Printf("distpack: %x %s\n", h.Sum(nil)[:8], filepath.Base(name))
+}
diff --git a/src/cmd/distpack/test.go b/src/cmd/distpack/test.go
new file mode 100644 (file)
index 0000000..93c6564
--- /dev/null
@@ -0,0 +1,166 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file contains tests applied to the archives before they are written.
+
+package main
+
+import (
+       "bytes"
+       "fmt"
+       "log"
+       "os"
+       "path"
+       "path/filepath"
+       "strings"
+)
+
+type testRule struct {
+       name    string
+       goos    string
+       exclude bool
+}
+
+var srcRules = []testRule{
+       {name: "go/VERSION"},
+       {name: "go/src/cmd/go/main.go"},
+       {name: "go/src/bytes/bytes.go"},
+       {name: "go/.DS_Store", exclude: true},
+       {name: "go/.git", exclude: true},
+       {name: "go/.gitattributes", exclude: true},
+       {name: "go/.github", exclude: true},
+       {name: "go/VERSION.cache", exclude: true},
+       {name: "go/bin/**", exclude: true},
+       {name: "go/pkg/**", exclude: true},
+       {name: "go/src/cmd/dist/dist", exclude: true},
+       {name: "go/src/cmd/dist/dist.exe", exclude: true},
+       {name: "go/src/runtime/internal/sys/zversion.go", exclude: true},
+       {name: "go/src/time/tzdata/zzipdata.go", exclude: true},
+}
+
+var zipRules = []testRule{
+       {name: "go/VERSION"},
+       {name: "go/src/cmd/go/main.go"},
+       {name: "go/src/bytes/bytes.go"},
+
+       {name: "go/.DS_Store", exclude: true},
+       {name: "go/.git", exclude: true},
+       {name: "go/.gitattributes", exclude: true},
+       {name: "go/.github", exclude: true},
+       {name: "go/VERSION.cache", exclude: true},
+       {name: "go/bin", exclude: true},
+       {name: "go/pkg", exclude: true},
+       {name: "go/src/cmd/dist/dist", exclude: true},
+       {name: "go/src/cmd/dist/dist.exe", exclude: true},
+
+       {name: "go/bin/go", goos: "linux"},
+       {name: "go/bin/go", goos: "darwin"},
+       {name: "go/bin/go", goos: "windows", exclude: true},
+       {name: "go/bin/go.exe", goos: "windows"},
+       {name: "go/bin/gofmt", goos: "linux"},
+       {name: "go/bin/gofmt", goos: "darwin"},
+       {name: "go/bin/gofmt", goos: "windows", exclude: true},
+       {name: "go/bin/gofmt.exe", goos: "windows"},
+       {name: "go/pkg/tool/*/compile", goos: "linux"},
+       {name: "go/pkg/tool/*/compile", goos: "darwin"},
+       {name: "go/pkg/tool/*/compile", goos: "windows", exclude: true},
+       {name: "go/pkg/tool/*/compile.exe", goos: "windows"},
+}
+
+var modRules = []testRule{
+       {name: "golang.org/toolchain@*/VERSION"},
+       {name: "golang.org/toolchain@*/src/cmd/go/main.go"},
+       {name: "golang.org/toolchain@*/src/bytes/bytes.go"},
+
+       {name: "golang.org/toolchain@*/.DS_Store", exclude: true},
+       {name: "golang.org/toolchain@*/.git", exclude: true},
+       {name: "golang.org/toolchain@*/.gitattributes", exclude: true},
+       {name: "golang.org/toolchain@*/.github", exclude: true},
+       {name: "golang.org/toolchain@*/VERSION.cache", exclude: true},
+       {name: "golang.org/toolchain@*/bin", exclude: true},
+       {name: "golang.org/toolchain@*/pkg", exclude: true},
+       {name: "golang.org/toolchain@*/src/cmd/dist/dist", exclude: true},
+       {name: "golang.org/toolchain@*/src/cmd/dist/dist.exe", exclude: true},
+
+       {name: "golang.org/toolchain@*/bin/go", goos: "linux"},
+       {name: "golang.org/toolchain@*/bin/go", goos: "darwin"},
+       {name: "golang.org/toolchain@*/bin/go", goos: "windows", exclude: true},
+       {name: "golang.org/toolchain@*/bin/go.exe", goos: "windows"},
+       {name: "golang.org/toolchain@*/bin/gofmt", goos: "linux"},
+       {name: "golang.org/toolchain@*/bin/gofmt", goos: "darwin"},
+       {name: "golang.org/toolchain@*/bin/gofmt", goos: "windows", exclude: true},
+       {name: "golang.org/toolchain@*/bin/gofmt.exe", goos: "windows"},
+       {name: "golang.org/toolchain@*/pkg/tool/*/compile", goos: "linux"},
+       {name: "golang.org/toolchain@*/pkg/tool/*/compile", goos: "darwin"},
+       {name: "golang.org/toolchain@*/pkg/tool/*/compile", goos: "windows", exclude: true},
+       {name: "golang.org/toolchain@*/pkg/tool/*/compile.exe", goos: "windows"},
+}
+
+func testSrc(a *Archive) {
+       test("source", a, srcRules)
+
+       // Check that no generated files slip in, even if new ones are added.
+       for _, f := range a.Files {
+               if strings.HasPrefix(path.Base(f.Name), "z") {
+                       data, err := os.ReadFile(filepath.Join(goroot, strings.TrimPrefix(f.Name, "go/")))
+                       if err != nil {
+                               log.Fatalf("checking source archive: %v", err)
+                       }
+                       if strings.Contains(string(data), "generated by go tool dist; DO NOT EDIT") {
+                               log.Fatalf("unexpected source archive file: %s (generated by dist)", f.Name)
+                       }
+               }
+       }
+}
+
+func testZip(a *Archive) { test("binary", a, zipRules) }
+func testMod(a *Archive) { test("module", a, modRules) }
+
+func test(kind string, a *Archive, rules []testRule) {
+       ok := true
+       have := make([]bool, len(rules))
+       for _, f := range a.Files {
+               for i, r := range rules {
+                       if r.goos != "" && r.goos != goos {
+                               continue
+                       }
+                       match, err := amatch(r.name, f.Name)
+                       if err != nil {
+                               log.Fatal(err)
+                       }
+                       if match {
+                               if r.exclude {
+                                       ok = false
+                                       if !have[i] {
+                                               log.Printf("unexpected %s archive file: %s", kind, f.Name)
+                                               have[i] = true // silence future prints for excluded directory
+                                       }
+                               } else {
+                                       have[i] = true
+                               }
+                       }
+               }
+       }
+       missing := false
+       for i, r := range rules {
+               if r.goos != "" && r.goos != goos {
+                       continue
+               }
+               if !r.exclude && !have[i] {
+                       missing = true
+                       log.Printf("missing %s archive file: %s", kind, r.name)
+               }
+       }
+       if missing {
+               ok = false
+               var buf bytes.Buffer
+               for _, f := range a.Files {
+                       fmt.Fprintf(&buf, "\n\t%s", f.Name)
+               }
+               log.Printf("archive contents: %d files%s", len(a.Files), buf.Bytes())
+       }
+       if !ok {
+               log.Fatalf("bad archive file")
+       }
+}
index 814d12c3005c9316956d9613ef2d4fdcd03b2e0c..3b861cb91d4acb4b527e98b305c01ab64ca1eb16 100644 (file)
 \r
 :: Keep environment variables within this script\r
 :: unless invoked with --no-local.\r
+if x%1==x-no-local goto nolocal\r
+if x%2==x-no-local goto nolocal\r
+if x%3==x-no-local goto nolocal\r
+if x%4==x-no-local goto nolocal\r
 if x%1==x--no-local goto nolocal\r
 if x%2==x--no-local goto nolocal\r
 if x%3==x--no-local goto nolocal\r
@@ -113,20 +117,40 @@ call .\env.bat
 del env.bat\r
 if x%vflag==x-v echo.\r
 \r
+if x%1==x-dist-tool goto copydist\r
+if x%2==x-dist-tool goto copydist\r
+if x%3==x-dist-tool goto copydist\r
+if x%4==x-dist-tool goto copydist\r
 if x%1==x--dist-tool goto copydist\r
 if x%2==x--dist-tool goto copydist\r
 if x%3==x--dist-tool goto copydist\r
 if x%4==x--dist-tool goto copydist\r
 \r
 set bootstrapflags=\r
-if x%1==x--no-clean set bootstrapflags=--no-clean\r
-if x%2==x--no-clean set bootstrapflags=--no-clean\r
-if x%3==x--no-clean set bootstrapflags=--no-clean\r
-if x%4==x--no-clean set bootstrapflags=--no-clean\r
-if x%1==x--no-banner set bootstrapflags=%bootstrapflags% --no-banner\r
-if x%2==x--no-banner set bootstrapflags=%bootstrapflags% --no-banner\r
-if x%3==x--no-banner set bootstrapflags=%bootstrapflags% --no-banner\r
-if x%4==x--no-banner set bootstrapflags=%bootstrapflags% --no-banner\r
+if x%1==x-no-clean set bootstrapflags=-no-clean\r
+if x%2==x-no-clean set bootstrapflags=-no-clean\r
+if x%3==x-no-clean set bootstrapflags=-no-clean\r
+if x%4==x-no-clean set bootstrapflags=-no-clean\r
+if x%1==x--no-clean set bootstrapflags=-no-clean\r
+if x%2==x--no-clean set bootstrapflags=-no-clean\r
+if x%3==x--no-clean set bootstrapflags=-no-clean\r
+if x%4==x--no-clean set bootstrapflags=-no-clean\r
+if x%1==x-no-banner set bootstrapflags=%bootstrapflags% -no-banner\r
+if x%2==x-no-banner set bootstrapflags=%bootstrapflags% -no-banner\r
+if x%3==x-no-banner set bootstrapflags=%bootstrapflags% -no-banner\r
+if x%4==x-no-banner set bootstrapflags=%bootstrapflags% -no-banner\r
+if x%1==x--no-banner set bootstrapflags=%bootstrapflags% -no-banner\r
+if x%2==x--no-banner set bootstrapflags=%bootstrapflags% -no-banner\r
+if x%3==x--no-banner set bootstrapflags=%bootstrapflags% -no-banner\r
+if x%4==x--no-banner set bootstrapflags=%bootstrapflags% -no-banner\r
+if x%1==x-distpack set bootstrapflags=%bootstrapflags% -distpack\r
+if x%2==x-distpack set bootstrapflags=%bootstrapflags% -distpack\r
+if x%3==x-distpack set bootstrapflags=%bootstrapflags% -distpack\r
+if x%4==x-distpack set bootstrapflags=%bootstrapflags% -distpack\r
+if x%1==x--distpack set bootstrapflags=%bootstrapflags% -distpack\r
+if x%2==x--distpack set bootstrapflags=%bootstrapflags% -distpack\r
+if x%3==x--distpack set bootstrapflags=%bootstrapflags% -distpack\r
+if x%4==x--distpack set bootstrapflags=%bootstrapflags% -distpack\r
 \r
 :: Run dist bootstrap to complete make.bash.\r
 :: Bootstrap installs a proper cmd/dist, built with the new toolchain.\r