load1 := func(path string, mode int) *load.Package {
if parent == nil {
mode := 0 // don't do module or vendor resolution
- return load.LoadImport(context.TODO(), path, base.Cwd, nil, stk, nil, mode)
+ return load.LoadImport(path, base.Cwd, nil, stk, nil, mode)
}
- return load.LoadImport(context.TODO(), path, parent.Dir, parent, stk, nil, mode|load.ResolveModule)
+ return load.LoadImport(path, parent.Dir, parent, stk, nil, mode|load.ResolveModule)
}
p := load1(arg, mode)
"cmd/go/internal/cache"
"cmd/go/internal/cfg"
"cmd/go/internal/load"
- "cmd/go/internal/modinfo"
"cmd/go/internal/modload"
"cmd/go/internal/str"
"cmd/go/internal/work"
fm := template.FuncMap{
"join": strings.Join,
"context": context,
- "module": func(path string) *modinfo.ModulePublic { return modload.ModuleInfo(ctx, path) },
+ "module": modload.ModuleInfo,
}
tmpl, err := template.New("main").Funcs(fm).Parse(*listFmt)
if err != nil {
base.Fatalf("go list -m: not using modules")
}
- modload.InitMod(ctx) // Parses go.mod and sets cfg.BuildMod.
+ modload.InitMod() // Parses go.mod and sets cfg.BuildMod.
if cfg.BuildMod == "vendor" {
const actionDisabledFormat = "go list -m: can't %s using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)"
ModBinDir func() string // return effective bin directory
ModLookup func(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) // lookup effective meaning of import
ModPackageModuleInfo func(path string) *modinfo.ModulePublic // return module info for Package struct
- ModImportPaths func(ctx context.Context, args []string) []*search.Match // expand import paths
+ ModImportPaths func(args []string) []*search.Match // expand import paths
ModPackageBuildInfo func(main string, deps []string) string // return module info to embed in binary
ModInfoProg func(info string, isgccgo bool) []byte // wrap module info in .go code for binary
- ModImportFromFiles func(context.Context, []string) // update go.mod to add modules for imports in these files
+ ModImportFromFiles func([]string) // update go.mod to add modules for imports in these files
ModDirImportPath func(string) string // return effective import path for directory
)
})
packageDataCache.Delete(p.ImportPath)
}
- return LoadImport(context.TODO(), arg, base.Cwd, nil, stk, nil, 0)
+ return LoadImport(arg, base.Cwd, nil, stk, nil, 0)
}
// dirToImportPath returns the pseudo-import path we use for a package
// LoadImport does not set tool flags and should only be used by
// this package, as part of a bigger load operation, and by GOPATH-based "go get".
// TODO(rsc): When GOPATH-based "go get" is removed, unexport this function.
-func LoadImport(ctx context.Context, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
- return loadImport(ctx, nil, path, srcDir, parent, stk, importPos, mode)
+func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
+ return loadImport(nil, path, srcDir, parent, stk, importPos, mode)
}
-func loadImport(ctx context.Context, pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
+func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
if path == "" {
panic("LoadImport called with empty package path")
}
// Load package.
// loadPackageData may return bp != nil even if an error occurs,
// in order to return partial information.
- p.load(ctx, path, stk, importPos, bp, err)
+ p.load(path, stk, importPos, bp, err)
if !cfg.ModulesEnabled && path != cleanImport(path) {
p.Error = &PackageError{
// load populates p using information from bp, err, which should
// be the result of calling build.Context.Import.
// stk contains the import stack, not including path itself.
-func (p *Package) load(ctx context.Context, path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) {
+func (p *Package) load(path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) {
p.copyBuild(bp)
// The localPrefix is the path we interpret ./ imports relative to.
if path == "C" {
continue
}
- p1 := LoadImport(ctx, path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport)
+ p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport)
path = p1.ImportPath
importPaths[i] = path
// TestPackageList returns the list of packages in the dag rooted at roots
// as visited in a depth-first post-order traversal, including the test
// imports of the roots. This ignores errors in test packages.
-func TestPackageList(ctx context.Context, roots []*Package) []*Package {
+func TestPackageList(roots []*Package) []*Package {
seen := map[*Package]bool{}
all := []*Package{}
var walk func(*Package)
}
walkTest := func(root *Package, path string) {
var stk ImportStack
- p1 := LoadImport(ctx, path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport)
+ p1 := LoadImport(path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport)
if p1.Error == nil {
walk(p1)
}
// TODO(jayconrod): delete this function and set flags automatically
// in LoadImport instead.
func LoadImportWithFlags(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package {
- p := LoadImport(context.TODO(), path, srcDir, parent, stk, importPos, mode)
+ p := LoadImport(path, srcDir, parent, stk, importPos, mode)
setToolFlags(p)
return p
}
// We need to test whether the path is an actual Go file and not a
// package path or pattern ending in '.go' (see golang.org/issue/34653).
if fi, err := os.Stat(p); err == nil && !fi.IsDir() {
- return []*Package{GoFilesPackage(ctx, patterns)}
+ return []*Package{GoFilesPackage(patterns)}
}
}
}
- matches := ImportPaths(ctx, patterns)
+ matches := ImportPaths(patterns)
var (
pkgs []*Package
stk ImportStack
if pkg == "" {
panic(fmt.Sprintf("ImportPaths returned empty package for pattern %s", m.Pattern()))
}
- p := loadImport(ctx, pre, pkg, base.Cwd, nil, &stk, nil, 0)
+ p := loadImport(pre, pkg, base.Cwd, nil, &stk, nil, 0)
p.Match = append(p.Match, m.Pattern())
p.Internal.CmdlinePkg = true
if m.IsLiteral() {
}
}
-func ImportPaths(ctx context.Context, args []string) []*search.Match {
+func ImportPaths(args []string) []*search.Match {
if ModInit(); cfg.ModulesEnabled {
- return ModImportPaths(ctx, args)
+ return ModImportPaths(args)
}
return search.ImportPaths(args)
}
// GoFilesPackage creates a package for building a collection of Go files
// (typically named on the command line). The target is named p.a for
// package p or named after the first Go file for package main.
-func GoFilesPackage(ctx context.Context, gofiles []string) *Package {
+func GoFilesPackage(gofiles []string) *Package {
ModInit()
for _, f := range gofiles {
ctxt.ReadDir = func(string) ([]os.FileInfo, error) { return dirent, nil }
if cfg.ModulesEnabled {
- ModImportFromFiles(ctx, gofiles)
+ ModImportFromFiles(gofiles)
}
var err error
pkg := new(Package)
pkg.Internal.Local = true
pkg.Internal.CmdlineFiles = true
- pkg.load(ctx, "command-line-arguments", &stk, nil, bp, err)
+ pkg.load("command-line-arguments", &stk, nil, bp, err)
pkg.Internal.LocalPrefix = dirToImportPath(dir)
pkg.ImportPath = "command-line-arguments"
pkg.Target = ""
stk.Push(p.ImportPath + " (test)")
rawTestImports := str.StringList(p.TestImports)
for i, path := range p.TestImports {
- p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport)
+ p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport)
if str.Contains(p1.Deps, p.ImportPath) || p1.ImportPath == p.ImportPath {
// Same error that loadPackage returns (via reusePackage) in pkg.go.
// Can't change that code, because that code is only for loading the
pxtestNeedsPtest := false
rawXTestImports := str.StringList(p.XTestImports)
for i, path := range p.XTestImports {
- p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport)
+ p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport)
if p1.ImportPath == p.ImportPath {
pxtestNeedsPtest = true
} else {
if dep == ptest.ImportPath {
pmain.Internal.Imports = append(pmain.Internal.Imports, ptest)
} else {
- p1 := loadImport(ctx, pre, dep, "", nil, &stk, nil, 0)
+ p1 := loadImport(pre, dep, "", nil, &stk, nil, 0)
pmain.Internal.Imports = append(pmain.Internal.Imports, p1)
}
}
package modcmd
import (
- "cmd/go/internal/modfetch"
"context"
"encoding/json"
"os"
- "runtime"
"cmd/go/internal/base"
"cmd/go/internal/cfg"
+ "cmd/go/internal/modfetch"
"cmd/go/internal/modload"
+ "cmd/go/internal/par"
"cmd/go/internal/work"
"golang.org/x/mod/module"
if len(args) == 0 {
args = []string{"all"}
} else if modload.HasModRoot() {
- modload.InitMod(ctx) // to fill Target
+ modload.InitMod() // to fill Target
targetAtLatest := modload.Target.Path + "@latest"
targetAtUpgrade := modload.Target.Path + "@upgrade"
targetAtPatch := modload.Target.Path + "@patch"
}
}
- downloadModule := func(m *moduleJSON) {
+ var mods []*moduleJSON
+ var work par.Work
+ listU := false
+ listVersions := false
+ for _, info := range modload.ListModules(ctx, args, listU, listVersions) {
+ if info.Replace != nil {
+ info = info.Replace
+ }
+ if info.Version == "" && info.Error == nil {
+ // main module or module replaced with file path.
+ // Nothing to download.
+ continue
+ }
+ m := &moduleJSON{
+ Path: info.Path,
+ Version: info.Version,
+ }
+ mods = append(mods, m)
+ if info.Error != nil {
+ m.Error = info.Error.Err
+ continue
+ }
+ work.Add(m)
+ }
+
+ work.Do(10, func(item interface{}) {
+ m := item.(*moduleJSON)
var err error
m.Info, err = modfetch.InfoFile(m.Path, m.Version)
if err != nil {
return
}
mod := module.Version{Path: m.Path, Version: m.Version}
- m.Zip, err = modfetch.DownloadZip(ctx, mod)
+ m.Zip, err = modfetch.DownloadZip(mod)
if err != nil {
m.Error = err.Error()
return
}
m.Sum = modfetch.Sum(mod)
- m.Dir, err = modfetch.Download(ctx, mod)
+ m.Dir, err = modfetch.Download(mod)
if err != nil {
m.Error = err.Error()
return
}
- }
-
- var mods []*moduleJSON
- listU := false
- listVersions := false
- type token struct{}
- sem := make(chan token, runtime.GOMAXPROCS(0))
- for _, info := range modload.ListModules(ctx, args, listU, listVersions) {
- if info.Replace != nil {
- info = info.Replace
- }
- if info.Version == "" && info.Error == nil {
- // main module or module replaced with file path.
- // Nothing to download.
- continue
- }
- m := &moduleJSON{
- Path: info.Path,
- Version: info.Version,
- }
- mods = append(mods, m)
- if info.Error != nil {
- m.Error = info.Error.Err
- continue
- }
- sem <- token{}
- go func() {
- downloadModule(m)
- <-sem
- }()
- }
-
- // Fill semaphore channel to wait for goroutines to finish.
- for n := cap(sem); n > 0; n-- {
- sem <- token{}
- }
+ })
if *downloadJSON {
for _, m := range mods {
"cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/go/internal/modload"
+ "cmd/go/internal/par"
"cmd/go/internal/work"
"golang.org/x/mod/module"
return m.Path + "@" + m.Version
}
+ // Note: using par.Work only to manage work queue.
+ // No parallelism here, so no locking.
var out []string
var deps int // index in out where deps start
- seen := map[module.Version]bool{modload.Target: true}
- queue := []module.Version{modload.Target}
- for len(queue) > 0 {
- var m module.Version
- m, queue = queue[0], queue[1:]
+ var work par.Work
+ work.Add(modload.Target)
+ work.Do(1, func(item interface{}) {
+ m := item.(module.Version)
list, _ := reqs.Required(m)
for _, r := range list {
- if !seen[r] {
- queue = append(queue, r)
- seen[r] = true
- }
+ work.Add(r)
out = append(out, format(m)+" "+format(r)+"\n")
}
if m == modload.Target {
deps = len(out)
}
- }
+ })
sort.Slice(out[deps:], func(i, j int) bool {
return out[deps+i][0] < out[deps+j][0]
if strings.Contains(modload.CmdModModule, "@") {
base.Fatalf("go mod init: module path must not contain '@'")
}
- modload.InitMod(ctx) // does all the hard work
+ modload.InitMod() // does all the hard work
modload.WriteGoMod()
}
base.Fatalf("go mod tidy: no arguments allowed")
}
- modload.LoadALL(ctx)
+ modload.LoadALL()
modload.TidyBuildList()
modload.TrimGoSum()
modload.WriteGoMod()
if len(args) != 0 {
base.Fatalf("go mod vendor: vendor takes no arguments")
}
- pkgs := modload.LoadVendor(ctx)
+ pkgs := modload.LoadVendor()
vdir := filepath.Join(modload.ModRoot(), "vendor")
if err := os.RemoveAll(vdir); err != nil {
}
mods := modload.ListModules(ctx, args, listU, listVersions)
byModule := make(map[module.Version][]string)
- for _, path := range loadALL(ctx) {
+ for _, path := range loadALL() {
m := modload.PackageModule(path)
if m.Path != "" {
byModule[m] = append(byModule[m], path)
sep = "\n"
}
} else {
- matches := modload.ImportPaths(ctx, args) // resolve to packages
- loadALL(ctx) // rebuild graph, from main module (not from named packages)
+ matches := modload.ImportPaths(args) // resolve to packages
+ loadALL() // rebuild graph, from main module (not from named packages)
sep := ""
for _, m := range matches {
for _, path := range m.Pkgs {
import (
"fmt"
"os"
- "runtime"
"sort"
"strings"
+ "sync"
"cmd/go/internal/base"
"cmd/go/internal/modfetch"
+ "cmd/go/internal/par"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
// Convert requirements block, which may use raw SHA1 hashes as versions,
// to valid semver requirement list, respecting major versions.
- versions := make([]*module.Version, len(mf.Require))
- replace := make(map[string]*modfile.Replace)
+ var (
+ work par.Work
+ mu sync.Mutex
+ need = make(map[string]string)
+ replace = make(map[string]*modfile.Replace)
+ )
for _, r := range mf.Replace {
replace[r.New.Path] = r
replace[r.Old.Path] = r
}
-
- type token struct{}
- sem := make(chan token, runtime.GOMAXPROCS(0))
- for i, r := range mf.Require {
+ for _, r := range mf.Require {
m := r.Mod
if m.Path == "" {
continue
}
if re, ok := replace[m.Path]; ok {
- m = re.New
+ work.Add(re.New)
+ continue
}
- sem <- token{}
- go func(i int, m module.Version) {
- repo, info, err := modfetch.ImportRepoRev(m.Path, m.Version)
- if err != nil {
- fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), m.Path, m.Version, err)
- return
- }
-
- path := repo.ModulePath()
- versions[i].Path = path
- versions[i].Version = info.Version
-
- <-sem
- }(i, m)
- }
- // Fill semaphore channel to wait for all tasks to finish.
- for n := cap(sem); n > 0; n-- {
- sem <- token{}
+ work.Add(r.Mod)
}
- need := map[string]string{}
- for _, v := range versions {
+ work.Do(10, func(item interface{}) {
+ r := item.(module.Version)
+ repo, info, err := modfetch.ImportRepoRev(r.Path, r.Version)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), r.Path, r.Version, err)
+ return
+ }
+ mu.Lock()
+ path := repo.ModulePath()
// Don't use semver.Max here; need to preserve +incompatible suffix.
- if needv, ok := need[v.Path]; !ok || semver.Compare(needv, v.Version) < 0 {
- need[v.Path] = v.Version
+ if v, ok := need[path]; !ok || semver.Compare(v, info.Version) < 0 {
+ need[path] = info.Version
}
- }
- paths := make([]string, 0, len(need))
+ mu.Unlock()
+ })
+
+ var paths []string
for path := range need {
paths = append(paths, path)
}
import (
"bytes"
- "context"
"fmt"
"internal/testenv"
"io/ioutil"
},
}
- ctx := context.Background()
-
for _, tt := range tests {
t.Run(strings.ReplaceAll(tt.path, "/", "_")+"_"+tt.vers, func(t *testing.T) {
f, err := modfile.Parse("golden", []byte(tt.gomod), nil)
t.Fatal(err)
}
- dir, err := modfetch.Download(ctx, module.Version{Path: tt.path, Version: tt.vers})
+ dir, err := modfetch.Download(module.Version{Path: tt.path, Version: tt.vers})
if err != nil {
t.Fatal(err)
}
import (
"archive/zip"
"bytes"
- "context"
"errors"
"fmt"
"io"
"cmd/go/internal/par"
"cmd/go/internal/renameio"
"cmd/go/internal/robustio"
- "cmd/go/internal/trace"
"golang.org/x/mod/module"
"golang.org/x/mod/sumdb/dirhash"
// Download downloads the specific module version to the
// local download cache and returns the name of the directory
// corresponding to the root of the module's file tree.
-func Download(ctx context.Context, mod module.Version) (dir string, err error) {
+func Download(mod module.Version) (dir string, err error) {
if cfg.GOMODCACHE == "" {
// modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE
// is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen.
err error
}
c := downloadCache.Do(mod, func() interface{} {
- dir, err := download(ctx, mod)
+ dir, err := download(mod)
if err != nil {
return cached{"", err}
}
return c.dir, c.err
}
-func download(ctx context.Context, mod module.Version) (dir string, err error) {
- ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String())
- defer span.Done()
-
+func download(mod module.Version) (dir string, err error) {
// If the directory exists, and no .partial file exists, the module has
// already been completely extracted. .partial files may be created when a
// module zip directory is extracted in place instead of being extracted to a
// To avoid cluttering the cache with extraneous files,
// DownloadZip uses the same lockfile as Download.
// Invoke DownloadZip before locking the file.
- zipfile, err := DownloadZip(ctx, mod)
+ zipfile, err := DownloadZip(mod)
if err != nil {
return "", err
}
return "", err
}
- ctx, span = trace.StartSpan(ctx, "unzip "+zipfile)
if unzipInPlace {
if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil {
return "", err
return "", err
}
}
- defer span.Done()
if !cfg.ModCacheRW {
// Make dir read-only only *after* renaming it.
// DownloadZip downloads the specific module version to the
// local zip cache and returns the name of the zip file.
-func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) {
+func DownloadZip(mod module.Version) (zipfile string, err error) {
// The par.Cache here avoids duplicate work.
type cached struct {
zipfile string
if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
return cached{"", err}
}
- if err := downloadZip(ctx, mod, zipfile); err != nil {
+ if err := downloadZip(mod, zipfile); err != nil {
return cached{"", err}
}
return cached{zipfile, nil}
return c.zipfile, c.err
}
-func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) {
- ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile)
- defer span.Done()
-
+func downloadZip(mod module.Version, zipfile string) (err error) {
// Clean up any remaining tempfiles from previous runs.
// This is only safe to do because the lock file ensures that their
// writers are no longer active.
package zip_sum_test
import (
- "context"
"crypto/sha256"
"encoding/csv"
"encoding/hex"
name := fmt.Sprintf("%s@%s", strings.ReplaceAll(test.m.Path, "/", "_"), test.m.Version)
t.Run(name, func(t *testing.T) {
t.Parallel()
- zipPath, err := modfetch.DownloadZip(context.Background(), test.m)
+ zipPath, err := modfetch.DownloadZip(test.m)
if err != nil {
if *updateTestData {
t.Logf("%s: could not download module: %s (will remove from testdata)", test.m, err)
"fmt"
"os"
"path/filepath"
- "runtime"
"sort"
"strings"
"sync"
"cmd/go/internal/load"
"cmd/go/internal/modload"
"cmd/go/internal/mvs"
+ "cmd/go/internal/par"
"cmd/go/internal/search"
"cmd/go/internal/work"
if !strings.Contains(path, "...") {
m := search.NewMatch(path)
if pkgPath := modload.DirImportPath(path); pkgPath != "." {
- m = modload.TargetPackages(ctx, pkgPath)
+ m = modload.TargetPackages(pkgPath)
}
if len(m.Pkgs) == 0 {
for _, err := range m.Errs {
default:
// The argument is a package or module path.
if modload.HasModRoot() {
- if m := modload.TargetPackages(ctx, path); len(m.Pkgs) != 0 {
+ if m := modload.TargetPackages(path); len(m.Pkgs) != 0 {
// The path is in the main module. Nothing to query.
if vers != "upgrade" && vers != "patch" {
base.Errorf("go get %s: can't request explicit version of path in main module", arg)
if q.path == q.m.Path {
wg.Add(1)
go func(q *query) {
- if hasPkg, err := modload.ModuleHasRootPackage(ctx, q.m); err != nil {
+ if hasPkg, err := modload.ModuleHasRootPackage(q.m); err != nil {
base.Errorf("go get: %v", err)
} else if !hasPkg {
modOnlyMu.Lock()
// Don't load packages if pkgPatterns is empty. Both
// modload.ImportPathsQuiet and ModulePackages convert an empty list
// of patterns to []string{"."}, which is not what we want.
- matches = modload.ImportPathsQuiet(ctx, pkgPatterns, imports.AnyTags())
+ matches = modload.ImportPathsQuiet(pkgPatterns, imports.AnyTags())
seenPkgs = make(map[string]bool)
for i, match := range matches {
arg := pkgGets[i]
// reported. A map from module paths to queries is returned, which includes
// queries and modOnly.
func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*query, modOnly map[string]*query) map[string]*query {
+ var lookup par.Work
+ for _, q := range queries {
+ if cached := cache[q.querySpec]; cached != nil {
+ *q = *cached
+ } else {
+ cache[q.querySpec] = q
+ lookup.Add(q)
+ }
+ }
- runQuery := func(q *query) {
+ lookup.Do(10, func(item interface{}) {
+ q := item.(*query)
if q.vers == "none" {
// Wait for downgrade step.
q.m = module.Version{Path: q.path, Version: "none"}
return
}
- m, err := getQuery(ctx, q.path, q.vers, q.prevM, q.forceModulePath)
+ m, err := getQuery(q.path, q.vers, q.prevM, q.forceModulePath)
if err != nil {
base.Errorf("go get %s: %v", q.arg, err)
}
q.m = m
- }
-
- type token struct{}
- sem := make(chan token, runtime.GOMAXPROCS(0))
- for _, q := range queries {
- if cached := cache[q.querySpec]; cached != nil {
- *q = *cached
- } else {
- sem <- token{}
- go func(q *query) {
- runQuery(q)
- <-sem
- }(q)
- }
- }
-
- // Fill semaphore channel to wait for goroutines to finish.
- for n := cap(sem); n > 0; n-- {
- sem <- token{}
- }
-
- // Add to cache after concurrent section to avoid races...
- for _, q := range queries {
- cache[q.querySpec] = q
- }
-
+ })
base.ExitIfErrors()
byPath := make(map[string]*query)
// to determine the underlying module version being requested.
// If forceModulePath is set, getQuery must interpret path
// as a module path.
-func getQuery(ctx context.Context, path, vers string, prevM module.Version, forceModulePath bool) (module.Version, error) {
+func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (module.Version, error) {
if (prevM.Version != "") != forceModulePath {
// We resolve package patterns by calling QueryPattern, which does not
// accept a previous version and therefore cannot take it into account for
}
}
- info, err := modload.Query(ctx, path, vers, prevM.Version, modload.Allowed)
+ info, err := modload.Query(path, vers, prevM.Version, modload.Allowed)
if err == nil {
if info.Version != vers && info.Version != prevM.Version {
logOncef("go: %s %s => %s", path, vers, info.Version)
// If it turns out to only exist as a module, we can detect the resulting
// PackageNotInModuleError and avoid a second round-trip through (potentially)
// all of the configured proxies.
- results, err := modload.QueryPattern(ctx, path, vers, modload.Allowed)
+ results, err := modload.QueryPattern(path, vers, modload.Allowed)
if err != nil {
// If the path doesn't contain a wildcard, check whether it was actually a
// module path instead. If so, return that.
// If we're querying "upgrade" or "patch", Query will compare the current
// version against the chosen version and will return the current version
// if it is newer.
- info, err := modload.Query(context.TODO(), m.Path, string(getU), m.Version, modload.Allowed)
+ info, err := modload.Query(m.Path, string(getU), m.Version, modload.Allowed)
if err != nil {
// Report error but return m, to let version selection continue.
// (Reporting the error will fail the command at the next base.ExitIfErrors.)
import (
"bytes"
- "context"
"encoding/hex"
"fmt"
"internal/goroot"
if !ok {
return nil
}
- return moduleInfo(context.TODO(), m, true)
+ return moduleInfo(m, true)
}
-func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic {
+func ModuleInfo(path string) *modinfo.ModulePublic {
if !Enabled() {
return nil
}
if i := strings.Index(path, "@"); i >= 0 {
- return moduleInfo(ctx, module.Version{Path: path[:i], Version: path[i+1:]}, false)
+ return moduleInfo(module.Version{Path: path[:i], Version: path[i+1:]}, false)
}
for _, m := range BuildList() {
if m.Path == path {
- return moduleInfo(ctx, m, true)
+ return moduleInfo(m, true)
}
}
}
// addUpdate fills in m.Update if an updated version is available.
-func addUpdate(ctx context.Context, m *modinfo.ModulePublic) {
+func addUpdate(m *modinfo.ModulePublic) {
if m.Version == "" {
return
}
- if info, err := Query(ctx, m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 {
+ if info, err := Query(m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 {
m.Update = &modinfo.ModulePublic{
Path: m.Path,
Version: info.Version,
m.Versions, _ = versions(m.Path)
}
-func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modinfo.ModulePublic {
+func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic {
if m == Target {
info := &modinfo.ModulePublic{
Path: m.Path,
// completeFromModCache fills in the extra fields in m using the module cache.
completeFromModCache := func(m *modinfo.ModulePublic) {
if m.Version != "" {
- if q, err := Query(ctx, m.Path, m.Version, "", nil); err != nil {
+ if q, err := Query(m.Path, m.Version, "", nil); err != nil {
m.Error = &modinfo.ModuleError{Err: err.Error()}
} else {
m.Version = q.Version
package modload
import (
- "context"
"errors"
"fmt"
"go/build"
// Import returns an ImportMissingError as the error.
// If Import can identify a module that could be added to supply the package,
// the ImportMissingError records that module.
-func Import(ctx context.Context, path string) (m module.Version, dir string, err error) {
+func Import(path string) (m module.Version, dir string, err error) {
if strings.Contains(path, "@") {
return module.Version{}, "", fmt.Errorf("import path should not have @version")
}
// Avoid possibly downloading irrelevant modules.
continue
}
- root, isLocal, err := fetch(ctx, m)
+ root, isLocal, err := fetch(m)
if err != nil {
// Report fetch error.
// Note that we don't know for sure this module is necessary,
return len(mods[i].Path) > len(mods[j].Path)
})
for _, m := range mods {
- root, isLocal, err := fetch(ctx, m)
+ root, isLocal, err := fetch(m)
if err != nil {
// Report fetch error as above.
return module.Version{}, "", err
fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path)
- candidates, err := QueryPackage(ctx, path, "latest", Allowed)
+ candidates, err := QueryPackage(path, "latest", Allowed)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
// Return "cannot find module providing package […]" instead of whatever
package modload
import (
- "context"
"internal/testenv"
"regexp"
"strings"
}(allowMissingModuleImports)
AllowMissingModuleImports()
- ctx := context.Background()
-
for _, tt := range importTests {
t.Run(strings.ReplaceAll(tt.path, "/", "_"), func(t *testing.T) {
// Note that there is no build list, so Import should always fail.
- m, dir, err := Import(ctx, tt.path)
+ m, dir, err := Import(tt.path)
if err == nil {
t.Fatalf("Import(%q) = %v, %v, nil; expected error", tt.path, m, dir)
}
import (
"bytes"
- "context"
"encoding/json"
"errors"
"fmt"
//
// As a side-effect, InitMod sets a default for cfg.BuildMod if it does not
// already have an explicit value.
-func InitMod(ctx context.Context) {
+func InitMod() {
if len(buildList) > 0 {
return
}
}
var fixed bool
- f, err := modfile.Parse(gomod, data, fixVersion(ctx, &fixed))
+ f, err := modfile.Parse(gomod, data, fixVersion(&fixed))
if err != nil {
// Errors returned by modfile.Parse begin with file:line.
base.Fatalf("go: errors parsing go.mod:\n%s\n", err)
// and does nothing for versions that already appear to be canonical.
//
// The VersionFixer sets 'fixed' if it ever returns a non-canonical version.
-func fixVersion(ctx context.Context, fixed *bool) modfile.VersionFixer {
+func fixVersion(fixed *bool) modfile.VersionFixer {
return func(path, vers string) (resolved string, err error) {
defer func() {
if err == nil && resolved != vers {
}
}
- info, err := Query(ctx, path, vers, "", nil)
+ info, err := Query(path, vers, "", nil)
if err != nil {
return "", err
}
"errors"
"fmt"
"os"
- "runtime"
"strings"
"cmd/go/internal/base"
"cmd/go/internal/cfg"
"cmd/go/internal/modinfo"
+ "cmd/go/internal/par"
"cmd/go/internal/search"
"golang.org/x/mod/module"
func ListModules(ctx context.Context, args []string, listU, listVersions bool) []*modinfo.ModulePublic {
mods := listModules(ctx, args, listVersions)
-
- type token struct{}
- sem := make(chan token, runtime.GOMAXPROCS(0))
if listU || listVersions {
+ var work par.Work
for _, m := range mods {
- add := func(m *modinfo.ModulePublic) {
- sem <- token{}
- go func() {
- if listU {
- addUpdate(ctx, m)
- }
- if listVersions {
- addVersions(m)
- }
- <-sem
- }()
- }
-
- add(m)
+ work.Add(m)
if m.Replace != nil {
- add(m.Replace)
+ work.Add(m.Replace)
}
}
+ work.Do(10, func(item interface{}) {
+ m := item.(*modinfo.ModulePublic)
+ if listU {
+ addUpdate(m)
+ }
+ if listVersions {
+ addVersions(m)
+ }
+ })
}
- // Fill semaphore channel to wait for all tasks to finish.
- for n := cap(sem); n > 0; n-- {
- sem <- token{}
- }
-
return mods
}
func listModules(ctx context.Context, args []string, listVersions bool) []*modinfo.ModulePublic {
LoadBuildList(ctx)
if len(args) == 0 {
- return []*modinfo.ModulePublic{moduleInfo(ctx, buildList[0], true)}
+ return []*modinfo.ModulePublic{moduleInfo(buildList[0], true)}
}
var mods []*modinfo.ModulePublic
}
}
- info, err := Query(ctx, path, vers, current, nil)
+ info, err := Query(path, vers, current, nil)
if err != nil {
mods = append(mods, &modinfo.ModulePublic{
Path: path,
})
continue
}
- mods = append(mods, moduleInfo(ctx, module.Version{Path: path, Version: info.Version}, false))
+ mods = append(mods, moduleInfo(module.Version{Path: path, Version: info.Version}, false))
continue
}
matched = true
if !matchedBuildList[i] {
matchedBuildList[i] = true
- mods = append(mods, moduleInfo(ctx, m, true))
+ mods = append(mods, moduleInfo(m, true))
}
}
}
// Don't make the user provide an explicit '@latest' when they're
// explicitly asking what the available versions are.
// Instead, resolve the module, even if it isn't an existing dependency.
- info, err := Query(ctx, arg, "latest", "", nil)
+ info, err := Query(arg, "latest", "", nil)
if err == nil {
- mods = append(mods, moduleInfo(ctx, module.Version{Path: arg, Version: info.Version}, false))
+ mods = append(mods, moduleInfo(module.Version{Path: arg, Version: info.Version}, false))
} else {
mods = append(mods, &modinfo.ModulePublic{
Path: arg,
// ImportPaths returns the set of packages matching the args (patterns),
// on the target platform. Modules may be added to the build list
// to satisfy new imports.
-func ImportPaths(ctx context.Context, patterns []string) []*search.Match {
- matches := ImportPathsQuiet(ctx, patterns, imports.Tags())
+func ImportPaths(patterns []string) []*search.Match {
+ matches := ImportPathsQuiet(patterns, imports.Tags())
search.WarnUnmatched(matches)
return matches
}
// no matches. It also lets the caller specify a set of build tags to match
// packages. The build tags should typically be imports.Tags() or
// imports.AnyTags(); a nil map has no special meaning.
-func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bool) []*search.Match {
+func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match {
updateMatches := func(matches []*search.Match, iterating bool) {
for _, m := range matches {
switch {
case strings.Contains(m.Pattern(), "..."):
m.Errs = m.Errs[:0]
- matchPackages(ctx, m, loaded.tags, includeStd, buildList)
+ matchPackages(m, loaded.tags, includeStd, buildList)
case m.Pattern() == "all":
loaded.testAll = true
// Enumerate the packages in the main module.
// We'll load the dependencies as we find them.
m.Errs = m.Errs[:0]
- matchPackages(ctx, m, loaded.tags, omitStd, []module.Version{Target})
+ matchPackages(m, loaded.tags, omitStd, []module.Version{Target})
} else {
// Starting with the packages in the main module,
// enumerate the full list of "all".
}
}
- InitMod(ctx)
+ InitMod()
var matches []*search.Match
for _, pattern := range search.CleanPatterns(patterns) {
// ImportFromFiles adds modules to the build list as needed
// to satisfy the imports in the named Go source files.
-func ImportFromFiles(ctx context.Context, gofiles []string) {
- InitMod(ctx)
+func ImportFromFiles(gofiles []string) {
+ InitMod()
tags := imports.Tags()
imports, testImports, err := imports.ScanFiles(gofiles, tags)
func LoadBuildList(ctx context.Context) []module.Version {
ctx, span := trace.StartSpan(ctx, "LoadBuildList")
defer span.Done()
- InitMod(ctx)
+ InitMod()
ReloadBuildList()
WriteGoMod()
return buildList
// It adds modules to the build list as needed to satisfy new imports.
// This set is useful for deciding whether a particular import is needed
// anywhere in a module.
-func LoadALL(ctx context.Context) []string {
- return loadAll(ctx, true)
+func LoadALL() []string {
+ return loadAll(true)
}
// LoadVendor is like LoadALL but only follows test dependencies
// for tests in the main module. Tests in dependency modules are
// ignored completely.
// This set is useful for identifying the which packages to include in a vendor directory.
-func LoadVendor(ctx context.Context) []string {
- return loadAll(ctx, false)
+func LoadVendor() []string {
+ return loadAll(false)
}
-func loadAll(ctx context.Context, testAll bool) []string {
- InitMod(ctx)
+func loadAll(testAll bool) []string {
+ InitMod()
loaded = newLoader(imports.AnyTags())
loaded.isALL = true
if !testAll {
loaded.testRoots = true
}
- all := TargetPackages(ctx, "...")
+ all := TargetPackages("...")
loaded.load(func() []string { return all.Pkgs })
checkMultiplePaths()
WriteGoMod()
// TargetPackages returns the list of packages in the target (top-level) module
// matching pattern, which may be relative to the working directory, under all
// build tag settings.
-func TargetPackages(ctx context.Context, pattern string) *search.Match {
+func TargetPackages(pattern string) *search.Match {
// TargetPackages is relative to the main module, so ensure that the main
// module is a thing that can contain packages.
ModRoot()
m := search.NewMatch(pattern)
- matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{Target})
+ matchPackages(m, imports.AnyTags(), omitStd, []module.Version{Target})
return m
}
return
}
- // TODO(matloob): Handle TODO context. This needs to be threaded through Do.
- pkg.mod, pkg.dir, pkg.err = Import(context.TODO(), pkg.path)
+ pkg.mod, pkg.dir, pkg.err = Import(pkg.path)
if pkg.dir == "" {
return
}
package modload
import (
- "context"
"errors"
"fmt"
"os"
//
// The isLocal return value reports whether the replacement,
// if any, is local to the filesystem.
-func fetch(ctx context.Context, mod module.Version) (dir string, isLocal bool, err error) {
+func fetch(mod module.Version) (dir string, isLocal bool, err error) {
if mod == Target {
return ModRoot(), true, nil
}
mod = r
}
- dir, err = modfetch.Download(ctx, mod)
+ dir, err = modfetch.Download(mod)
return dir, false, err
}
package modload
import (
- "context"
"errors"
"fmt"
"os"
"cmd/go/internal/modfetch"
"cmd/go/internal/search"
"cmd/go/internal/str"
- "cmd/go/internal/trace"
"golang.org/x/mod/module"
"golang.org/x/mod/semver"
//
// If path is the path of the main module and the query is "latest",
// Query returns Target.Version as the version.
-func Query(ctx context.Context, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) {
+func Query(path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) {
var info *modfetch.RevInfo
err := modfetch.TryProxies(func(proxy string) (err error) {
- info, err = queryProxy(ctx, proxy, path, query, current, allowed)
+ info, err = queryProxy(proxy, path, query, current, allowed)
return err
})
return info, err
return fmt.Sprintf("cannot query module due to -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason)
}
-func queryProxy(ctx context.Context, proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) {
- ctx, span := trace.StartSpan(ctx, "modload.queryProxy "+path+" "+query)
- defer span.Done()
-
+func queryProxy(proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) {
if current != "" && !semver.IsValid(current) {
return nil, fmt.Errorf("invalid previous version %q", current)
}
if err != nil {
return nil, err
}
- releases, prereleases, err := filterVersions(ctx, path, versions, ok, preferIncompatible)
+ releases, prereleases, err := filterVersions(path, versions, ok, preferIncompatible)
if err != nil {
return nil, err
}
// 1. versions that do not satisfy the 'ok' predicate, and
// 2. "+incompatible" versions, if a compatible one satisfies the predicate
// and the incompatible version is not preferred.
-func filterVersions(ctx context.Context, path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) {
+func filterVersions(path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) {
var lastCompatible string
for _, v := range versions {
if !ok(module.Version{Path: path, Version: v}) {
// https://golang.org/issue/34165.) Note that we even prefer a
// compatible pre-release over an incompatible release.
- ok, err := versionHasGoMod(ctx, module.Version{Path: path, Version: lastCompatible})
+ ok, err := versionHasGoMod(module.Version{Path: path, Version: lastCompatible})
if err != nil {
return nil, nil, err
}
// If the package is in the main module, QueryPackage considers only the main
// module and only the version "latest", without checking for other possible
// modules.
-func QueryPackage(ctx context.Context, path, query string, allowed func(module.Version) bool) ([]QueryResult, error) {
+func QueryPackage(path, query string, allowed func(module.Version) bool) ([]QueryResult, error) {
m := search.NewMatch(path)
if m.IsLocal() || !m.IsLiteral() {
return nil, fmt.Errorf("pattern %s is not an importable package", path)
}
- return QueryPattern(ctx, path, query, allowed)
+ return QueryPattern(path, query, allowed)
}
// QueryPattern looks up the module(s) containing at least one package matching
// If any matching package is in the main module, QueryPattern considers only
// the main module and only the version "latest", without checking for other
// possible modules.
-func QueryPattern(ctx context.Context, pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) {
- ctx, span := trace.StartSpan(ctx, "modload.QueryPattern "+pattern+" "+query)
- defer span.Done()
-
+func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) {
base := pattern
firstError := func(m *search.Match) error {
base = pathpkg.Dir(pattern[:i+3])
match = func(mod module.Version, root string, isLocal bool) *search.Match {
m := search.NewMatch(pattern)
- matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{mod})
+ matchPackages(m, imports.AnyTags(), omitStd, []module.Version{mod})
return m
}
} else {
}
err := modfetch.TryProxies(func(proxy string) error {
- queryModule := func(ctx context.Context, path string) (r QueryResult, err error) {
- ctx, span := trace.StartSpan(ctx, "modload.QueryPattern.queryModule ["+proxy+"] "+path)
- defer span.Done()
-
+ queryModule := func(path string) (r QueryResult, err error) {
current := findCurrentVersion(path)
r.Mod.Path = path
- r.Rev, err = queryProxy(ctx, proxy, path, query, current, allowed)
+ r.Rev, err = queryProxy(proxy, path, query, current, allowed)
if err != nil {
return r, err
}
r.Mod.Version = r.Rev.Version
- root, isLocal, err := fetch(ctx, r.Mod)
+ root, isLocal, err := fetch(r.Mod)
if err != nil {
return r, err
}
}
var err error
- results, err = queryPrefixModules(ctx, candidateModules, queryModule)
+ results, err = queryPrefixModules(candidateModules, queryModule)
return err
})
err error
}
-func queryPrefixModules(ctx context.Context, candidateModules []string, queryModule func(ctx context.Context, path string) (QueryResult, error)) (found []QueryResult, err error) {
- ctx, span := trace.StartSpan(ctx, "modload.queryPrefixModules")
- defer span.Done()
-
+func queryPrefixModules(candidateModules []string, queryModule func(path string) (QueryResult, error)) (found []QueryResult, err error) {
// If the path we're attempting is not in the module cache and we don't have a
// fetch result cached either, we'll end up making a (potentially slow)
// request to the proxy or (often even slower) the origin server.
var wg sync.WaitGroup
wg.Add(len(candidateModules))
for i, p := range candidateModules {
- ctx := trace.StartGoroutine(ctx)
go func(p string, r *result) {
- r.QueryResult, r.err = queryModule(ctx, p)
+ r.QueryResult, r.err = queryModule(p)
wg.Done()
}(p, &results[i])
}
}
// ModuleHasRootPackage returns whether module m contains a package m.Path.
-func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) {
- root, isLocal, err := fetch(ctx, m)
+func ModuleHasRootPackage(m module.Version) (bool, error) {
+ root, isLocal, err := fetch(m)
if err != nil {
return false, err
}
return ok, err
}
-func versionHasGoMod(ctx context.Context, m module.Version) (bool, error) {
- root, _, err := fetch(ctx, m)
+func versionHasGoMod(m module.Version) (bool, error) {
+ root, _, err := fetch(m)
if err != nil {
return false, err
}
package modload
import (
- "context"
"internal/testenv"
"io/ioutil"
"log"
testenv.MustHaveExternalNetwork(t)
testenv.MustHaveExecPath(t, "git")
- ctx := context.Background()
-
for _, tt := range queryTests {
allow := tt.allow
if allow == "" {
t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.query+"/"+tt.current+"/"+allow, func(t *testing.T) {
t.Parallel()
- info, err := Query(ctx, tt.path, tt.query, tt.current, allowed)
+ info, err := Query(tt.path, tt.query, tt.current, allowed)
if tt.err != "" {
if err == nil {
t.Errorf("Query(%q, %q, %v) = %v, want error %q", tt.path, tt.query, allow, info.Version, tt.err)
package modload
import (
- "context"
"fmt"
"os"
"path/filepath"
// matchPackages is like m.MatchPackages, but uses a local variable (rather than
// a global) for tags, can include or exclude packages in the standard library,
// and is restricted to the given list of modules.
-func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) {
+func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) {
m.Pkgs = []string{}
isMatch := func(string) bool { return true }
isLocal = true
} else {
var err error
- root, isLocal, err = fetch(ctx, mod)
+ root, isLocal, err = fetch(mod)
if err != nil {
m.AddError(err)
continue
base.Fatalf("go run: cannot run *_test.go files (%s)", file)
}
}
- p = load.GoFilesPackage(ctx, files)
+ p = load.GoFilesPackage(files)
} else if len(args) > 0 && !strings.HasPrefix(args[0], "-") {
pkgs := load.PackagesAndErrors(ctx, args[:1])
if len(pkgs) == 0 {
}
// Select for coverage all dependencies matching the testCoverPaths patterns.
- for _, p := range load.TestPackageList(ctx, pkgs) {
+ for _, p := range load.TestPackageList(pkgs) {
haveMatch := false
for i := range testCoverPaths {
if match[i](p) {
}
srcs := []string{src}
- p := load.GoFilesPackage(context.TODO(), srcs)
+ p := load.GoFilesPackage(srcs)
if _, _, e := BuildToolchain.gc(b, &Action{Mode: "swigDoIntSize", Package: p, Objdir: objdir}, "", nil, "", false, srcs); e != nil {
return "32", nil