// Package modget implements the module-aware ``go get'' command.
package modget
+// The arguments to 'go get' are patterns with optional version queries, with
+// the version queries defaulting to "upgrade".
+//
+// The patterns are normally interpreted as package patterns. However, if a
+// pattern cannot match a package, it is instead interpreted as a *module*
+// pattern. For version queries such as "upgrade" and "patch" that depend on the
+// selected version of a module (or of the module containing a package),
+// whether a pattern denotes a package or module may change as updates are
+// applied (see the example in mod_get_patchmod.txt).
+//
+// There are a few other ambiguous cases to resolve, too. A package can exist in
+// two different modules at the same version: for example, the package
+// example.com/foo might be found in module example.com and also in module
+// example.com/foo, and those modules may have independent v0.1.0 tags — so the
+// input 'example.com/foo@v0.1.0' could syntactically refer to the variant of
+// the package loaded from either module! (See mod_get_ambiguous_pkg.txt.)
+// If the argument is ambiguous, the user can often disambiguate by specifying
+// explicit versions for *all* of the potential module paths involved.
+
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
+ "reflect"
"runtime"
"sort"
"strings"
"cmd/go/internal/imports"
"cmd/go/internal/load"
"cmd/go/internal/modload"
- "cmd/go/internal/mvs"
+ "cmd/go/internal/par"
"cmd/go/internal/search"
"cmd/go/internal/work"
"golang.org/x/mod/module"
- "golang.org/x/mod/semver"
)
var CmdGet = &base.Command{
dependency should be removed entirely, downgrading or removing modules
depending on it as needed.
-The version suffix @latest explicitly requests the latest minor release of the
+The version suffix @latest explicitly requests the latest minor release ofthe
module named by the given path. The suffix @upgrade is like @latest but
will not downgrade a module if it is already required at a revision or
pre-release version newer than the latest released version. The suffix
@patch requests the latest patch release: the latest released version
with the same major and minor version numbers as the currently required
version. Like @upgrade, @patch will not downgrade a module already required
-at a newer version. If the path is not already required, @upgrade and @patch
-are equivalent to @latest.
+at a newer version. If the path is not already required, @upgrade is
+equivalent to @latest, and @patch is disallowed.
Although get defaults to using the latest version of the module containing
a named package, it does not use the latest version of that module's
)
// upgradeFlag is a custom flag.Value for -u.
-type upgradeFlag string
+type upgradeFlag struct {
+ rawVersion string
+ version string
+}
func (*upgradeFlag) IsBoolFlag() bool { return true } // allow -u
func (v *upgradeFlag) Set(s string) error {
if s == "false" {
- s = ""
+ v.version = ""
+ v.rawVersion = ""
+ } else if s == "true" {
+ v.version = "upgrade"
+ v.rawVersion = ""
+ } else {
+ v.version = s
+ v.rawVersion = s
}
- if s == "true" {
- s = "upgrade"
- }
- *v = upgradeFlag(s)
return nil
}
CmdGet.Flag.Var(&getU, "u", "")
}
-// A getArg holds a parsed positional argument for go get (path@vers).
-type getArg struct {
- // raw is the original argument, to be printed in error messages.
- raw string
-
- // path is the part of the argument before "@" (or the whole argument
- // if there is no "@"). path specifies the modules or packages to get.
- path string
-
- // vers is the part of the argument after "@" or an implied
- // "upgrade" or "patch" if there is no "@". vers specifies the
- // module version to get.
- vers string
-}
-
-func (arg getArg) String() string { return arg.raw }
-
-// querySpec describes a query for a specific module. path may be a
-// module path, package path, or package pattern. vers is a version
-// query string from a command line argument.
-type querySpec struct {
- // path is a module path, package path, or package pattern that
- // specifies which module to query.
- path string
-
- // vers specifies what version of the module to get.
- vers string
-
- // forceModulePath is true if path should be interpreted as a module path.
- // If forceModulePath is true, prevM must be set.
- forceModulePath bool
-
- // prevM is the previous version of the module. prevM is needed
- // to determine the minor version number if vers is "patch". It's also
- // used to avoid downgrades from prerelease versions newer than
- // "latest" and "patch". If prevM is set, forceModulePath must be true.
- prevM module.Version
-}
-
-// query holds the state for a query made for a specific module.
-// After a query is performed, we know the actual module path and
-// version and whether any packages were matched by the query path.
-type query struct {
- querySpec
-
- // arg is the command line argument that matched the specified module.
- arg string
-
- // m is the module path and version found by the query.
- m module.Version
-}
-
func runGet(ctx context.Context, cmd *base.Command, args []string) {
- switch getU {
+ switch getU.version {
case "", "upgrade", "patch":
// ok
default:
- base.Fatalf("go get: unknown upgrade flag -u=%s", getU)
+ base.Fatalf("go get: unknown upgrade flag -u=%s", getU.rawVersion)
}
if *getF {
fmt.Fprintf(os.Stderr, "go get: -f flag is a no-op when using modules\n")
// 'go get' is expected to do this, unlike other commands.
modload.AllowMissingModuleImports()
- getArgs := parseArgs(args)
-
- buildList := modload.LoadAllModules(ctx)
- buildList = buildList[:len(buildList):len(buildList)] // copy on append
- selectedVersion := make(map[string]string)
- for _, m := range buildList {
- selectedVersion[m.Path] = m.Version
- }
- queries := classifyArgs(ctx, selectedVersion, getArgs)
+ modload.LoadModFile(ctx) // Initializes modload.Target.
- // Query modules referenced by command line arguments at requested versions.
- // We need to do this before loading packages since patterns that refer to
- // packages in unknown modules can't be expanded. This also avoids looking
- // up new modules while loading packages, only to downgrade later.
- queryCache := make(map[querySpec]*query)
- byPath := runQueries(ctx, queryCache, queries, nil)
-
- // Add missing modules to the build list.
- // We call SetBuildList here and elsewhere, since newUpgrader,
- // LoadPackages, and other functions read the global build list.
- for _, q := range queries {
- if _, ok := selectedVersion[q.m.Path]; !ok && q.m.Version != "none" {
- buildList = append(buildList, q.m)
- }
- }
- selectedVersion = nil // out of date now; rebuilt later when needed
- modload.SetBuildList(buildList)
-
- // Upgrade modules specifically named on the command line. This is our only
- // chance to upgrade modules without root packages (modOnly below).
- // This also skips loading packages at an old version, only to upgrade
- // and reload at a new version.
- upgrade := make(map[string]*query)
- for path, q := range byPath {
- if q.path == q.m.Path && q.m.Version != "none" {
- upgrade[path] = q
- }
- }
- buildList, err := mvs.UpgradeAll(modload.Target, newUpgrader(upgrade, nil))
- if err != nil {
- base.Fatalf("go get: %v", err)
- }
- modload.SetBuildList(buildList)
- base.ExitIfErrors()
- prevBuildList := buildList
-
- // Build a set of module paths that we don't plan to load packages from.
- // This includes explicitly requested modules that don't have a root package
- // and modules with a target version of "none".
- var wg sync.WaitGroup
- var modOnlyMu sync.Mutex
- modOnly := make(map[string]*query)
- for _, q := range queries {
- if q.m.Version == "none" {
- modOnlyMu.Lock()
- modOnly[q.m.Path] = q
- modOnlyMu.Unlock()
- continue
- }
- if q.path == q.m.Path {
- wg.Add(1)
- go func(q *query) {
- if hasPkg, err := modload.ModuleHasRootPackage(ctx, q.m); err != nil {
- base.Errorf("go get: %v", err)
- } else if !hasPkg {
- modOnlyMu.Lock()
- modOnly[q.m.Path] = q
- modOnlyMu.Unlock()
- }
- wg.Done()
- }(q)
- }
- }
- wg.Wait()
- base.ExitIfErrors()
+ queries := parseArgs(ctx, args)
- // Build a list of arguments that may refer to packages.
- var pkgPatterns []string
- var pkgGets []getArg
- for _, arg := range getArgs {
- if modOnly[arg.path] == nil && arg.vers != "none" {
- pkgPatterns = append(pkgPatterns, arg.path)
- pkgGets = append(pkgGets, arg)
- }
- }
+ r := newResolver(ctx, queries)
+ r.performLocalQueries(ctx)
+ r.performPathQueries(ctx)
- // Load packages and upgrade the modules that provide them. We do this until
- // we reach a fixed point, since modules providing packages may change as we
- // change versions. This must terminate because the module graph is finite,
- // and the load and upgrade operations may only add and upgrade modules
- // in the build list.
- var matches []*search.Match
for {
- var seenPkgs map[string]bool
- seenQuery := make(map[querySpec]bool)
- var queries []*query
- addQuery := func(q *query) {
- if !seenQuery[q.querySpec] {
- seenQuery[q.querySpec] = true
- queries = append(queries, q)
- }
- }
-
- if len(pkgPatterns) > 0 {
- // Don't load packages if pkgPatterns is empty. Both
- // modload.LoadPackages and ModulePackages convert an empty list
- // of patterns to []string{"."}, which is not what we want.
- loadOpts := modload.PackageOpts{
- Tags: imports.AnyTags(),
- ResolveMissingImports: true, // dubious; see https://golang.org/issue/32567
- LoadTests: *getT,
- SilenceErrors: true, // Errors may be fixed by subsequent upgrades or downgrades.
- SilenceUnmatchedWarnings: true, // We will warn after iterating below.
- }
- matches, _ = modload.LoadPackages(ctx, loadOpts, pkgPatterns...)
- seenPkgs = make(map[string]bool)
- for i, match := range matches {
- arg := pkgGets[i]
-
- if len(match.Pkgs) == 0 {
- // If the pattern did not match any packages, look up a new module.
- // If the pattern doesn't match anything on the last iteration,
- // we'll print a warning after the outer loop.
- if !match.IsLocal() && !match.IsLiteral() && arg.path != "all" {
- addQuery(&query{querySpec: querySpec{path: arg.path, vers: arg.vers}, arg: arg.raw})
- } else {
- for _, err := range match.Errs {
- base.Errorf("go get: %v", err)
- }
- }
- continue
- }
-
- allStd := true
- for _, pkg := range match.Pkgs {
- if !seenPkgs[pkg] {
- seenPkgs[pkg] = true
- if _, _, err := modload.Lookup("", false, pkg); err != nil {
- allStd = false
- base.Errorf("go get %s: %v", arg.raw, err)
- continue
- }
- }
- m := modload.PackageModule(pkg)
- if m.Path == "" {
- // pkg is in the standard library.
- continue
- }
- allStd = false
- if m.Path == modload.Target.Path {
- // pkg is in the main module.
- continue
- }
- addQuery(&query{querySpec: querySpec{path: m.Path, vers: arg.vers, forceModulePath: true, prevM: m}, arg: arg.raw})
- }
- if allStd && arg.path != arg.raw {
- base.Errorf("go get %s: cannot use pattern %q with explicit version", arg.raw, arg.raw)
- }
- }
- }
- base.ExitIfErrors()
-
- // Query target versions for modules providing packages matched by
- // command line arguments.
- byPath = runQueries(ctx, queryCache, queries, modOnly)
-
- // Handle upgrades. This is needed for arguments that didn't match
- // modules or matched different modules from a previous iteration. It
- // also upgrades modules providing package dependencies if -u is set.
- buildList, err := mvs.UpgradeAll(modload.Target, newUpgrader(byPath, seenPkgs))
- if err != nil {
- base.Fatalf("go get: %v", err)
+ r.performWildcardQueries(ctx)
+ r.performPatternAllQueries(ctx)
+
+ if changed := r.resolveCandidates(ctx, queries, nil); changed {
+ // 'go get' arguments can be (and often are) package patterns rather than
+ // (just) modules. A package can be provided by any module with a prefix
+ // of its import path, and a wildcard can even match packages in modules
+ // with totally different paths. Because of these effects, and because any
+ // change to the selected version of a module can bring in entirely new
+ // module paths as dependencies, we need to reissue queries whenever we
+ // change the build list.
+ //
+ // The result of any version query for a given module — even "upgrade" or
+ // "patch" — is always relative to the build list at the start of
+ // the 'go get' command, not an intermediate state, and is therefore
+ // dederministic and therefore cachable, and the constraints on the
+ // selected version of each module can only narrow as we iterate.
+ //
+ // "all" is functionally very similar to a wildcard pattern. The set of
+ // packages imported by the main module does not change, and the query
+ // result for the module containing each such package also does not change
+ // (it is always relative to the initial build list, before applying
+ // queries). So the only way that the result of an "all" query can change
+ // is if some matching package moves from one module in the build list
+ // to another, which should not happen very often.
+ continue
}
- modload.SetBuildList(buildList)
- base.ExitIfErrors()
- // Stop if no changes have been made to the build list.
- buildList = modload.LoadedModules()
- eq := len(buildList) == len(prevBuildList)
- for i := 0; eq && i < len(buildList); i++ {
- eq = buildList[i] == prevBuildList[i]
- }
- if eq {
- break
+ // When we load imports, we detect the following conditions:
+ //
+ // - missing transitive depencies that need to be resolved from outside the
+ // current build list (note that these may add new matches for existing
+ // pattern queries!)
+ //
+ // - transitive dependencies that didn't match any other query,
+ // but need to be upgraded due to the -u flag
+ //
+ // - ambiguous import errors.
+ // TODO(#27899): Try to resolve ambiguous import errors automatically.
+ upgrades := r.findAndUpgradeImports(ctx, queries)
+ if changed := r.resolveCandidates(ctx, nil, upgrades); changed {
+ continue
}
- prevBuildList = buildList
- }
- // Handle downgrades.
- var down []module.Version
- for _, m := range modload.LoadedModules() {
- q := byPath[m.Path]
- if q != nil && semver.Compare(m.Version, q.m.Version) > 0 {
- down = append(down, module.Version{Path: m.Path, Version: q.m.Version})
- }
- }
- if len(down) > 0 {
- buildList, err := mvs.Downgrade(modload.Target, modload.Reqs(), down...)
- if err != nil {
- base.Fatalf("go: %v", err)
+ r.findMissingWildcards(ctx)
+ if changed := r.resolveCandidates(ctx, r.wildcardQueries, nil); changed {
+ continue
}
- // TODO(bcmills) What should happen here under lazy loading?
- // Downgrading may intentionally violate the lazy-loading invariants.
-
- modload.SetBuildList(buildList)
- modload.ReloadBuildList() // note: does not update go.mod
- base.ExitIfErrors()
+ break
}
- // Scan for any upgrades lost by the downgrades.
- var lostUpgrades []*query
- if len(down) > 0 {
- selectedVersion = make(map[string]string)
- for _, m := range modload.LoadedModules() {
- selectedVersion[m.Path] = m.Version
- }
- for _, q := range byPath {
- if v, ok := selectedVersion[q.m.Path]; q.m.Version != "none" && (!ok || semver.Compare(v, q.m.Version) != 0) {
- lostUpgrades = append(lostUpgrades, q)
- }
- }
- sort.Slice(lostUpgrades, func(i, j int) bool {
- return lostUpgrades[i].m.Path < lostUpgrades[j].m.Path
- })
- }
- if len(lostUpgrades) > 0 {
- desc := func(m module.Version) string {
- s := m.Path + "@" + m.Version
- t := byPath[m.Path]
- if t != nil && t.arg != s {
- s += " from " + t.arg
- }
- return s
- }
- downByPath := make(map[string]module.Version)
- for _, d := range down {
- downByPath[d.Path] = d
- }
-
- var buf strings.Builder
- fmt.Fprintf(&buf, "go get: inconsistent versions:")
- reqs := modload.Reqs()
- for _, q := range lostUpgrades {
- // We lost q because its build list requires a newer version of something in down.
- // Figure out exactly what.
- // Repeatedly constructing the build list is inefficient
- // if there are MANY command-line arguments,
- // but at least all the necessary requirement lists are cached at this point.
- list, err := buildListForLostUpgrade(q.m, reqs)
- if err != nil {
- base.Fatalf("go: %v", err)
- }
+ r.checkWildcardVersions(ctx)
- fmt.Fprintf(&buf, "\n\t%s", desc(q.m))
- sep := " requires"
- for _, m := range list {
- if down, ok := downByPath[m.Path]; ok && semver.Compare(down.Version, m.Version) < 0 {
- fmt.Fprintf(&buf, "%s %s@%s (not %s)", sep, m.Path, m.Version, desc(down))
- sep = ","
- }
- }
- if sep != "," {
- // We have no idea why this happened.
- // At least report the problem.
- if v := selectedVersion[q.m.Path]; v == "" {
- fmt.Fprintf(&buf, " removed unexpectedly")
- } else {
- fmt.Fprintf(&buf, " ended up at %s unexpectedly", v)
- }
- fmt.Fprintf(&buf, " (please report at golang.org/issue/new)")
- }
+ var pkgPatterns []string
+ for _, q := range queries {
+ if q.matchesPackages {
+ pkgPatterns = append(pkgPatterns, q.pattern)
}
- base.Fatalf("%v", buf.String())
}
-
- if len(pkgPatterns) > 0 || len(args) == 0 {
- // Before we write the updated go.mod file, reload the requested packages to
- // check for errors.
- loadOpts := modload.PackageOpts{
- Tags: imports.AnyTags(),
- LoadTests: *getT,
-
- // Only print warnings after the last iteration, and only if we aren't going
- // to build (to avoid doubled warnings).
- //
- // Only local patterns in the main module, such as './...', can be unmatched.
- // (See the mod_get_nopkgs test for more detail.)
- SilenceUnmatchedWarnings: !*getD,
+ if len(pkgPatterns) > 0 {
+ // We skipped over missing-package errors earlier: we want to resolve
+ // pathSets ourselves, but at that point we don't have enough context
+ // to log the package-import chains leading to the error. Reload the package
+ // import graph one last time to report any remaining unresolved
+ // dependencies.
+ pkgOpts := modload.PackageOpts{
+ LoadTests: *getT,
+ ResolveMissingImports: false,
+ AllowErrors: false,
}
- modload.LoadPackages(ctx, loadOpts, pkgPatterns...)
+ modload.LoadPackages(ctx, pkgOpts, pkgPatterns...)
+ base.ExitIfErrors()
}
- // If -d was specified, we're done after the module work.
- // We've already downloaded modules by loading packages above.
+ // We've already downloaded modules (and identified direct and indirect
+ // dependencies) by loading packages in findAndUpgradeImports.
+ // So if -d is set, we're done after the module work.
+ //
// Otherwise, we need to build and install the packages matched by
- // command line arguments. This may be a different set of packages,
- // since we only build packages for the target platform.
+ // command line arguments.
// Note that 'go get -u' without arguments is equivalent to
// 'go get -u .', so we'll typically build the package in the current
// directory.
- if !*getD && len(pkgPatterns) > 0 {
- work.BuildInit()
- pkgs := load.PackagesForBuild(ctx, pkgPatterns)
- work.InstallPackages(ctx, pkgPatterns, pkgs)
+ if !*getD {
+ if len(pkgPatterns) > 0 {
+ work.BuildInit()
+ pkgs := load.PackagesForBuild(ctx, pkgPatterns)
+ work.InstallPackages(ctx, pkgPatterns, pkgs)
+ }
}
// Everything succeeded. Update go.mod.
// Report warnings if any retracted versions are in the build list.
// This must be done after writing go.mod to avoid spurious '// indirect'
// comments. These functions read and write global state.
- // TODO(golang.org/issue/40775): ListModules resets modload.loader, which
- // contains information about direct dependencies that WriteGoMod uses.
- // Refactor to avoid these kinds of global side effects.
+ //
+ // TODO(golang.org/issue/40775): ListModules (called from reportRetractions)
+ // resets modload.loader, which contains information about direct dependencies
+ // that WriteGoMod uses. Refactor to avoid these kinds of global side effects.
reportRetractions(ctx)
}
//
// The command-line arguments are of the form path@version or simply path, with
// implicit @upgrade. path@none is "downgrade away".
-func parseArgs(rawArgs []string) []getArg {
+func parseArgs(ctx context.Context, rawArgs []string) []*query {
defer base.ExitIfErrors()
- var gets []getArg
- for _, raw := range search.CleanPatterns(rawArgs) {
- // Argument is path or path@vers.
- path := raw
- vers := ""
- if i := strings.Index(raw, "@"); i >= 0 {
- path, vers = raw[:i], raw[i+1:]
- }
- if strings.Contains(vers, "@") || raw != path && vers == "" {
- base.Errorf("go get %s: invalid module version syntax", raw)
+ var queries []*query
+ for _, arg := range search.CleanPatterns(rawArgs) {
+ q, err := newQuery(arg)
+ if err != nil {
+ base.Errorf("go get: %v", err)
continue
}
// Guard against 'go get x.go', a common mistake.
// Note that package and module paths may end with '.go', so only print an error
// if the argument has no version and either has no slash or refers to an existing file.
- if strings.HasSuffix(raw, ".go") && vers == "" {
- if !strings.Contains(raw, "/") {
- base.Errorf("go get %s: arguments must be package or module paths", raw)
+ if strings.HasSuffix(q.raw, ".go") && q.rawVersion == "" {
+ if !strings.Contains(q.raw, "/") {
+ base.Errorf("go get %s: arguments must be package or module paths", q.raw)
continue
}
- if fi, err := os.Stat(raw); err == nil && !fi.IsDir() {
- base.Errorf("go get: %s exists as a file, but 'go get' requires package arguments", raw)
+ if fi, err := os.Stat(q.raw); err == nil && !fi.IsDir() {
+ base.Errorf("go get: %s exists as a file, but 'go get' requires package arguments", q.raw)
continue
}
}
- // If no version suffix is specified, assume @upgrade.
- // If -u=patch was specified, assume @patch instead.
- if vers == "" {
- if getU != "" {
- vers = string(getU)
+ queries = append(queries, q)
+ }
+
+ return queries
+}
+
+// reportRetractions prints warnings if any modules in the build list are
+// retracted.
+func reportRetractions(ctx context.Context) {
+ // Query for retractions of modules in the build list.
+ // Use modload.ListModules, since that provides information in the same format
+ // as 'go list -m'. Don't query for "all", since that's not allowed outside a
+ // module.
+ buildList := modload.LoadedModules()
+ args := make([]string, 0, len(buildList))
+ for _, m := range buildList {
+ if m.Version == "" {
+ // main module or dummy target module
+ continue
+ }
+ args = append(args, m.Path+"@"+m.Version)
+ }
+ listU := false
+ listVersions := false
+ listRetractions := true
+ mods := modload.ListModules(ctx, args, listU, listVersions, listRetractions)
+ retractPath := ""
+ for _, mod := range mods {
+ if len(mod.Retracted) > 0 {
+ if retractPath == "" {
+ retractPath = mod.Path
} else {
- vers = "upgrade"
+ retractPath = "<module>"
}
+ rationale := modload.ShortRetractionRationale(mod.Retracted[0])
+ fmt.Fprintf(os.Stderr, "go: warning: %s@%s is retracted: %s\n", mod.Path, mod.Version, rationale)
+ }
+ }
+ if modload.HasModRoot() && retractPath != "" {
+ fmt.Fprintf(os.Stderr, "go: run 'go get %s@latest' to switch to the latest unretracted version\n", retractPath)
+ }
+}
+
+type resolver struct {
+ localQueries []*query // queries for absolute or relative paths
+ pathQueries []*query // package path literal queries in original order
+ wildcardQueries []*query // path wildcard queries in original order
+ patternAllQueries []*query // queries with the pattern "all"
+
+ // Indexed "none" queries. These are also included in the slices above;
+ // they are indexed here to speed up noneForPath.
+ nonesByPath map[string]*query // path-literal "@none" queries indexed by path
+ wildcardNones []*query // wildcard "@none" queries
+
+ // resolvedVersion maps each module path to the version of that module that
+ // must be selected in the final build list, along with the first query
+ // that resolved the module to that version (the “reason”).
+ resolvedVersion map[string]versionReason
+
+ buildList []module.Version
+ buildListResolvedVersions int // len(resolvedVersion) when buildList was computed
+ buildListVersion map[string]string // index of buildList (module path → version)
+
+ initialVersion map[string]string // index of the initial build list at the start of 'go get'
+
+ missing []pathSet // candidates for missing transitive dependencies
+
+ work *par.Queue
+
+ queryModuleCache par.Cache
+ queryPackagesCache par.Cache
+ queryPatternCache par.Cache
+ matchInModuleCache par.Cache
+}
+
+type versionReason struct {
+ version string
+ reason *query
+}
+
+func newResolver(ctx context.Context, queries []*query) *resolver {
+ buildList := modload.LoadAllModules(ctx)
+ initialVersion := make(map[string]string, len(buildList))
+ for _, m := range buildList {
+ initialVersion[m.Path] = m.Version
+ }
+
+ r := &resolver{
+ work: par.NewQueue(runtime.GOMAXPROCS(0)),
+ resolvedVersion: map[string]versionReason{},
+ buildList: buildList,
+ buildListVersion: initialVersion,
+ initialVersion: initialVersion,
+ nonesByPath: map[string]*query{},
+ }
+
+ for _, q := range queries {
+ if q.pattern == "all" {
+ r.patternAllQueries = append(r.patternAllQueries, q)
+ } else if q.patternIsLocal {
+ r.localQueries = append(r.localQueries, q)
+ } else if q.isWildcard() {
+ r.wildcardQueries = append(r.wildcardQueries, q)
+ } else {
+ r.pathQueries = append(r.pathQueries, q)
+ }
+
+ if q.version == "none" {
+ // Index "none" queries to make noneForPath more efficient.
+ if q.isWildcard() {
+ r.wildcardNones = append(r.wildcardNones, q)
+ } else {
+ // All "<path>@none" queries for the same path are identical; we only
+ // need to index one copy.
+ r.nonesByPath[q.pattern] = q
+ }
+ }
+ }
+
+ return r
+}
+
+// initialSelected returns the version of the module with the given path that
+// was selected at the start of this 'go get' invocation.
+func (r *resolver) initialSelected(mPath string) (version string) {
+ v, ok := r.initialVersion[mPath]
+ if !ok {
+ return "none"
+ }
+ return v
+}
+
+// selected returns the version of the module with the given path that is
+// selected in the resolver's current build list.
+func (r *resolver) selected(mPath string) (version string) {
+ v, ok := r.buildListVersion[mPath]
+ if !ok {
+ return "none"
+ }
+ return v
+}
+
+// noneForPath returns a "none" query matching the given module path,
+// or found == false if no such query exists.
+func (r *resolver) noneForPath(mPath string) (nq *query, found bool) {
+ if nq = r.nonesByPath[mPath]; nq != nil {
+ return nq, true
+ }
+ for _, nq := range r.wildcardNones {
+ if nq.matchesPath(mPath) {
+ return nq, true
+ }
+ }
+ return nil, false
+}
+
+// queryModule wraps modload.Query, substituting r.checkAllowedor to decide
+// allowed versions.
+func (r *resolver) queryModule(ctx context.Context, mPath, query string, selected func(string) string) (module.Version, error) {
+ current := r.initialSelected(mPath)
+ rev, err := modload.Query(ctx, mPath, query, current, r.checkAllowedOr(query, selected))
+ if err != nil {
+ return module.Version{}, err
+ }
+ return module.Version{Path: mPath, Version: rev.Version}, nil
+}
+
+// queryPackage wraps modload.QueryPackage, substituting r.checkAllowedOr to
+// decide allowed versions.
+func (r *resolver) queryPackages(ctx context.Context, pattern, query string, selected func(string) string) (pkgMods []module.Version, err error) {
+ results, err := modload.QueryPackages(ctx, pattern, query, selected, r.checkAllowedOr(query, selected))
+ if len(results) > 0 {
+ pkgMods = make([]module.Version, 0, len(results))
+ for _, qr := range results {
+ pkgMods = append(pkgMods, qr.Mod)
+ }
+ }
+ return pkgMods, err
+}
+
+// queryPattern wraps modload.QueryPattern, substituting r.checkAllowedOr to
+// decide allowed versions.
+func (r *resolver) queryPattern(ctx context.Context, pattern, query string, selected func(string) string) (pkgMods []module.Version, mod module.Version, err error) {
+ results, modOnly, err := modload.QueryPattern(ctx, pattern, query, selected, r.checkAllowedOr(query, selected))
+ if len(results) > 0 {
+ pkgMods = make([]module.Version, 0, len(results))
+ for _, qr := range results {
+ pkgMods = append(pkgMods, qr.Mod)
}
+ }
+ if modOnly != nil {
+ mod = modOnly.Mod
+ }
+ return pkgMods, mod, err
+}
+
+// checkAllowedOr is like modload.CheckAllowed, but it always allows the requested
+// and current versions (even if they are retracted or otherwise excluded).
+func (r *resolver) checkAllowedOr(requested string, selected func(string) string) modload.AllowedFunc {
+ return func(ctx context.Context, m module.Version) error {
+ if m.Version == requested {
+ return modload.CheckExclusions(ctx, m)
+ }
+ if (requested == "upgrade" || requested == "patch") && m.Version == selected(m.Path) {
+ return nil
+ }
+ return modload.CheckAllowed(ctx, m)
+ }
+}
- gets = append(gets, getArg{raw: raw, path: path, vers: vers})
+// matchInModule is a caching wrapper around modload.MatchInModule.
+func (r *resolver) matchInModule(ctx context.Context, pattern string, m module.Version) (packages []string, err error) {
+ type key struct {
+ pattern string
+ m module.Version
+ }
+ type entry struct {
+ packages []string
+ err error
}
- return gets
+ e := r.matchInModuleCache.Do(key{pattern, m}, func() interface{} {
+ match := modload.MatchInModule(ctx, pattern, m, imports.AnyTags())
+ if len(match.Errs) > 0 {
+ return entry{match.Pkgs, match.Errs[0]}
+ }
+ return entry{match.Pkgs, nil}
+ }).(entry)
+
+ return e.packages, e.err
}
-// classifyArgs determines which arguments refer to packages and which refer to
-// modules, and creates queries to look up modules at target versions before
-// loading packages.
+// queryNone adds a candidate set to q for each module matching q.pattern.
+// Each candidate set has only one possible module version: the matched
+// module at version "none".
//
-// This is an imprecise process, but it helps reduce unnecessary
-// queries and package loading. It's also necessary for handling
-// patterns like golang.org/x/tools/..., which can't be expanded
-// during package loading until they're in the build list.
-func classifyArgs(ctx context.Context, selectedVersion map[string]string, args []getArg) []*query {
- defer base.ExitIfErrors()
+// We interpret arguments to 'go get' as packages first, and fall back to
+// modules second. However, no module exists at version "none", and therefore no
+// package exists at that version either: we know that the argument cannot match
+// any packages, and thus it must match modules instead.
+func (r *resolver) queryNone(ctx context.Context, q *query) {
+ if search.IsMetaPackage(q.pattern) {
+ panic(fmt.Sprintf("internal error: queryNone called with pattern %q", q.pattern))
+ }
- queries := make([]*query, 0, len(args))
-
- for _, arg := range args {
- path := arg.path
- switch {
- case filepath.IsAbs(path) || search.IsRelativePath(path):
- // Absolute paths like C:\foo and relative paths like ../foo...
- // are restricted to matching packages in the main module. If the path
- // is explicit and contains no wildcards (...), check that it is a
- // package in the main module. If the path contains wildcards but
- // matches no packages, we'll warn after package loading.
- if !strings.Contains(path, "...") {
- m := search.NewMatch(path)
- if pkgPath := modload.DirImportPath(path); pkgPath != "." {
- m = modload.TargetPackages(ctx, pkgPath)
- }
- if len(m.Pkgs) == 0 {
- for _, err := range m.Errs {
- base.Errorf("go get %s: %v", arg, err)
- }
+ if !q.isWildcard() {
+ q.pathOnce(q.pattern, func() pathSet {
+ if modload.HasModRoot() && q.pattern == modload.Target.Path {
+ // The user has explicitly requested to downgrade their own module to
+ // version "none". This is not an entirely unreasonable request: it
+ // could plausibly mean “downgrade away everything that depends on any
+ // explicit version of the main module”, or “downgrade away the
+ // package with the same path as the main module, found in a module
+ // with a prefix of the main module's path”.
+ //
+ // However, neither of those behaviors would be consistent with the
+ // plain meaning of the query. To try to reduce confusion, reject the
+ // query explicitly.
+ return errSet(&modload.QueryMatchesMainModuleError{Pattern: q.pattern, Query: q.version})
+ }
- abs, err := filepath.Abs(path)
- if err != nil {
- abs = path
- }
- base.Errorf("go get %s: path %s is not a package in module rooted at %s", arg, abs, modload.ModRoot())
- continue
+ return pathSet{mod: module.Version{Path: q.pattern, Version: "none"}}
+ })
+ }
+
+ for _, curM := range r.buildList {
+ if !q.matchesPath(curM.Path) {
+ continue
+ }
+ q.pathOnce(curM.Path, func() pathSet {
+ if modload.HasModRoot() && curM == modload.Target {
+ return errSet(&modload.QueryMatchesMainModuleError{Pattern: q.pattern, Query: q.version})
+ }
+ return pathSet{mod: module.Version{Path: curM.Path, Version: "none"}}
+ })
+ }
+}
+
+func (r *resolver) performLocalQueries(ctx context.Context) {
+ for _, q := range r.localQueries {
+ q.pathOnce(q.pattern, func() pathSet {
+ absDetail := ""
+ if !filepath.IsAbs(q.pattern) {
+ if absPath, err := filepath.Abs(q.pattern); err == nil {
+ absDetail = fmt.Sprintf(" (%s)", absPath)
}
}
- if arg.path != arg.raw {
- base.Errorf("go get %s: can't request explicit version of path in main module", arg)
- continue
+ // Absolute paths like C:\foo and relative paths like ../foo... are
+ // restricted to matching packages in the main module.
+ pkgPattern := modload.DirImportPath(q.pattern)
+ if pkgPattern == "." {
+ return errSet(fmt.Errorf("%s%s is not within module rooted at %s", q.pattern, absDetail, modload.ModRoot()))
}
- case strings.Contains(path, "..."):
- // Wait until we load packages to look up modules.
- // We don't know yet whether any modules in the build list provide
- // packages matching the pattern. For example, suppose
- // golang.org/x/tools and golang.org/x/tools/playground are separate
- // modules, and only golang.org/x/tools is in the build list. If the
- // user runs 'go get golang.org/x/tools/playground/...', we should
- // add a requirement for golang.org/x/tools/playground. We should not
- // upgrade golang.org/x/tools.
+ match := modload.MatchInModule(ctx, pkgPattern, modload.Target, imports.AnyTags())
+ if len(match.Errs) > 0 {
+ return pathSet{err: match.Errs[0]}
+ }
- case path == "all":
- // If there is no main module, "all" is not meaningful.
- if !modload.HasModRoot() {
- base.Errorf(`go get %s: cannot match "all": working directory is not part of a module`, arg)
+ if len(match.Pkgs) == 0 {
+ if !q.isWildcard() {
+ return errSet(fmt.Errorf("%s%s is not a package in module rooted at %s", q.pattern, absDetail, modload.ModRoot()))
+ }
+ search.WarnUnmatched([]*search.Match{match})
+ return pathSet{}
}
- // Don't query modules until we load packages. We'll automatically
- // look up any missing modules.
- case search.IsMetaPackage(path):
- base.Errorf("go get %s: explicit requirement on standard-library module %s not allowed", path, path)
+ return pathSet{pkgMods: []module.Version{modload.Target}}
+ })
+ }
+}
+
+// performWildcardQueries populates the candidates for each query whose pattern
+// is a wildcard.
+//
+// The candidates for a given module path matching (or containing a package
+// matching) a wildcard query depend only on the initial build list, but the set
+// of modules may be expanded by other queries, so wildcard queries need to be
+// re-evaluated whenever a potentially-matching module path is added to the
+// build list.
+func (r *resolver) performWildcardQueries(ctx context.Context) {
+ for _, q := range r.wildcardQueries {
+ q := q
+ r.work.Add(func() {
+ if q.version == "none" {
+ r.queryNone(ctx, q)
+ } else {
+ r.queryWildcard(ctx, q)
+ }
+ })
+ }
+ <-r.work.Idle()
+}
+
+// queryWildcard adds a candidate set to q for each module for which:
+// - some version of the module is already in the build list, and
+// - that module exists at some version matching q.version, and
+// - either the module path itself matches q.pattern, or some package within
+// the module at q.version matches q.pattern.
+func (r *resolver) queryWildcard(ctx context.Context, q *query) {
+ // For wildcard patterns, modload.QueryPattern only identifies modules
+ // matching the prefix of the path before the wildcard. However, the build
+ // list may already contain other modules with matching packages, and we
+ // should consider those modules to satisfy the query too.
+ // We want to match any packages in existing dependencies, but we only want to
+ // resolve new dependencies if nothing else turns up.
+ for _, curM := range r.buildList {
+ if !q.canMatchInModule(curM.Path) {
continue
+ }
+ q.pathOnce(curM.Path, func() pathSet {
+ if _, hit := r.noneForPath(curM.Path); hit {
+ // This module is being removed, so it will no longer be in the build list
+ // (and thus will no longer match the pattern).
+ return pathSet{}
+ }
- default:
- // The argument is a package or module path.
- if modload.HasModRoot() {
- if m := modload.TargetPackages(ctx, path); len(m.Pkgs) != 0 {
- // The path is in the main module. Nothing to query.
- if arg.vers != "upgrade" && arg.vers != "patch" {
- base.Errorf("go get %s: can't request explicit version of path in main module", arg)
- }
- continue
+ if curM.Path == modload.Target.Path && !versionOkForMainModule(q.version) {
+ if q.matchesPath(curM.Path) {
+ return errSet(&modload.QueryMatchesMainModuleError{
+ Pattern: q.pattern,
+ Query: q.version,
+ })
}
+
+ packages, err := r.matchInModule(ctx, q.pattern, curM)
+ if err != nil {
+ return errSet(err)
+ }
+ if len(packages) > 0 {
+ return errSet(&modload.QueryMatchesPackagesInMainModuleError{
+ Pattern: q.pattern,
+ Query: q.version,
+ Packages: packages,
+ })
+ }
+
+ return r.tryWildcard(ctx, q, curM)
}
- first := path
- if i := strings.IndexByte(first, '/'); i >= 0 {
- first = path
+ m, err := r.queryModule(ctx, curM.Path, q.version, r.initialSelected)
+ if err != nil {
+ if !isNoSuchModuleVersion(err) {
+ // We can't tell whether a matching version exists.
+ return errSet(err)
+ }
+ // There is no version of curM.Path matching the query.
+
+ // We haven't checked whether curM contains any matching packages at its
+ // currently-selected version, or whether curM.Path itself matches q. If
+ // either of those conditions holds, *and* no other query changes the
+ // selected version of curM, then we will fail in checkWildcardVersions.
+ // (This could be an error, but it's too soon to tell.)
+ //
+ // However, even then the transitive requirements of some other query
+ // may downgrade this module out of the build list entirely, in which
+ // case the pattern will no longer include it and it won't be an error.
+ //
+ // Either way, punt on the query rather than erroring out just yet.
+ return pathSet{}
}
- if !strings.Contains(first, ".") {
- // The path doesn't have a dot in the first component and cannot be
- // queried as a module. It may be a package in the standard library,
- // which is fine, so don't report an error unless we encounter
- // a problem loading packages.
+
+ return r.tryWildcard(ctx, q, m)
+ })
+ }
+
+ // Even if no modules matched, we shouldn't query for a new module to provide
+ // the pattern yet: some other query may yet induce a new requirement that
+ // will match the wildcard. Instead, we'll check in findMissingWildcards.
+}
+
+// tryWildcard returns a pathSet for module m matching query q.
+// If m does not actually match q, tryWildcard returns an empty pathSet.
+func (r *resolver) tryWildcard(ctx context.Context, q *query, m module.Version) pathSet {
+ mMatches := q.matchesPath(m.Path)
+ packages, err := r.matchInModule(ctx, q.pattern, m)
+ if err != nil {
+ return errSet(err)
+ }
+ if len(packages) > 0 {
+ return pathSet{pkgMods: []module.Version{m}}
+ }
+ if mMatches {
+ return pathSet{mod: m}
+ }
+ return pathSet{}
+}
+
+// findMissingWildcards adds a candidate set for each query in r.wildcardQueries
+// that has not yet resolved to any version containing packages.
+func (r *resolver) findMissingWildcards(ctx context.Context) {
+ for _, q := range r.wildcardQueries {
+ if q.version == "none" || q.matchesPackages {
+ continue // q is not “missing”
+ }
+ r.work.Add(func() {
+ q.pathOnce(q.pattern, func() pathSet {
+ pkgMods, mod, err := r.queryPattern(ctx, q.pattern, q.version, r.initialSelected)
+ if err != nil {
+ if isNoSuchPackageVersion(err) && len(q.resolved) > 0 {
+ // q already resolved one or more modules but matches no packages.
+ // That's ok: this pattern is just a module pattern, and we don't
+ // need to add any more modules to satisfy it.
+ return pathSet{}
+ }
+ return errSet(err)
+ }
+
+ return pathSet{pkgMods: pkgMods, mod: mod}
+ })
+ })
+ }
+ <-r.work.Idle()
+}
+
+// checkWildcardVersions reports an error if any module in the build list has a
+// path (or contains a package) matching a query with a wildcard pattern, but
+// has a selected version that does *not* match the query.
+func (r *resolver) checkWildcardVersions(ctx context.Context) {
+ defer base.ExitIfErrors()
+
+ for _, q := range r.wildcardQueries {
+ for _, curM := range r.buildList {
+ if !q.canMatchInModule(curM.Path) {
continue
}
+ if !q.matchesPath(curM.Path) {
+ packages, err := r.matchInModule(ctx, q.pattern, curM)
+ if len(packages) == 0 {
+ if err != nil {
+ reportError(q, err)
+ }
+ continue // curM is not relevant to q.
+ }
+ }
- // If we're querying "upgrade" or "patch", we need to know the current
- // version of the module. For "upgrade", we want to avoid accidentally
- // downgrading from a newer prerelease. For "patch", we need to query
- // the correct minor version.
- // Here, we check if "path" is the name of a module in the build list
- // (other than the main module) and set prevM if so. If "path" isn't
- // a module in the build list, the current version doesn't matter
- // since it's either an unknown module or a package within a module
- // that we'll discover later.
- q := &query{querySpec: querySpec{path: arg.path, vers: arg.vers}, arg: arg.raw}
- if v, ok := selectedVersion[path]; ok {
- if path == modload.Target.Path {
- // TODO(bcmills): This is held over from a previous version of the get
- // implementation. Why was it a special case?
- } else {
- q.prevM = module.Version{Path: path, Version: v}
- q.forceModulePath = true
+ rev, err := r.queryModule(ctx, curM.Path, q.version, r.initialSelected)
+ if err != nil {
+ reportError(q, err)
+ continue
+ }
+ if rev.Version == curM.Version {
+ continue // curM already matches q.
+ }
+
+ if !q.matchesPath(curM.Path) {
+ m := module.Version{Path: curM.Path, Version: rev.Version}
+ packages, err := r.matchInModule(ctx, q.pattern, m)
+ if err != nil {
+ reportError(q, err)
+ continue
+ }
+ if len(packages) == 0 {
+ // curM at its original version contains a path matching q.pattern,
+ // but at rev.Version it does not, so (somewhat paradoxically) if
+ // we changed the version of curM it would no longer match the query.
+ var version interface{} = m
+ if rev.Version != q.version {
+ version = fmt.Sprintf("%s@%s (%s)", m.Path, q.version, m.Version)
+ }
+ reportError(q, fmt.Errorf("%v matches packages in %v but not %v: specify a different version for module %s", q, curM, version, m.Path))
+ continue
}
}
- queries = append(queries, q)
+
+ // Since queryModule succeeded and either curM or one of the packages it
+ // contains matches q.pattern, we should have either selected the version
+ // of curM matching q, or reported a conflict error (and exited).
+ // If we're still here and the version doesn't match,
+ // something has gone very wrong.
+ reportError(q, fmt.Errorf("internal error: selected %v instead of %v", curM, rev.Version))
}
}
+}
- return queries
+// performPathQueries populates the candidates for each query whose pattern is
+// a path literal.
+//
+// The candidate packages and modules for path literals depend only on the
+// initial build list, not the current build list, so we only need to query path
+// literals once.
+func (r *resolver) performPathQueries(ctx context.Context) {
+ for _, q := range r.pathQueries {
+ q := q
+ r.work.Add(func() {
+ if q.version == "none" {
+ r.queryNone(ctx, q)
+ } else {
+ r.queryPath(ctx, q)
+ }
+ })
+ }
+ <-r.work.Idle()
}
-// runQueries looks up modules at target versions in parallel. Results will be
-// cached. If the same module is referenced by multiple queries at different
-// versions (including earlier queries in the modOnly map), an error will be
-// reported. A map from module paths to queries is returned, which includes
-// queries and modOnly.
-func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*query, modOnly map[string]*query) map[string]*query {
+// queryPath adds a candidate set to q for the package with path q.pattern.
+// The candidate set consists of all modules that could provide q.pattern
+// and have a version matching q, plus (if it exists) the module whose path
+// is itself q.pattern (at a matching version).
+func (r *resolver) queryPath(ctx context.Context, q *query) {
+ q.pathOnce(q.pattern, func() pathSet {
+ if search.IsMetaPackage(q.pattern) || q.isWildcard() {
+ panic(fmt.Sprintf("internal error: queryPath called with pattern %q", q.pattern))
+ }
+ if q.version == "none" {
+ panic(`internal error: queryPath called with version "none"`)
+ }
- runQuery := func(q *query) {
- if q.vers == "none" {
- // Wait for downgrade step.
- q.m = module.Version{Path: q.path, Version: "none"}
- return
+ if search.IsStandardImportPath(q.pattern) {
+ stdOnly := module.Version{}
+ packages, _ := r.matchInModule(ctx, q.pattern, stdOnly)
+ if len(packages) > 0 {
+ if q.rawVersion != "" {
+ return errSet(fmt.Errorf("can't request explicit version %q of standard library package %s", q.version, q.pattern))
+ }
+
+ q.matchesPackages = true
+ return pathSet{} // No module needed for standard library.
+ }
}
- m, err := getQuery(ctx, q.path, q.vers, q.prevM, q.forceModulePath)
+
+ pkgMods, mod, err := r.queryPattern(ctx, q.pattern, q.version, r.initialSelected)
if err != nil {
- base.Errorf("go get %s: %v", q.arg, err)
+ return errSet(err)
}
- q.m = m
+ return pathSet{pkgMods: pkgMods, mod: mod}
+ })
+}
+
+// performPatternAllQueries populates the candidates for each query whose
+// pattern is "all".
+//
+// The candidate modules for a given package in "all" depend only on the initial
+// build list, but we cannot follow the dependencies of a given package until we
+// know which candidate is selected — and that selection may depend on the
+// results of other queries. We need to re-evaluate the "all" queries whenever
+// the module for one or more packages in "all" are resolved.
+func (r *resolver) performPatternAllQueries(ctx context.Context) {
+ if len(r.patternAllQueries) == 0 {
+ return
}
- type token struct{}
- sem := make(chan token, runtime.GOMAXPROCS(0))
- for _, q := range queries {
- if cached := cache[q.querySpec]; cached != nil {
- *q = *cached
- } else {
- sem <- token{}
- go func(q *query) {
- runQuery(q)
- <-sem
- }(q)
+ findPackage := func(ctx context.Context, path string, m module.Version) (versionOk bool) {
+ versionOk = true
+ for _, q := range r.patternAllQueries {
+ q.pathOnce(path, func() pathSet {
+ pkgMods, err := r.queryPackages(ctx, path, q.version, r.initialSelected)
+ if len(pkgMods) != 1 || pkgMods[0] != m {
+ // There are candidates other than m for the given path, so we can't
+ // be certain that m will actually be the module selected to provide
+ // the package. Don't load its dependencies just yet, because they
+ // might no longer be dependencies after we resolve the correct
+ // version.
+ versionOk = false
+ }
+ return pathSet{pkgMods: pkgMods, err: err}
+ })
}
+ return versionOk
}
- // Fill semaphore channel to wait for goroutines to finish.
- for n := cap(sem); n > 0; n-- {
- sem <- token{}
+ r.loadPackages(ctx, []string{"all"}, findPackage)
+
+ // Since we built up the candidate lists concurrently, they may be in a
+ // nondeterministic order. We want 'go get' to be fully deterministic,
+ // including in which errors it chooses to report, so sort the candidates
+ // into a deterministic-but-arbitrary order.
+ for _, q := range r.patternAllQueries {
+ sort.Slice(q.candidates, func(i, j int) bool {
+ return q.candidates[i].path < q.candidates[j].path
+ })
}
+}
- // Add to cache after concurrent section to avoid races...
+// findAndUpgradeImports returns a pathSet for each package that is not yet
+// in the build list but is transitively imported by the packages matching the
+// given queries (which must already have been resolved).
+//
+// If the getU flag ("-u") is set, findAndUpgradeImports also returns a
+// pathSet for each module that is not constrained by any other
+// command-line argument and has an available matching upgrade.
+func (r *resolver) findAndUpgradeImports(ctx context.Context, queries []*query) (upgrades []pathSet) {
+ patterns := make([]string, 0, len(queries))
for _, q := range queries {
- cache[q.querySpec] = q
+ if q.matchesPackages {
+ patterns = append(patterns, q.pattern)
+ }
+ }
+ if len(patterns) == 0 {
+ return nil
}
- base.ExitIfErrors()
+ // mu guards concurrent writes to upgrades, which will be sorted
+ // (to restore determinism) after loading.
+ var mu sync.Mutex
- byPath := make(map[string]*query)
- check := func(q *query) {
- if prev, ok := byPath[q.m.Path]; prev != nil && prev.m != q.m {
- base.Errorf("go get: conflicting versions for module %s: %s and %s", q.m.Path, prev.m.Version, q.m.Version)
- byPath[q.m.Path] = nil // sentinel to stop errors
- return
- } else if !ok {
- byPath[q.m.Path] = q
+ findPackage := func(ctx context.Context, path string, m module.Version) (versionOk bool) {
+ version := "latest"
+ if m.Path != "" {
+ if getU.version == "" {
+ // The user did not request that we upgrade transitive dependencies.
+ return true
+ }
+ if _, ok := r.resolvedVersion[m.Path]; ok {
+ // We cannot upgrade m implicitly because its version is determined by
+ // an explicit pattern argument.
+ return true
+ }
+ version = getU.version
+ }
+
+ // Unlike other queries, the "-u" flag upgrades relative to the build list
+ // after applying changes so far, not the initial build list.
+ // This is for two reasons:
+ //
+ // - The "-u" flag intentionally applies to transitive dependencies,
+ // which may not be known or even resolved in advance of applying
+ // other version changes.
+ //
+ // - The "-u" flag, unlike other arguments, does not cause version
+ // conflicts with other queries. (The other query always wins.)
+
+ pkgMods, err := r.queryPackages(ctx, path, version, r.selected)
+ for _, u := range pkgMods {
+ if u == m {
+ // The selected package version is already upgraded appropriately; there
+ // is no need to change it.
+ return true
+ }
+ }
+
+ if err != nil {
+ if isNoSuchPackageVersion(err) || (m.Path == "" && module.CheckPath(path) != nil) {
+ // We can't find the package because it doesn't — or can't — even exist
+ // in any module at the latest version. (Note that invalid module paths
+ // could in general exist due to replacements, so we at least need to
+ // run the query to check those.)
+ //
+ // There is no version change we can make to fix the package, so leave
+ // it unresolved. Either some other query (perhaps a wildcard matching a
+ // newly-added dependency for some other missing package) will fill in
+ // the gaps, or we will report an error (with a better import stack) in
+ // the final LoadPackages call.
+ return true
+ }
}
+
+ mu.Lock()
+ upgrades = append(upgrades, pathSet{pkgMods: pkgMods, err: err})
+ mu.Unlock()
+ return false
}
- for _, q := range queries {
- check(q)
+
+ r.loadPackages(ctx, patterns, findPackage)
+
+ // Since we built up the candidate lists concurrently, they may be in a
+ // nondeterministic order. We want 'go get' to be fully deterministic,
+ // including in which errors it chooses to report, so sort the candidates
+ // into a deterministic-but-arbitrary order.
+ sort.Slice(upgrades, func(i, j int) bool {
+ return upgrades[i].path < upgrades[j].path
+ })
+ return upgrades
+}
+
+// loadPackages loads the packages matching the given patterns, invoking the
+// findPackage function for each package that may require a change to the
+// build list.
+//
+// loadPackages invokes the findPackage function for each package loaded from a
+// module outside the main module. If the module or version that supplies that
+// package needs to be changed due to a query, findPackage may return false
+// and the imports of that package will not be loaded.
+//
+// loadPackages also invokes the findPackage function for each imported package
+// that is neither present in the standard library nor in any module in the
+// build list.
+func (r *resolver) loadPackages(ctx context.Context, patterns []string, findPackage func(ctx context.Context, path string, m module.Version) (versionOk bool)) {
+ opts := modload.PackageOpts{
+ Tags: imports.AnyTags(),
+ LoadTests: *getT,
+ SilenceErrors: true, // May be fixed by subsequent upgrades or downgrades.
}
- for _, q := range modOnly {
- check(q)
+
+ opts.AllowPackage = func(ctx context.Context, path string, m module.Version) error {
+ if m.Path == "" || m == modload.Target {
+ // Packages in the standard library and main module are already at their
+ // latest (and only) available versions.
+ return nil
+ }
+ if ok := findPackage(ctx, path, m); !ok {
+ return errVersionChange
+ }
+ return nil
}
- base.ExitIfErrors()
- return byPath
+ _, pkgs := modload.LoadPackages(ctx, opts, patterns...)
+ for _, path := range pkgs {
+ const (
+ parentPath = ""
+ parentIsStd = false
+ )
+ _, _, err := modload.Lookup(parentPath, parentIsStd, path)
+ if err == nil {
+ continue
+ }
+ if errors.Is(err, errVersionChange) {
+ // We already added candidates during loading.
+ continue
+ }
+
+ var (
+ importMissing *modload.ImportMissingError
+ ambiguous *modload.AmbiguousImportError
+ )
+ if !errors.As(err, &importMissing) && !errors.As(err, &ambiguous) {
+ // The package, which is a dependency of something we care about, has some
+ // problem that we can't resolve with a version change.
+ // Leave the error for the final LoadPackages call.
+ continue
+ }
+
+ path := path
+ r.work.Add(func() {
+ findPackage(ctx, path, module.Version{})
+ })
+ }
+ <-r.work.Idle()
}
-// getQuery evaluates the given (package or module) path and version
-// to determine the underlying module version being requested.
-// If forceModulePath is set, getQuery must interpret path
-// as a module path.
-func getQuery(ctx context.Context, path, vers string, prevM module.Version, forceModulePath bool) (module.Version, error) {
- if (prevM.Version != "") != forceModulePath {
- // We resolve package patterns by calling QueryPattern, which does not
- // accept a previous version and therefore cannot take it into account for
- // the "latest" or "patch" queries.
- // If we are resolving a package path or pattern, the caller has already
- // resolved any existing packages to their containing module(s), and
- // will set both prevM.Version and forceModulePath for those modules.
- // The only remaining package patterns are those that are not already
- // provided by the build list, which are indicated by
- // an empty prevM.Version.
- base.Fatalf("go get: internal error: prevM may be set if and only if forceModulePath is set")
+// errVersionChange is a sentinel error indicating that a module's version needs
+// to be updated before its dependencies can be loaded.
+var errVersionChange = errors.New("version change needed")
+
+// resolveCandidates resolves candidates sets that are attached to the given
+// queries and/or needed to provide the given missing-package dependencies.
+//
+// resolveCandidates starts by resolving one module version from each
+// unambiguous pathSet attached to the given queries.
+//
+// If no unambiguous query results in a change to the build list,
+// resolveCandidates modifies the build list by adding one module version from
+// each pathSet in missing, but does not mark those versions as resolved
+// (so they can still be modified by other queries).
+//
+// If that still does not result in any changes to the build list,
+// resolveCandidates revisits the ambiguous query candidates and resolves them
+// arbitrarily in order to guarantee forward progress.
+//
+// If all pathSets are resolved without any changes to the build list,
+// resolveCandidates returns with changed=false.
+func (r *resolver) resolveCandidates(ctx context.Context, queries []*query, upgrades []pathSet) (changed bool) {
+ defer base.ExitIfErrors()
+
+ // Note: this is O(N²) with the number of pathSets in the worst case.
+ //
+ // We could perhaps get it down to O(N) if we were to index the pathSets
+ // by module path, so that we only revisit a given pathSet when the
+ // version of some module in its containingPackage list has been determined.
+ //
+ // However, N tends to be small, and most candidate sets will include only one
+ // candidate module (so they will be resolved in the first iteration), so for
+ // now we'll stick to the simple O(N²) approach.
+
+ resolved := 0
+ for {
+ prevResolved := resolved
+
+ for _, q := range queries {
+ unresolved := q.candidates[:0]
+
+ for _, cs := range q.candidates {
+ if cs.err != nil {
+ reportError(q, cs.err)
+ resolved++
+ continue
+ }
+
+ filtered, isPackage, m, unique := r.disambiguate(cs)
+ if !unique {
+ unresolved = append(unresolved, filtered)
+ continue
+ }
+
+ if m.Path == "" {
+ // The query is not viable. Choose an arbitrary candidate from
+ // before filtering and “resolve” it to report a conflict.
+ isPackage, m = r.chooseArbitrarily(cs)
+ }
+ if isPackage {
+ q.matchesPackages = true
+ }
+ r.resolve(q, m)
+ resolved++
+ }
+
+ q.candidates = unresolved
+ }
+
+ base.ExitIfErrors()
+ if resolved == prevResolved {
+ break // No unambiguous candidate remains.
+ }
}
- // If vers is a query like "latest", we should ignore retracted and excluded
- // versions. If vers refers to a specific version or commit like "v1.0.0"
- // or "master", we should only ignore excluded versions.
- allowed := modload.CheckAllowed
- if modload.IsRevisionQuery(vers) {
- allowed = modload.CheckExclusions
- } else if vers == "upgrade" || vers == "patch" {
- allowed = checkAllowedOrCurrent(prevM.Version)
+ if changed := r.updateBuildList(ctx, nil); changed {
+ // The build list has changed, so disregard any missing packages: they might
+ // now be determined by requirements in the build list, which we would
+ // prefer to use instead of arbitrary "latest" versions.
+ return true
}
- // If the query must be a module path, try only that module path.
- if forceModulePath {
- if path == modload.Target.Path {
- if vers != "latest" {
- return module.Version{}, fmt.Errorf("can't get a specific version of the main module")
- }
+ // Arbitrarily add a "latest" version that provides each missing package, but
+ // do not mark the version as resolved: we still want to allow the explicit
+ // queries to modify the resulting versions.
+ var tentative []module.Version
+ for _, cs := range upgrades {
+ if cs.err != nil {
+ base.Errorf("go get: %v", cs.err)
+ continue
}
- info, err := modload.Query(ctx, path, vers, prevM.Version, allowed)
- if err == nil {
- if info.Version != vers && info.Version != prevM.Version {
- logOncef("go: %s %s => %s", path, vers, info.Version)
+ filtered, _, m, unique := r.disambiguate(cs)
+ if !unique {
+ _, m = r.chooseArbitrarily(filtered)
+ }
+ if m.Path == "" {
+ // There is no viable candidate for the missing package.
+ // Leave it unresolved.
+ continue
+ }
+ tentative = append(tentative, m)
+ }
+ base.ExitIfErrors()
+ if changed := r.updateBuildList(ctx, tentative); changed {
+ return true
+ }
+
+ // The build list will be the same on the next iteration as it was on this
+ // iteration, so any ambiguous queries will remain so. In order to make
+ // progress, resolve them arbitrarily but deterministically.
+ //
+ // If that results in conflicting versions, the user can re-run 'go get'
+ // with additional explicit versions for the conflicting packages or
+ // modules.
+ for _, q := range queries {
+ for _, cs := range q.candidates {
+ isPackage, m := r.chooseArbitrarily(cs)
+ if isPackage {
+ q.matchesPackages = true
}
- return module.Version{Path: path, Version: info.Version}, nil
+ r.resolve(q, m)
+ }
+ }
+ return r.updateBuildList(ctx, nil)
+}
+
+// disambiguate eliminates candidates from cs that conflict with other module
+// versions that have already been resolved. If there is only one (unique)
+// remaining candidate, disambiguate returns that candidate, along with
+// an indication of whether that result interprets cs.path as a package
+//
+// Note: we're only doing very simple disambiguation here. The goal is to
+// reproduce the user's intent, not to find a solution that a human couldn't.
+// In the vast majority of cases, we expect only one module per pathSet,
+// but we want to give some minimal additional tools so that users can add an
+// extra argument or two on the command line to resolve simple ambiguities.
+func (r *resolver) disambiguate(cs pathSet) (filtered pathSet, isPackage bool, m module.Version, unique bool) {
+ if len(cs.pkgMods) == 0 && cs.mod.Path == "" {
+ panic("internal error: resolveIfUnambiguous called with empty pathSet")
+ }
+
+ for _, m := range cs.pkgMods {
+ if _, ok := r.noneForPath(m.Path); ok {
+ // A query with version "none" forces the candidate module to version
+ // "none", so we cannot use any other version for that module.
+ continue
}
- // If the query was "upgrade" or "patch" and the current version has been
- // replaced, check to see whether the error was for that same version:
- // if so, the version was probably replaced because it is invalid,
- // and we should keep that replacement without complaining.
- if vers == "upgrade" || vers == "patch" {
- var vErr *module.InvalidVersionError
- if errors.As(err, &vErr) && vErr.Version == prevM.Version && modload.Replacement(prevM).Path != "" {
- return prevM, nil
+ if m.Path == modload.Target.Path {
+ if m.Version == modload.Target.Version {
+ return pathSet{}, true, m, true
}
+ // The main module can only be set to its own version.
+ continue
}
- return module.Version{}, err
+ vr, ok := r.resolvedVersion[m.Path]
+ if !ok {
+ // m is a viable answer to the query, but other answers may also
+ // still be viable.
+ filtered.pkgMods = append(filtered.pkgMods, m)
+ continue
+ }
+
+ if vr.version != m.Version {
+ // Some query forces the candidate module to a version other than this
+ // one.
+ //
+ // The command could be something like
+ //
+ // go get example.com/foo/bar@none example.com/foo/bar/baz@latest
+ //
+ // in which case we *cannot* resolve the package from
+ // example.com/foo/bar (because it is constrained to version
+ // "none") and must fall through to module example.com/foo@latest.
+ continue
+ }
+
+ // Some query forces the candidate module *to* the candidate version.
+ // As a result, this candidate is the only viable choice to provide
+ // its package(s): any other choice would result in an ambiguous import
+ // for this path.
+ //
+ // For example, consider the command
+ //
+ // go get example.com/foo@latest example.com/foo/bar/baz@latest
+ //
+ // If modules example.com/foo and example.com/foo/bar both provide
+ // package example.com/foo/bar/baz, then we *must* resolve the package
+ // from example.com/foo: if we instead resolved it from
+ // example.com/foo/bar, we would have two copies of the package.
+ return pathSet{}, true, m, true
}
- // If the query may be either a package or a module, try it as a package path.
- // If it turns out to only exist as a module, we can detect the resulting
- // PackageNotInModuleError and avoid a second round-trip through (potentially)
- // all of the configured proxies.
- results, modOnly, err := modload.QueryPattern(ctx, path, vers, modload.Selected, allowed)
- if err != nil {
- return module.Version{}, err
+ if cs.mod.Path != "" {
+ vr, ok := r.resolvedVersion[cs.mod.Path]
+ if !ok || vr.version == cs.mod.Version {
+ filtered.mod = cs.mod
+ }
}
- if len(results) == 0 {
- // The path doesn't contain a wildcard, but was actually a
- // module path instead. Return that.
- return modOnly.Mod, nil
+
+ if len(filtered.pkgMods) == 1 &&
+ (filtered.mod.Path == "" || filtered.mod == filtered.pkgMods[0]) {
+ // Exactly one viable module contains the package with the given path
+ // (by far the common case), so we can resolve it unambiguously.
+ return pathSet{}, true, filtered.pkgMods[0], true
}
- m := results[0].Mod
- if m.Path != path {
- logOncef("go: found %s in %s %s", path, m.Path, m.Version)
- } else if m.Version != vers {
- logOncef("go: %s %s => %s", path, vers, m.Version)
+ if len(filtered.pkgMods) == 0 {
+ // All modules that could provide the path as a package conflict with other
+ // resolved arguments. If it can refer to a module instead, return that;
+ // otherwise, this pathSet cannot be resolved (and we will return the
+ // zero module.Version).
+ return pathSet{}, false, filtered.mod, true
}
- return m, nil
+
+ // The query remains ambiguous: there are at least two different modules
+ // to which cs.path could refer.
+ return filtered, false, module.Version{}, false
}
-// reportRetractions prints warnings if any modules in the build list are
-// retracted.
-func reportRetractions(ctx context.Context) {
- // Query for retractions of modules in the build list.
- // Use modload.ListModules, since that provides information in the same format
- // as 'go list -m'. Don't query for "all", since that's not allowed outside a
- // module.
- buildList := modload.LoadedModules()
- args := make([]string, 0, len(buildList))
- for _, m := range buildList {
- if m.Version == "" {
- // main module or dummy target module
- continue
+// chooseArbitrarily returns an arbitrary (but deterministic) module version
+// from among those in the given set.
+//
+// chooseArbitrarily prefers module paths that were already in the build list at
+// the start of 'go get', prefers modules that provide packages over those that
+// do not, and chooses the first module meeting those criteria (so biases toward
+// longer paths).
+func (r *resolver) chooseArbitrarily(cs pathSet) (isPackage bool, m module.Version) {
+ // Prefer to upgrade some module that was already in the build list.
+ for _, m := range cs.pkgMods {
+ if r.initialSelected(m.Path) != "none" {
+ return true, m
}
- args = append(args, m.Path+"@"+m.Version)
}
- listU := false
- listVersions := false
- listRetractions := true
- mods := modload.ListModules(ctx, args, listU, listVersions, listRetractions)
- retractPath := ""
- for _, mod := range mods {
- if len(mod.Retracted) > 0 {
- if retractPath == "" {
- retractPath = mod.Path
- } else {
- retractPath = "<module>"
+
+ // Otherwise, arbitrarily choose the first module that provides the package.
+ if len(cs.pkgMods) > 0 {
+ return true, cs.pkgMods[0]
+ }
+
+ return false, cs.mod
+}
+
+// reportChanges logs resolved version changes to os.Stderr.
+func (r *resolver) reportChanges(queries []*query) {
+ for _, q := range queries {
+ if q.version == "none" {
+ continue
+ }
+
+ if q.pattern == "all" {
+ // To reduce noise for "all", describe module version changes rather than
+ // package versions.
+ seen := make(map[module.Version]bool)
+ for _, m := range q.resolved {
+ if seen[m] {
+ continue
+ }
+ seen[m] = true
+
+ before := r.initialSelected(m.Path)
+ if before == m.Version {
+ continue // m was resolved, but not changed
+ }
+
+ was := ""
+ if before != "" {
+ was = fmt.Sprintf(" (was %s)", before)
+ }
+ fmt.Fprintf(os.Stderr, "go: %v added %s %s%s\n", q, m.Path, m.Version, was)
+ }
+ continue
+ }
+
+ for _, m := range q.resolved {
+ before := r.initialSelected(m.Path)
+ if before == m.Version {
+ continue // m was resolved, but not changed
+ }
+
+ was := ""
+ if before != "" {
+ was = fmt.Sprintf(" (was %s)", before)
+ }
+ switch {
+ case q.isWildcard():
+ if q.matchesPath(m.Path) {
+ fmt.Fprintf(os.Stderr, "go: matched %v as %s %s%s\n", q, m.Path, m.Version, was)
+ } else {
+ fmt.Fprintf(os.Stderr, "go: matched %v in %s %s%s\n", q, m.Path, m.Version, was)
+ }
+ case q.matchesPackages:
+ fmt.Fprintf(os.Stderr, "go: found %v in %s %s%s\n", q, m.Path, m.Version, was)
+ default:
+ fmt.Fprintf(os.Stderr, "go: found %v in %s %s%s\n", q, m.Path, m.Version, was)
}
- rationale := modload.ShortRetractionRationale(mod.Retracted[0])
- logOncef("go: warning: %s@%s is retracted: %s", mod.Path, mod.Version, rationale)
}
}
- if modload.HasModRoot() && retractPath != "" {
- logOncef("go: run 'go get %s@latest' to switch to the latest unretracted version", retractPath)
- }
+
+ // TODO(#33284): Also print relevant upgrades.
}
-var loggedLines sync.Map
+// resolve records that module m must be at its indicated version (which may be
+// "none") due to query q. If some other query forces module m to be at a
+// different version, resolve reports a conflict error.
+func (r *resolver) resolve(q *query, m module.Version) {
+ if m.Path == "" {
+ panic("internal error: resolving a module.Version with an empty path")
+ }
+
+ if m.Path == modload.Target.Path && m.Version != modload.Target.Version {
+ reportError(q, &modload.QueryMatchesMainModuleError{
+ Pattern: q.pattern,
+ Query: q.version,
+ })
+ return
+ }
-func logOncef(format string, args ...interface{}) {
- msg := fmt.Sprintf(format, args...)
- if _, dup := loggedLines.LoadOrStore(msg, true); !dup {
- fmt.Fprintln(os.Stderr, msg)
+ vr, ok := r.resolvedVersion[m.Path]
+ if ok && vr.version != m.Version {
+ reportConflict(q, m, vr)
+ return
}
+ r.resolvedVersion[m.Path] = versionReason{m.Version, q}
+ q.resolved = append(q.resolved, m)
}
-// checkAllowedOrCurrent is like modload.CheckAllowed, but always allows the
-// current version (even if it is retracted or otherwise excluded).
-func checkAllowedOrCurrent(current string) modload.AllowedFunc {
- if current == "" {
- return modload.CheckAllowed
+// updateBuildList updates the module loader's global build list to be
+// consistent with r.resolvedVersion, and to include additional modules
+// provided that they do not conflict with the resolved versions.
+//
+// If the additional modules conflict with the resolved versions, they will be
+// downgraded to a non-conflicting version (possibly "none").
+func (r *resolver) updateBuildList(ctx context.Context, additions []module.Version) (changed bool) {
+ if len(additions) == 0 && len(r.resolvedVersion) == r.buildListResolvedVersions {
+ return false
}
- return func(ctx context.Context, m module.Version) error {
- if m.Version == current {
- return nil
+ defer base.ExitIfErrors()
+
+ resolved := make([]module.Version, 0, len(r.resolvedVersion))
+ for mPath, rv := range r.resolvedVersion {
+ if mPath != modload.Target.Path {
+ resolved = append(resolved, module.Version{Path: mPath, Version: rv.version})
}
- return modload.CheckAllowed(ctx, m)
}
+
+ if err := modload.EditBuildList(ctx, additions, resolved); err != nil {
+ var constraint *modload.ConstraintError
+ if !errors.As(err, &constraint) {
+ base.Errorf("go get: %v", err)
+ return false
+ }
+
+ reason := func(m module.Version) string {
+ rv, ok := r.resolvedVersion[m.Path]
+ if !ok {
+ panic(fmt.Sprintf("internal error: can't find reason for requirement on %v", m))
+ }
+ return rv.reason.ResolvedString(module.Version{Path: m.Path, Version: rv.version})
+ }
+ for _, c := range constraint.Conflicts {
+ base.Errorf("go get: %v requires %v, not %v", reason(c.Source), c.Dep, reason(c.Constraint))
+ }
+ return false
+ }
+
+ buildList := modload.LoadAllModules(ctx)
+ r.buildListResolvedVersions = len(r.resolvedVersion)
+ if reflect.DeepEqual(r.buildList, buildList) {
+ return false
+ }
+ r.buildList = buildList
+ r.buildListVersion = make(map[string]string, len(r.buildList))
+ for _, m := range r.buildList {
+ r.buildListVersion[m.Path] = m.Version
+ }
+ return true
+}
+
+// isNoSuchModuleVersion reports whether err indicates that the requested module
+// does not exist at the requested version, either because the module does not
+// exist at all or because it does not include that specific version.
+func isNoSuchModuleVersion(err error) bool {
+ var noMatch *modload.NoMatchingVersionError
+ return errors.Is(err, os.ErrNotExist) || errors.As(err, &noMatch)
+}
+
+// isNoSuchPackageVersion reports whether err indicates that the requested
+// package does not exist at the requested version, either because no module
+// that could contain it exists at that version, or because every such module
+// that does exist does not actually contain the package.
+func isNoSuchPackageVersion(err error) bool {
+ var noPackage *modload.PackageNotInModuleError
+ return isNoSuchModuleVersion(err) || errors.As(err, &noPackage)
}