I needed to also update TestScript/mod_sumdb_golang.
It had been relying on 'go list -mod=mod' to add both the go.mod and
go.sum entries for the named package, but when 'go get' actually adds
all of the needed dependencies, lazy loading kicks in and 'go list'
doesn't end up needing the checksums for go.mod files.
We didn't detect the skew before because the 'go list' command was
(unexpectedly) also adding the missing dependencies, which triggered a
deep scan of the complete module graph.
For #45979
Change-Id: Ica917dee22c83ffa71c6ad0f2e189f911b73edf4
Reviewed-on: https://go-review.googlesource.com/c/go/+/328231
Trust: Bryan C. Mills <bcmills@google.com>
Run-TryBot: Bryan C. Mills <bcmills@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Michael Matloob <matloob@golang.org>
Tags: imports.AnyTags(),
VendorModulesInGOROOTSrc: true,
LoadTests: *getT,
+ AssumeRootsImported: true, // After 'go get foo', imports of foo should build.
SilencePackageErrors: true, // May be fixed by subsequent upgrades or downgrades.
}
// roots — but in a lazy module it may pull in previously-irrelevant
// transitive dependencies.
- newRS, rsErr := updateRoots(ctx, rs.direct, rs, nil, nil)
+ newRS, rsErr := updateRoots(ctx, rs.direct, rs, nil, nil, false)
if rsErr != nil {
// Failed to update roots, perhaps because of an error in a transitive
// dependency needed for the update. Return the original Requirements
return tidyLazyRoots(ctx, rs.direct, pkgs)
}
-func updateRoots(ctx context.Context, direct map[string]bool, rs *Requirements, pkgs []*loadPkg, add []module.Version) (*Requirements, error) {
+func updateRoots(ctx context.Context, direct map[string]bool, rs *Requirements, pkgs []*loadPkg, add []module.Version, rootsImported bool) (*Requirements, error) {
if rs.depth == eager {
return updateEagerRoots(ctx, direct, rs, add)
}
- return updateLazyRoots(ctx, direct, rs, pkgs, add)
+ return updateLazyRoots(ctx, direct, rs, pkgs, add, rootsImported)
}
// tidyLazyRoots returns a minimal set of root requirements that maintains the
//
// (See https://golang.org/design/36460-lazy-module-loading#invariants for more
// detail.)
-func updateLazyRoots(ctx context.Context, direct map[string]bool, rs *Requirements, pkgs []*loadPkg, add []module.Version) (*Requirements, error) {
+func updateLazyRoots(ctx context.Context, direct map[string]bool, rs *Requirements, pkgs []*loadPkg, add []module.Version, rootsImported bool) (*Requirements, error) {
roots := rs.rootModules
rootsUpgraded := false
//
// (This is the “import invariant” that makes lazy loading possible.)
+ case rootsImported && pkg.flags.has(pkgFromRoot):
+ // pkg is a transitive dependency of some root, and we are treating the
+ // roots as if they are imported by the main module (as in 'go get').
+
case pkg.flags.has(pkgIsRoot):
// pkg is a root of the package-import graph. (Generally this means that
// it matches a command-line argument.) We want future invocations of the
for _, n := range mPathCount {
if n > 1 {
var err error
- rs, err = updateRoots(ctx, rs.direct, rs, nil, nil)
+ rs, err = updateRoots(ctx, rs.direct, rs, nil, nil, false)
if err != nil {
base.Fatalf("go: %v", err)
}
// if the flag is set to "readonly" (the default) or "vendor".
ResolveMissingImports bool
+ // AssumeRootsImported indicates that the transitive dependencies of the root
+ // packages should be treated as if those roots will be imported by the main
+ // module.
+ AssumeRootsImported bool
+
// AllowPackage, if non-nil, is called after identifying the module providing
// each package. If AllowPackage returns a non-nil error, that error is set
// for the package, and the imports and test of that package will not be
// are also roots (and must be marked pkgIsRoot).
pkgIsRoot
+ // pkgFromRoot indicates that the package is in the transitive closure of
+ // imports starting at the roots. (Note that every package marked as pkgIsRoot
+ // is also trivially marked pkgFromRoot.)
+ pkgFromRoot
+
// pkgImportsLoaded indicates that the imports and testImports fields of a
// loadPkg have been populated.
pkgImportsLoaded
// iteration so we don't need to also update it here. (That would waste time
// computing a "direct" map that we'll have to recompute later anyway.)
direct := ld.requirements.direct
- rs, err := updateRoots(ctx, direct, ld.requirements, noPkgs, toAdd)
+ rs, err := updateRoots(ctx, direct, ld.requirements, noPkgs, toAdd, ld.AssumeRootsImported)
if err != nil {
// If an error was found in a newly added module, report the package
// import stack instead of the module requirement stack. Packages
addRoots = tidy.rootModules
}
- rs, err = updateRoots(ctx, direct, rs, ld.pkgs, addRoots)
+ rs, err = updateRoots(ctx, direct, rs, ld.pkgs, addRoots, ld.AssumeRootsImported)
if err != nil {
// We don't actually know what even the root requirements are supposed to be,
// so we can't proceed with loading. Return the error to the caller
// This package matches a root pattern by virtue of being in "all".
flags |= pkgIsRoot
}
+ if flags.has(pkgIsRoot) {
+ flags |= pkgFromRoot
+ }
old := pkg.flags.update(flags)
new := old | flags
ld.applyPkgFlags(ctx, dep, pkgInAll)
}
}
+
+ if new.has(pkgFromRoot) && !old.has(pkgFromRoot|pkgImportsLoaded) {
+ for _, dep := range pkg.imports {
+ ld.applyPkgFlags(ctx, dep, pkgFromRoot)
+ }
+ }
}
// preloadRootModules loads the module requirements needed to identify the
}
module.Sort(toAdd)
- rs, err := updateRoots(ctx, ld.requirements.direct, ld.requirements, nil, toAdd)
+ rs, err := updateRoots(ctx, ld.requirements.direct, ld.requirements, nil, toAdd, ld.AssumeRootsImported)
if err != nil {
// We are missing some root dependency, and for some reason we can't load
// enough of the module dependency graph to add the missing root. Package
stderr '^m.go:3:8: no required module provides package rsc\.io/quote; to add it:\n\tgo get rsc.io/quote$'
-# Unfortunately, the suggested 'go get' command leaves us with another problem.
+# When we run the suggested 'go get' command, the new dependency can be used
+# immediately, even though 'go get' marks it as 'indirect'.
#
-# TODO(#45979): After 'go get', the 'go list' command from above should succeed.
+# TODO(#45979): Should we swap this default state, so that new dependencies
+# are added as direct unless otherwise noted?
go get rsc.io/quote
+grep 'rsc.io/quote v\d+\.\d+\.\d+ // indirect$' go.mod
+! grep 'rsc.io/quote v\d+\.\d+\.\d+$' go.mod
-! go list -deps .
-stderr '^go: updates to go.mod needed; to update it:\n\tgo mod tidy'
-[!short] ! go build .
-stderr '^go: updates to go.mod needed; to update it:\n\tgo mod tidy'
-
-
-# After running the suggested 'go mod tidy' command, the build
-# should succeed.
-go mod tidy
go list -deps .
+! stderr .
[!short] go build .
+[!short] ! stderr .
+
+
+# 'go get .' (or 'go mod tidy') removes the indirect mark.
+
+go get .
+grep 'rsc.io/quote v\d+\.\d+\.\d+$' go.mod
+! grep 'rsc.io/quote v\d+\.\d+\.\d+ // indirect$' go.mod
-- go.mod --
stdout '^sum.golang.org$'
# Download direct from github.
+
[!net] skip
[!exec:git] skip
env GOSUMDB=sum.golang.org
env GOPROXY=direct
+
go get -d rsc.io/quote@v1.5.2
cp go.sum saved.sum
+
# Download from proxy.golang.org with go.sum entry already.
# Use 'go list' instead of 'go get' since the latter may download extra go.mod
# files not listed in go.sum.
+
go clean -modcache
env GOSUMDB=
env GOPROXY=
-go list -x -deps rsc.io/quote
+
+go list -x -m all # Download go.mod files.
! stderr github
stderr proxy.golang.org/rsc.io/quote
! stderr sum.golang.org/tile
! stderr sum.golang.org/lookup/rsc.io/quote
+
+go list -x -deps rsc.io/quote # Download module source.
+! stderr github
+stderr proxy.golang.org/rsc.io/quote
+! stderr sum.golang.org/tile
+! stderr sum.golang.org/lookup/rsc.io/quote
+
cmp go.sum saved.sum
+
# Download again.
# Should use the checksum database to validate new go.sum lines,
# but not need to fetch any new data from the proxy.
+
rm go.sum
-go list -mod=mod -x rsc.io/quote
+
+go list -mod=mod -x -m all # Add checksums for go.mod files.
+stderr sum.golang.org/tile
! stderr github
! stderr proxy.golang.org/rsc.io/quote
-stderr sum.golang.org/tile
stderr sum.golang.org/lookup/rsc.io/quote
+
+go list -mod=mod -x rsc.io/quote # Add checksums for module source.
+! stderr . # Adds checksums, but for entities already in the module cache.
+
cmp go.sum saved.sum
+
# test fallback to direct
+
env TESTGOPROXY404=1
go clean -modcache
rm go.sum
-go list -mod=mod -x rsc.io/quote
+
+go list -mod=mod -x -m all # Download go.mod files
+stderr 'proxy.golang.org.*404 testing'
+stderr github.com/rsc
+
+go list -mod=mod -x rsc.io/quote # Download module source.
stderr 'proxy.golang.org.*404 testing'
stderr github.com/rsc
+
cmp go.sum saved.sum
+
-- go.mod --
module m