cmd/go: parallelize matchPackages work in each module

In each module matchPackages looks in, when doing the walk, do the
scanDir call in a par.Queue so all the read work can be done in
parallel.

Change-Id: I27153dbb3a2ed417ca24972f47134e9e914a55d1
Reviewed-on: https://go-review.googlesource.com/c/go/+/404097
Reviewed-by: Michael Matloob <matloob@golang.org>
Run-TryBot: Michael Matloob <matloob@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Bryan Mills <bcmills@google.com>

diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go
index 60c68860edf328fbe1d557272b715f9dee6eb38b..4b90392d94d459e4e6d944a2e519a67dc7ebed9f 100644 (file)
--- a/src/cmd/go/internal/modload/search.go
+++ b/src/cmd/go/internal/modload/search.go
@@ -12,12 +12,16 @@ import (
        "os"
        "path"
        "path/filepath"
+       "runtime"
+       "sort"
        "strings"
+       "sync"
 
        "cmd/go/internal/cfg"
        "cmd/go/internal/fsys"
        "cmd/go/internal/imports"
        "cmd/go/internal/modindex"
+       "cmd/go/internal/par"
        "cmd/go/internal/search"
 
        "golang.org/x/mod/module"
@@ -43,9 +47,15 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
                treeCanMatch = search.TreeCanMatchPattern(m.Pattern())
        }
 
+       var mu sync.Mutex
        have := map[string]bool{
                "builtin": true, // ignore pseudo-package that exists only for documentation
        }
+       addPkg := func(p string) {
+               mu.Lock()
+               m.Pkgs = append(m.Pkgs, p)
+               mu.Unlock()
+       }
        if !cfg.BuildContext.CgoEnabled {
                have["runtime/cgo"] = true // ignore during walk
        }
@@ -56,6 +66,8 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
                pruneGoMod
        )
 
+       q := par.NewQueue(runtime.GOMAXPROCS(0))
+
        walkPkgs := func(root, importPathRoot string, prune pruning) {
                root = filepath.Clean(root)
                err := fsys.Walk(root, func(path string, fi fs.FileInfo, err error) error {
@@ -110,9 +122,11 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
                        if !have[name] {
                                have[name] = true
                                if isMatch(name) {
-                                       if _, _, err := scanDir(root, path, tags); err != imports.ErrNoGo {
-                                               m.Pkgs = append(m.Pkgs, name)
-                                       }
+                                       q.Add(func() {
+                                               if _, _, err := scanDir(root, path, tags); err != imports.ErrNoGo {
+                                                       addPkg(name)
+                                               }
+                                       })
                                }
                        }
 
@@ -126,6 +140,12 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
                }
        }
 
+       // Wait for all in-flight operations to complete before returning.
+       defer func() {
+               <-q.Idle()
+               sort.Strings(m.Pkgs) // sort everything we added for determinism
+       }()
+
        if filter == includeStd {
                walkPkgs(cfg.GOROOTsrc, "", pruneGoMod)
                if treeCanMatch("cmd") {
@@ -169,7 +189,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
                        modPrefix = mod.Path
                }
                if mi, err := modindex.Get(root); err == nil {
-                       walkFromIndex(ctx, m, tags, root, mi, have, modPrefix)
+                       walkFromIndex(mi, modPrefix, isMatch, treeCanMatch, tags, have, addPkg)
                        continue
                } else if !errors.Is(err, modindex.ErrNotIndexed) {
                        m.AddError(err)
@@ -188,13 +208,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
 // walkFromIndex matches packages in a module using the module index. modroot
 // is the module's root directory on disk, index is the ModuleIndex for the
 // module, and importPathRoot is the module's path prefix.
-func walkFromIndex(ctx context.Context, m *search.Match, tags map[string]bool, modroot string, index *modindex.ModuleIndex, have map[string]bool, importPathRoot string) {
-       isMatch := func(string) bool { return true }
-       treeCanMatch := func(string) bool { return true }
-       if !m.IsMeta() {
-               isMatch = search.MatchPattern(m.Pattern())
-               treeCanMatch = search.TreeCanMatchPattern(m.Pattern())
-       }
+func walkFromIndex(index *modindex.ModuleIndex, importPathRoot string, isMatch, treeCanMatch func(string) bool, tags, have map[string]bool, addPkg func(string)) {
 loopPackages:
        for _, reldir := range index.Packages() {
                // Avoid .foo, _foo, and testdata subdirectory trees.
@@ -232,7 +246,7 @@ loopPackages:
                        have[name] = true
                        if isMatch(name) {
                                if _, _, err := index.ScanDir(reldir, tags); err != imports.ErrNoGo {
-                                       m.Pkgs = append(m.Pkgs, name)
+                                       addPkg(name)
                                }
                        }
                }