1 // Copyright 2018 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
15 "cmd/go/internal/base"
17 "cmd/go/internal/gover"
18 "cmd/go/internal/modfetch"
19 "cmd/go/internal/modfetch/codehost"
20 "cmd/go/internal/modload"
21 "cmd/go/internal/toolchain"
23 "golang.org/x/mod/module"
26 var cmdDownload = &base.Command{
27 UsageLine: "go mod download [-x] [-json] [-reuse=old.json] [modules]",
28 Short: "download modules to local cache",
30 Download downloads the named modules, which can be module patterns selecting
31 dependencies of the main module or module queries of the form path@version.
33 With no arguments, download applies to the modules needed to build and test
34 the packages in the main module: the modules explicitly required by the main
35 module if it is at 'go 1.17' or higher, or all transitively-required modules
36 if at 'go 1.16' or lower.
38 The go command will automatically download modules as needed during ordinary
39 execution. The "go mod download" command is useful mainly for pre-filling
40 the local cache or to compute the answers for a Go module proxy.
42 By default, download writes nothing to standard output. It may print progress
43 messages and errors to standard error.
45 The -json flag causes download to print a sequence of JSON objects
46 to standard output, describing each downloaded module (or failure),
47 corresponding to this Go struct:
50 Path string // module path
51 Query string // version query corresponding to this version
52 Version string // module version
53 Error string // error loading module
54 Info string // absolute path to cached .info file
55 GoMod string // absolute path to cached .mod file
56 Zip string // absolute path to cached .zip file
57 Dir string // absolute path to cached source root directory
58 Sum string // checksum for path, version (as in go.sum)
59 GoModSum string // checksum for go.mod (as in go.sum)
60 Origin any // provenance of module
61 Reuse bool // reuse of old module info is safe
64 The -reuse flag accepts the name of file containing the JSON output of a
65 previous 'go mod download -json' invocation. The go command may use this
66 file to determine that a module is unchanged since the previous invocation
67 and avoid redownloading it. Modules that are not redownloaded will be marked
68 in the new output by setting the Reuse field to true. Normally the module
69 cache provides this kind of reuse automatically; the -reuse flag can be
70 useful on systems that do not preserve the module cache.
72 The -x flag causes download to print the commands download executes.
74 See https://golang.org/ref/mod#go-mod-download for more about 'go mod download'.
76 See https://golang.org/ref/mod#version-queries for more about version queries.
81 downloadJSON = cmdDownload.Flag.Bool("json", false, "")
82 downloadReuse = cmdDownload.Flag.String("reuse", "", "")
86 cmdDownload.Run = runDownload // break init cycle
88 // TODO(jayconrod): https://golang.org/issue/35849 Apply -x to other 'go mod' commands.
89 cmdDownload.Flag.BoolVar(&cfg.BuildX, "x", false, "")
90 base.AddChdirFlag(&cmdDownload.Flag)
91 base.AddModCommonFlags(&cmdDownload.Flag)
94 // A ModuleJSON describes the result of go mod download.
95 type ModuleJSON struct {
96 Path string `json:",omitempty"`
97 Version string `json:",omitempty"`
98 Query string `json:",omitempty"`
99 Error string `json:",omitempty"`
100 Info string `json:",omitempty"`
101 GoMod string `json:",omitempty"`
102 Zip string `json:",omitempty"`
103 Dir string `json:",omitempty"`
104 Sum string `json:",omitempty"`
105 GoModSum string `json:",omitempty"`
107 Origin *codehost.Origin `json:",omitempty"`
108 Reuse bool `json:",omitempty"`
111 func runDownload(ctx context.Context, cmd *base.Command, args []string) {
112 modload.InitWorkfile()
114 // Check whether modules are enabled and whether we're in a module.
115 modload.ForceUseModules = true
116 modload.ExplicitWriteGoMod = true
117 haveExplicitArgs := len(args) > 0
119 if modload.HasModRoot() || modload.WorkFilePath() != "" {
120 modload.LoadModFile(ctx) // to fill MainModules
122 if haveExplicitArgs {
123 for _, mainModule := range modload.MainModules.Versions() {
124 targetAtUpgrade := mainModule.Path + "@upgrade"
125 targetAtPatch := mainModule.Path + "@patch"
126 for _, arg := range args {
128 case mainModule.Path, targetAtUpgrade, targetAtPatch:
129 os.Stderr.WriteString("go: skipping download of " + arg + " that resolves to the main module\n")
133 } else if modload.WorkFilePath() != "" {
134 // TODO(#44435): Think about what the correct query is to download the
135 // right set of modules. Also see code review comment at
136 // https://go-review.googlesource.com/c/go/+/359794/comments/ce946a80_6cf53992.
137 args = []string{"all"}
139 mainModule := modload.MainModules.Versions()[0]
140 modFile := modload.MainModules.ModFile(mainModule)
141 if modFile.Go == nil || gover.Compare(modFile.Go.Version, gover.ExplicitIndirectVersion) < 0 {
142 if len(modFile.Require) > 0 {
143 args = []string{"all"}
146 // As of Go 1.17, the go.mod file explicitly requires every module
147 // that provides any package imported by the main module.
148 // 'go mod download' is typically run before testing packages in the
149 // main module, so by default we shouldn't download the others
150 // (which are presumed irrelevant to the packages in the main module).
151 // See https://golang.org/issue/44435.
153 // However, we also need to load the full module graph, to ensure that
154 // we have downloaded enough of the module graph to run 'go list all',
155 // 'go mod graph', and similar commands.
156 _, err := modload.LoadModGraph(ctx, "")
158 // TODO(#64008): call base.Fatalf instead of toolchain.SwitchOrFatal
159 // here, since we can only reach this point with an outdated toolchain
160 // if the go.mod file is inconsistent.
161 toolchain.SwitchOrFatal(ctx, err)
164 for _, m := range modFile.Require {
165 args = append(args, m.Mod.Path)
172 if modload.HasModRoot() {
173 os.Stderr.WriteString("go: no module dependencies to download\n")
175 base.Errorf("go: no modules specified (see 'go help mod download')")
180 if *downloadReuse != "" && modload.HasModRoot() {
181 base.Fatalf("go mod download -reuse cannot be used inside a module")
184 var mods []*ModuleJSON
186 sem := make(chan token, runtime.GOMAXPROCS(0))
187 infos, infosErr := modload.ListModules(ctx, args, 0, *downloadReuse)
189 // There is a bit of a chicken-and-egg problem here: ideally we need to know
190 // which Go version to switch to to download the requested modules, but if we
191 // haven't downloaded the module's go.mod file yet the GoVersion field of its
192 // info struct is not yet populated.
194 // We also need to be careful to only print the info for each module once
195 // if the -json flag is set.
197 // In theory we could go through each module in the list, attempt to download
198 // its go.mod file, and record the maximum version (either from the file or
199 // from the resulting TooNewError), all before we try the actual full download
202 // For now, we go ahead and try all the downloads and collect the errors, and
203 // if any download failed due to a TooNewError, we switch toolchains and try
204 // again. Any downloads that already succeeded will still be in cache.
205 // That won't give optimal concurrency (we'll do two batches of concurrent
206 // downloads instead of all in one batch), and it might add a little overhead
207 // to look up the downloads from the first batch in the module cache when
208 // we see them again in the second batch. On the other hand, it's way simpler
209 // to implement, and not really any more expensive if the user is requesting
210 // no explicit arguments (their go.mod file should already list an appropriate
211 // toolchain version) or only one module (as is used by the Go Module Proxy).
214 var sw toolchain.Switcher
219 // Otherwise, wait to report infosErr after we have downloaded
223 if !haveExplicitArgs && modload.WorkFilePath() == "" {
224 // 'go mod download' is sometimes run without arguments to pre-populate the
225 // module cache. In modules that aren't at go 1.17 or higher, it may fetch
226 // modules that aren't needed to build packages in the main module. This is
227 // usually not intended, so don't save sums for downloaded modules
228 // (golang.org/issue/45332). We do still fix inconsistencies in go.mod
231 // TODO(#64008): In the future, report an error if go.mod or go.sum need to
232 // be updated after loading the build list. This may require setting
233 // the mode to "mod" or "readonly" depending on haveExplicitArgs.
234 if err := modload.WriteGoMod(ctx, modload.WriteOpts{}); err != nil {
239 var downloadErrs sync.Map
240 for _, info := range infos {
241 if info.Replace != nil {
244 if info.Version == "" && info.Error == nil {
245 // main module or module replaced with file path.
246 // Nothing to download.
251 Version: info.Version,
256 mods = append(mods, m)
257 if info.Error != nil {
258 m.Error = info.Error.Err
266 err := DownloadModule(ctx, m)
268 downloadErrs.Store(m, err)
269 m.Error = err.Error()
275 // Fill semaphore channel to wait for goroutines to finish.
276 for n := cap(sem); n > 0; n-- {
280 // If there were explicit arguments
281 // (like 'go mod download golang.org/x/tools@latest'),
282 // check whether we need to upgrade the toolchain in order to download them.
284 // (If invoked without arguments, we expect the module graph to already
285 // be tidy and the go.mod file to declare a 'go' version that satisfies
286 // transitive requirements. If that invariant holds, then we should have
287 // already upgraded when we loaded the module graph, and should not need
288 // an additional check here. See https://go.dev/issue/45551.)
290 // We also allow upgrades if in a workspace because in workspace mode
291 // with no arguments we download the module pattern "all",
292 // which may include dependencies that are normally pruned out
293 // of the individual modules in the workspace.
294 if haveExplicitArgs || modload.WorkFilePath() != "" {
295 var sw toolchain.Switcher
296 // Add errors to the Switcher in deterministic order so that they will be
297 // logged deterministically.
298 for _, m := range mods {
299 if erri, ok := downloadErrs.Load(m); ok {
300 sw.Error(erri.(error))
303 // Only call sw.Switch if it will actually switch.
304 // Otherwise, we may want to write the errors as JSON
305 // (instead of using base.Error as sw.Switch would),
306 // and we may also have other errors to report from the
307 // initial infos returned by ListModules.
314 for _, m := range mods {
315 b, err := json.MarshalIndent(m, "", "\t")
319 os.Stdout.Write(append(b, '\n'))
321 base.SetExitStatus(1)
325 for _, m := range mods {
327 base.Error(errors.New(m.Error))
333 // If there were explicit arguments, update go.mod and especially go.sum.
334 // 'go mod download mod@version' is a useful way to add a sum without using
335 // 'go get mod@version', which may have other side effects. We print this in
336 // some error message hints.
338 // If we're in workspace mode, update go.work.sum with checksums for all of
339 // the modules we downloaded that aren't already recorded. Since a requirement
340 // in one module may upgrade a dependency of another, we can't be sure that
341 // the import graph matches the import graph of any given module in isolation,
342 // so we may end up needing to load packages from modules that wouldn't
343 // otherwise be relevant.
345 // TODO(#44435): If we adjust the set of modules downloaded in workspace mode,
346 // we may also need to adjust the logic for saving checksums here.
348 // Don't save sums for 'go mod download' without arguments unless we're in
349 // workspace mode; see comment above.
350 if haveExplicitArgs || modload.WorkFilePath() != "" {
351 if err := modload.WriteGoMod(ctx, modload.WriteOpts{}); err != nil {
356 // If there was an error matching some of the requested packages, emit it now
357 // (after we've written the checksums for the modules that were downloaded
364 // DownloadModule runs 'go mod download' for m.Path@m.Version,
365 // leaving the results (including any error) in m itself.
366 func DownloadModule(ctx context.Context, m *ModuleJSON) error {
368 _, file, err := modfetch.InfoFile(ctx, m.Path, m.Version)
373 m.GoMod, err = modfetch.GoModFile(ctx, m.Path, m.Version)
377 m.GoModSum, err = modfetch.GoModSum(ctx, m.Path, m.Version)
381 mod := module.Version{Path: m.Path, Version: m.Version}
382 m.Zip, err = modfetch.DownloadZip(ctx, mod)
386 m.Sum = modfetch.Sum(ctx, mod)
387 m.Dir, err = modfetch.Download(ctx, mod)