1 // Copyright 2012 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
12 "internal/singleflight"
25 "cmd/go/internal/base"
27 "cmd/go/internal/search"
31 "golang.org/x/mod/module"
34 // A Cmd describes how to use a version control system
35 // like Mercurial, Git, or Subversion.
38 Cmd string // name of binary to invoke command
39 RootNames []rootName // filename and mode indicating the root of a checkout directory
41 CreateCmd []string // commands to download a fresh copy of a repository
42 DownloadCmd []string // commands to download updates into an existing repository
44 TagCmd []tagCmd // commands to list tags
45 TagLookupCmd []tagCmd // commands to lookup tags before running tagSyncCmd
46 TagSyncCmd []string // commands to sync to specific tag
47 TagSyncDefault []string // commands to sync to default tag
52 RemoteRepo func(v *Cmd, rootDir string) (remoteRepo string, err error)
53 ResolveRepo func(v *Cmd, rootDir, remoteRepo string) (realRepo string, err error)
54 Status func(v *Cmd, rootDir string) (Status, error)
57 // Status is the current state of a local repository.
59 Revision string // Optional.
60 CommitTime time.Time // Optional.
61 Uncommitted bool // Required.
65 // VCSTestRepoURL is the URL of the HTTP server that serves the repos for
66 // vcs-test.golang.org.
68 // In tests, this is set to the URL of an httptest.Server hosting a
69 // cmd/go/internal/vcweb.Server.
72 // VCSTestHosts is the set of hosts supported by the vcs-test server.
75 // VCSTestIsLocalHost reports whether the given URL refers to a local
76 // (loopback) host, such as "localhost" or "127.0.0.1:8080".
77 VCSTestIsLocalHost func(*urlpkg.URL) bool
80 var defaultSecureScheme = map[string]bool{
88 func (v *Cmd) IsSecure(repo string) bool {
89 u, err := urlpkg.Parse(repo)
91 // If repo is not a URL, it's not secure.
94 if VCSTestRepoURL != "" && web.IsLocalHost(u) {
95 // If the vcstest server is in use, it may redirect to other local ports for
96 // other protocols (such as svn). Assume that all loopback addresses are
97 // secure during testing.
100 return v.isSecureScheme(u.Scheme)
103 func (v *Cmd) isSecureScheme(scheme string) bool {
106 // GIT_ALLOW_PROTOCOL is an environment variable defined by Git. It is a
107 // colon-separated list of schemes that are allowed to be used with git
108 // fetch/clone. Any scheme not mentioned will be considered insecure.
109 if allow := os.Getenv("GIT_ALLOW_PROTOCOL"); allow != "" {
110 for _, s := range strings.Split(allow, ":") {
118 return defaultSecureScheme[scheme]
121 // A tagCmd describes a command to list available tags
122 // that can be passed to tagSyncCmd.
124 cmd string // command to list tags
125 pattern string // regexp to extract tags from list
128 // vcsList lists the known version control systems
129 var vcsList = []*Cmd{
137 // vcsMod is a stub for the "mod" scheme. It's returned by
138 // repoRootForImportPathDynamic, but is otherwise not treated as a VCS command.
139 var vcsMod = &Cmd{Name: "mod"}
141 // vcsByCmd returns the version control system for the given
142 // command name (hg, git, svn, bzr).
143 func vcsByCmd(cmd string) *Cmd {
144 for _, vcs := range vcsList {
152 // vcsHg describes how to use Mercurial.
156 RootNames: []rootName{
157 {filename: ".hg", isDir: true},
160 CreateCmd: []string{"clone -U -- {repo} {dir}"},
161 DownloadCmd: []string{"pull"},
163 // We allow both tag and branch names as 'tags'
164 // for selecting a version. This lets people have
165 // a go.release.r60 branch and a go1 branch
166 // and make changes in both, without constantly
170 {"branches", `^(\S+)`},
172 TagSyncCmd: []string{"update -r {tag}"},
173 TagSyncDefault: []string{"update default"},
175 Scheme: []string{"https", "http", "ssh"},
176 PingCmd: "identify -- {scheme}://{repo}",
177 RemoteRepo: hgRemoteRepo,
181 func hgRemoteRepo(vcsHg *Cmd, rootDir string) (remoteRepo string, err error) {
182 out, err := vcsHg.runOutput(rootDir, "paths default")
186 return strings.TrimSpace(string(out)), nil
189 func hgStatus(vcsHg *Cmd, rootDir string) (Status, error) {
190 // Output changeset ID and seconds since epoch.
191 out, err := vcsHg.runOutputVerboseOnly(rootDir, `log -l1 -T {node}:{date|hgdate}`)
196 // Successful execution without output indicates an empty repo (no commits).
198 var commitTime time.Time
200 // Strip trailing timezone offset.
201 if i := bytes.IndexByte(out, ' '); i > 0 {
204 rev, commitTime, err = parseRevTime(out)
210 // Also look for untracked files.
211 out, err = vcsHg.runOutputVerboseOnly(rootDir, "status")
215 uncommitted := len(out) > 0
219 CommitTime: commitTime,
220 Uncommitted: uncommitted,
224 // parseRevTime parses commit details in "revision:seconds" format.
225 func parseRevTime(out []byte) (string, time.Time, error) {
226 buf := string(bytes.TrimSpace(out))
228 i := strings.IndexByte(buf, ':')
230 return "", time.Time{}, errors.New("unrecognized VCS tool output")
234 secs, err := strconv.ParseInt(string(buf[i+1:]), 10, 64)
236 return "", time.Time{}, fmt.Errorf("unrecognized VCS tool output: %v", err)
239 return rev, time.Unix(secs, 0), nil
242 // vcsGit describes how to use Git.
246 RootNames: []rootName{
247 {filename: ".git", isDir: true},
250 CreateCmd: []string{"clone -- {repo} {dir}", "-go-internal-cd {dir} submodule update --init --recursive"},
251 DownloadCmd: []string{"pull --ff-only", "submodule update --init --recursive"},
254 // tags/xxx matches a git tag named xxx
255 // origin/xxx matches a git branch named xxx on the default remote repository
256 {"show-ref", `(?:tags|origin)/(\S+)$`},
258 TagLookupCmd: []tagCmd{
259 {"show-ref tags/{tag} origin/{tag}", `((?:tags|origin)/\S+)$`},
261 TagSyncCmd: []string{"checkout {tag}", "submodule update --init --recursive"},
262 // both createCmd and downloadCmd update the working dir.
263 // No need to do more here. We used to 'checkout master'
264 // but that doesn't work if the default branch is not named master.
265 // DO NOT add 'checkout master' here.
266 // See golang.org/issue/9032.
267 TagSyncDefault: []string{"submodule update --init --recursive"},
269 Scheme: []string{"git", "https", "http", "git+ssh", "ssh"},
271 // Leave out the '--' separator in the ls-remote command: git 2.7.4 does not
272 // support such a separator for that command, and this use should be safe
273 // without it because the {scheme} value comes from the predefined list above.
274 // See golang.org/issue/33836.
275 PingCmd: "ls-remote {scheme}://{repo}",
277 RemoteRepo: gitRemoteRepo,
281 // scpSyntaxRe matches the SCP-like addresses used by Git to access
282 // repositories by SSH.
283 var scpSyntaxRe = lazyregexp.New(`^(\w+)@([\w.-]+):(.*)$`)
285 func gitRemoteRepo(vcsGit *Cmd, rootDir string) (remoteRepo string, err error) {
286 const cmd = "config remote.origin.url"
287 outb, err := vcsGit.run1(rootDir, cmd, nil, false)
289 // if it doesn't output any message, it means the config argument is correct,
290 // but the config value itself doesn't exist
291 if outb != nil && len(outb) == 0 {
292 return "", errors.New("remote origin not found")
296 out := strings.TrimSpace(string(outb))
298 var repoURL *urlpkg.URL
299 if m := scpSyntaxRe.FindStringSubmatch(out); m != nil {
300 // Match SCP-like syntax and convert it to a URL.
301 // Eg, "git@github.com:user/repo" becomes
302 // "ssh://git@github.com/user/repo".
303 repoURL = &urlpkg.URL{
305 User: urlpkg.User(m[1]),
310 repoURL, err = urlpkg.Parse(out)
316 // Iterate over insecure schemes too, because this function simply
317 // reports the state of the repo. If we can't see insecure schemes then
318 // we can't report the actual repo URL.
319 for _, s := range vcsGit.Scheme {
320 if repoURL.Scheme == s {
321 return repoURL.String(), nil
324 return "", errors.New("unable to parse output of git " + cmd)
327 func gitStatus(vcsGit *Cmd, rootDir string) (Status, error) {
328 out, err := vcsGit.runOutputVerboseOnly(rootDir, "status --porcelain")
332 uncommitted := len(out) > 0
334 // "git status" works for empty repositories, but "git show" does not.
335 // Assume there are no commits in the repo when "git show" fails with
336 // uncommitted files and skip tagging revision / committime.
338 var commitTime time.Time
339 out, err = vcsGit.runOutputVerboseOnly(rootDir, "-c log.showsignature=false show -s --format=%H:%ct")
340 if err != nil && !uncommitted {
342 } else if err == nil {
343 rev, commitTime, err = parseRevTime(out)
351 CommitTime: commitTime,
352 Uncommitted: uncommitted,
356 // vcsBzr describes how to use Bazaar.
360 RootNames: []rootName{
361 {filename: ".bzr", isDir: true},
364 CreateCmd: []string{"branch -- {repo} {dir}"},
366 // Without --overwrite bzr will not pull tags that changed.
367 // Replace by --overwrite-tags after http://pad.lv/681792 goes in.
368 DownloadCmd: []string{"pull --overwrite"},
370 TagCmd: []tagCmd{{"tags", `^(\S+)`}},
371 TagSyncCmd: []string{"update -r {tag}"},
372 TagSyncDefault: []string{"update -r revno:-1"},
374 Scheme: []string{"https", "http", "bzr", "bzr+ssh"},
375 PingCmd: "info -- {scheme}://{repo}",
376 RemoteRepo: bzrRemoteRepo,
377 ResolveRepo: bzrResolveRepo,
381 func bzrRemoteRepo(vcsBzr *Cmd, rootDir string) (remoteRepo string, err error) {
382 outb, err := vcsBzr.runOutput(rootDir, "config parent_location")
386 return strings.TrimSpace(string(outb)), nil
389 func bzrResolveRepo(vcsBzr *Cmd, rootDir, remoteRepo string) (realRepo string, err error) {
390 outb, err := vcsBzr.runOutput(rootDir, "info "+remoteRepo)
398 // (branch root|repository branch): <URL>
402 for _, prefix := range []string{"\n branch root: ", "\n repository branch: "} {
403 i := strings.Index(out, prefix)
405 out = out[i+len(prefix):]
411 return "", fmt.Errorf("unable to parse output of bzr info")
414 i := strings.Index(out, "\n")
416 return "", fmt.Errorf("unable to parse output of bzr info")
419 return strings.TrimSpace(out), nil
422 func bzrStatus(vcsBzr *Cmd, rootDir string) (Status, error) {
423 outb, err := vcsBzr.runOutputVerboseOnly(rootDir, "version-info")
429 // Expect (non-empty repositories only):
431 // revision-id: gopher@gopher.net-20211021072330-qshok76wfypw9lpm
432 // date: 2021-09-21 12:00:00 +1000
435 var commitTime time.Time
437 for _, line := range strings.Split(out, "\n") {
438 i := strings.IndexByte(line, ':')
443 value := strings.TrimSpace(line[i+1:])
450 commitTime, err = time.Parse("2006-01-02 15:04:05 -0700", value)
452 return Status{}, errors.New("unable to parse output of bzr version-info")
457 outb, err = vcsBzr.runOutputVerboseOnly(rootDir, "status")
462 // Skip warning when working directory is set to an older revision.
463 if bytes.HasPrefix(outb, []byte("working tree is out of date")) {
464 i := bytes.IndexByte(outb, '\n')
470 uncommitted := len(outb) > 0
474 CommitTime: commitTime,
475 Uncommitted: uncommitted,
479 // vcsSvn describes how to use Subversion.
483 RootNames: []rootName{
484 {filename: ".svn", isDir: true},
487 CreateCmd: []string{"checkout -- {repo} {dir}"},
488 DownloadCmd: []string{"update"},
490 // There is no tag command in subversion.
491 // The branch information is all in the path names.
493 Scheme: []string{"https", "http", "svn", "svn+ssh"},
494 PingCmd: "info -- {scheme}://{repo}",
495 RemoteRepo: svnRemoteRepo,
498 func svnRemoteRepo(vcsSvn *Cmd, rootDir string) (remoteRepo string, err error) {
499 outb, err := vcsSvn.runOutput(rootDir, "info")
511 // Note that we're not using the Repository Root line,
512 // because svn allows checking out subtrees.
513 // The URL will be the URL of the subtree (what we used with 'svn co')
514 // while the Repository Root may be a much higher parent.
515 i := strings.Index(out, "\nURL: ")
517 return "", fmt.Errorf("unable to parse output of svn info")
519 out = out[i+len("\nURL: "):]
520 i = strings.Index(out, "\n")
522 return "", fmt.Errorf("unable to parse output of svn info")
525 return strings.TrimSpace(out), nil
528 // fossilRepoName is the name go get associates with a fossil repository. In the
529 // real world the file can be named anything.
530 const fossilRepoName = ".fossil"
532 // vcsFossil describes how to use Fossil (fossil-scm.org)
533 var vcsFossil = &Cmd{
536 RootNames: []rootName{
537 {filename: ".fslckout", isDir: false},
538 {filename: "_FOSSIL_", isDir: false},
541 CreateCmd: []string{"-go-internal-mkdir {dir} clone -- {repo} " + filepath.Join("{dir}", fossilRepoName), "-go-internal-cd {dir} open .fossil"},
542 DownloadCmd: []string{"up"},
544 TagCmd: []tagCmd{{"tag ls", `(.*)`}},
545 TagSyncCmd: []string{"up tag:{tag}"},
546 TagSyncDefault: []string{"up trunk"},
548 Scheme: []string{"https", "http"},
549 RemoteRepo: fossilRemoteRepo,
550 Status: fossilStatus,
553 func fossilRemoteRepo(vcsFossil *Cmd, rootDir string) (remoteRepo string, err error) {
554 out, err := vcsFossil.runOutput(rootDir, "remote-url")
558 return strings.TrimSpace(string(out)), nil
561 var errFossilInfo = errors.New("unable to parse output of fossil info")
563 func fossilStatus(vcsFossil *Cmd, rootDir string) (Status, error) {
564 outb, err := vcsFossil.runOutputVerboseOnly(rootDir, "info")
572 // checkout: 91ed71f22c77be0c3e250920f47bfd4e1f9024d2 2021-09-21 12:00:00 UTC
575 // Extract revision and commit time.
576 // Ensure line ends with UTC (known timezone offset).
577 const prefix = "\ncheckout:"
578 const suffix = " UTC"
579 i := strings.Index(out, prefix)
581 return Status{}, errFossilInfo
583 checkout := out[i+len(prefix):]
584 i = strings.Index(checkout, suffix)
586 return Status{}, errFossilInfo
588 checkout = strings.TrimSpace(checkout[:i])
590 i = strings.IndexByte(checkout, ' ')
592 return Status{}, errFossilInfo
596 commitTime, err := time.ParseInLocation(time.DateTime, checkout[i+1:], time.UTC)
598 return Status{}, fmt.Errorf("%v: %v", errFossilInfo, err)
601 // Also look for untracked changes.
602 outb, err = vcsFossil.runOutputVerboseOnly(rootDir, "changes --differ")
606 uncommitted := len(outb) > 0
610 CommitTime: commitTime,
611 Uncommitted: uncommitted,
615 func (v *Cmd) String() string {
619 // run runs the command line cmd in the given directory.
620 // keyval is a list of key, value pairs. run expands
621 // instances of {key} in cmd into value, but only after
622 // splitting cmd into individual arguments.
623 // If an error occurs, run prints the command line and the
624 // command's combined stdout+stderr to standard error.
625 // Otherwise run discards the command's output.
626 func (v *Cmd) run(dir string, cmd string, keyval ...string) error {
627 _, err := v.run1(dir, cmd, keyval, true)
631 // runVerboseOnly is like run but only generates error output to standard error in verbose mode.
632 func (v *Cmd) runVerboseOnly(dir string, cmd string, keyval ...string) error {
633 _, err := v.run1(dir, cmd, keyval, false)
637 // runOutput is like run but returns the output of the command.
638 func (v *Cmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) {
639 return v.run1(dir, cmd, keyval, true)
642 // runOutputVerboseOnly is like runOutput but only generates error output to
643 // standard error in verbose mode.
644 func (v *Cmd) runOutputVerboseOnly(dir string, cmd string, keyval ...string) ([]byte, error) {
645 return v.run1(dir, cmd, keyval, false)
648 // run1 is the generalized implementation of run and runOutput.
649 func (v *Cmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) {
650 m := make(map[string]string)
651 for i := 0; i < len(keyval); i += 2 {
652 m[keyval[i]] = keyval[i+1]
654 args := strings.Fields(cmdline)
655 for i, arg := range args {
656 args[i] = expand(m, arg)
659 if len(args) >= 2 && args[0] == "-go-internal-mkdir" {
661 if filepath.IsAbs(args[1]) {
662 err = os.Mkdir(args[1], fs.ModePerm)
664 err = os.Mkdir(filepath.Join(dir, args[1]), fs.ModePerm)
672 if len(args) >= 2 && args[0] == "-go-internal-cd" {
673 if filepath.IsAbs(args[1]) {
676 dir = filepath.Join(dir, args[1])
681 _, err := cfg.LookPath(v.Cmd)
683 fmt.Fprintf(os.Stderr,
684 "go: missing %s command. See https://golang.org/s/gogetcmd\n",
689 cmd := exec.Command(v.Cmd, args...)
692 fmt.Fprintf(os.Stderr, "cd %s\n", dir)
693 fmt.Fprintf(os.Stderr, "%s %s\n", v.Cmd, strings.Join(args, " "))
695 out, err := cmd.Output()
697 if verbose || cfg.BuildV {
698 fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.Cmd, strings.Join(args, " "))
699 if ee, ok := err.(*exec.ExitError); ok && len(ee.Stderr) > 0 {
700 os.Stderr.Write(ee.Stderr)
702 fmt.Fprintln(os.Stderr, err.Error())
709 // Ping pings to determine scheme to use.
710 func (v *Cmd) Ping(scheme, repo string) error {
711 // Run the ping command in an arbitrary working directory,
712 // but don't let the current working directory pollute the results.
713 // In module mode, we expect GOMODCACHE to exist and be a safe place for
714 // commands; in GOPATH mode, we expect that to be true of GOPATH/src.
715 dir := cfg.GOMODCACHE
716 if !cfg.ModulesEnabled {
717 dir = filepath.Join(cfg.BuildContext.GOPATH, "src")
719 os.MkdirAll(dir, 0777) // Ignore errors — if unsuccessful, the command will likely fail.
721 release, err := base.AcquireNet()
727 return v.runVerboseOnly(dir, v.PingCmd, "scheme", scheme, "repo", repo)
730 // Create creates a new copy of repo in dir.
731 // The parent of dir must exist; dir must not.
732 func (v *Cmd) Create(dir, repo string) error {
733 release, err := base.AcquireNet()
739 for _, cmd := range v.CreateCmd {
740 if err := v.run(filepath.Dir(dir), cmd, "dir", dir, "repo", repo); err != nil {
747 // Download downloads any new changes for the repo in dir.
748 func (v *Cmd) Download(dir string) error {
749 release, err := base.AcquireNet()
755 for _, cmd := range v.DownloadCmd {
756 if err := v.run(dir, cmd); err != nil {
763 // Tags returns the list of available tags for the repo in dir.
764 func (v *Cmd) Tags(dir string) ([]string, error) {
766 for _, tc := range v.TagCmd {
767 out, err := v.runOutput(dir, tc.cmd)
771 re := regexp.MustCompile(`(?m-s)` + tc.pattern)
772 for _, m := range re.FindAllStringSubmatch(string(out), -1) {
773 tags = append(tags, m[1])
779 // TagSync syncs the repo in dir to the named tag,
780 // which either is a tag returned by tags or is v.tagDefault.
781 func (v *Cmd) TagSync(dir, tag string) error {
782 if v.TagSyncCmd == nil {
786 for _, tc := range v.TagLookupCmd {
787 out, err := v.runOutput(dir, tc.cmd, "tag", tag)
791 re := regexp.MustCompile(`(?m-s)` + tc.pattern)
792 m := re.FindStringSubmatch(string(out))
800 release, err := base.AcquireNet()
806 if tag == "" && v.TagSyncDefault != nil {
807 for _, cmd := range v.TagSyncDefault {
808 if err := v.run(dir, cmd); err != nil {
815 for _, cmd := range v.TagSyncCmd {
816 if err := v.run(dir, cmd, "tag", tag); err != nil {
823 // A vcsPath describes how to convert an import path into a
824 // version control system and repository name.
825 type vcsPath struct {
826 pathPrefix string // prefix this description applies to
827 regexp *lazyregexp.Regexp // compiled pattern for import path
828 repo string // repository to use (expand with match of re)
829 vcs string // version control system to use (expand with match of re)
830 check func(match map[string]string) error // additional checks
831 schemelessRepo bool // if true, the repo pattern lacks a scheme
834 // FromDir inspects dir and its parents to determine the
835 // version control system and code repository to use.
836 // If no repository is found, FromDir returns an error
837 // equivalent to os.ErrNotExist.
838 func FromDir(dir, srcRoot string, allowNesting bool) (repoDir string, vcsCmd *Cmd, err error) {
839 // Clean and double-check that dir is in (a subdirectory of) srcRoot.
840 dir = filepath.Clean(dir)
842 srcRoot = filepath.Clean(srcRoot)
843 if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator {
844 return "", nil, fmt.Errorf("directory %q is outside source root %q", dir, srcRoot)
849 for len(dir) > len(srcRoot) {
850 for _, vcs := range vcsList {
851 if isVCSRoot(dir, vcs.RootNames) {
852 // Record first VCS we find.
853 // If allowNesting is false (as it is in GOPATH), keep looking for
854 // repositories in parent directories and report an error if one is
855 // found to mitigate VCS injection attacks.
860 return repoDir, vcsCmd, nil
864 // Otherwise, we have one VCS inside a different VCS.
865 return "", nil, fmt.Errorf("directory %q uses %s, but parent %q uses %s",
866 repoDir, vcsCmd.Cmd, dir, vcs.Cmd)
871 ndir := filepath.Dir(dir)
872 if len(ndir) >= len(dir) {
878 return "", nil, &vcsNotFoundError{dir: origDir}
880 return repoDir, vcsCmd, nil
883 // isVCSRoot identifies a VCS root by checking whether the directory contains
884 // any of the listed root names.
885 func isVCSRoot(dir string, rootNames []rootName) bool {
886 for _, root := range rootNames {
887 fi, err := os.Stat(filepath.Join(dir, root.filename))
888 if err == nil && fi.IsDir() == root.isDir {
896 type rootName struct {
901 type vcsNotFoundError struct {
905 func (e *vcsNotFoundError) Error() string {
906 return fmt.Sprintf("directory %q is not using a known version control system", e.dir)
909 func (e *vcsNotFoundError) Is(err error) bool {
910 return err == os.ErrNotExist
913 // A govcsRule is a single GOVCS rule like private:hg|svn.
914 type govcsRule struct {
919 // A govcsConfig is a full GOVCS configuration.
920 type govcsConfig []govcsRule
922 func parseGOVCS(s string) (govcsConfig, error) {
923 s = strings.TrimSpace(s)
928 have := make(map[string]string)
929 for _, item := range strings.Split(s, ",") {
930 item = strings.TrimSpace(item)
932 return nil, fmt.Errorf("empty entry in GOVCS")
934 pattern, list, found := strings.Cut(item, ":")
936 return nil, fmt.Errorf("malformed entry in GOVCS (missing colon): %q", item)
938 pattern, list = strings.TrimSpace(pattern), strings.TrimSpace(list)
940 return nil, fmt.Errorf("empty pattern in GOVCS: %q", item)
943 return nil, fmt.Errorf("empty VCS list in GOVCS: %q", item)
945 if search.IsRelativePath(pattern) {
946 return nil, fmt.Errorf("relative pattern not allowed in GOVCS: %q", pattern)
948 if old := have[pattern]; old != "" {
949 return nil, fmt.Errorf("unreachable pattern in GOVCS: %q after %q", item, old)
952 allowed := strings.Split(list, "|")
953 for i, a := range allowed {
954 a = strings.TrimSpace(a)
956 return nil, fmt.Errorf("empty VCS name in GOVCS: %q", item)
960 cfg = append(cfg, govcsRule{pattern, allowed})
965 func (c *govcsConfig) allow(path string, private bool, vcs string) bool {
966 for _, rule := range *c {
968 switch rule.pattern {
974 // Note: rule.pattern is known to be comma-free,
975 // so MatchPrefixPatterns is only matching a single pattern for us.
976 match = module.MatchPrefixPatterns(rule.pattern, path)
981 for _, allow := range rule.allowed {
982 if allow == vcs || allow == "all" {
989 // By default, nothing is allowed.
999 // defaultGOVCS is the default setting for GOVCS.
1000 // Setting GOVCS adds entries ahead of these but does not remove them.
1001 // (They are appended to the parsed GOVCS setting.)
1003 // The rationale behind allowing only Git and Mercurial is that
1004 // these two systems have had the most attention to issues
1005 // of being run as clients of untrusted servers. In contrast,
1006 // Bazaar, Fossil, and Subversion have primarily been used
1007 // in trusted, authenticated environments and are not as well
1008 // scrutinized as attack surfaces.
1010 // See golang.org/issue/41730 for details.
1011 var defaultGOVCS = govcsConfig{
1012 {"private", []string{"all"}},
1013 {"public", []string{"git", "hg"}},
1016 // checkGOVCS checks whether the policy defined by the environment variable
1017 // GOVCS allows the given vcs command to be used with the given repository
1018 // root path. Note that root may not be a real package or module path; it's
1019 // the same as the root path in the go-import meta tag.
1020 func checkGOVCS(vcs *Cmd, root string) error {
1022 // Direct module (proxy protocol) fetches don't
1023 // involve an external version control system
1024 // and are always allowed.
1028 govcsOnce.Do(func() {
1029 govcs, govcsErr = parseGOVCS(os.Getenv("GOVCS"))
1030 govcs = append(govcs, defaultGOVCS...)
1032 if govcsErr != nil {
1036 private := module.MatchPrefixPatterns(cfg.GOPRIVATE, root)
1037 if !govcs.allow(root, private, vcs.Cmd) {
1042 return fmt.Errorf("GOVCS disallows using %s for %s %s; see 'go help vcs'", vcs.Cmd, what, root)
1048 // RepoRoot describes the repository root for a tree of source code.
1049 type RepoRoot struct {
1050 Repo string // repository URL, including scheme
1051 Root string // import path corresponding to root of repo
1052 IsCustom bool // defined by served <meta> tags (as opposed to hard-coded pattern)
1056 func httpPrefix(s string) string {
1057 for _, prefix := range [...]string{"http:", "https:"} {
1058 if strings.HasPrefix(s, prefix) {
1065 // ModuleMode specifies whether to prefer modules when looking up code sources.
1069 IgnoreMod ModuleMode = iota
1073 // RepoRootForImportPath analyzes importPath to determine the
1074 // version control system, and code repository to use.
1075 func RepoRootForImportPath(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) {
1076 rr, err := repoRootFromVCSPaths(importPath, security, vcsPaths)
1077 if err == errUnknownSite {
1078 rr, err = repoRootForImportDynamic(importPath, mod, security)
1080 err = importErrorf(importPath, "unrecognized import path %q: %v", importPath, err)
1084 rr1, err1 := repoRootFromVCSPaths(importPath, security, vcsPathsAfterDynamic)
1091 // Should have been taken care of above, but make sure.
1092 if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.Root, "...") {
1093 // Do not allow wildcards in the repo root.
1095 err = importErrorf(importPath, "cannot expand ... in %q", importPath)
1100 var errUnknownSite = errors.New("dynamic lookup required to find mapping")
1102 // repoRootFromVCSPaths attempts to map importPath to a repoRoot
1103 // using the mappings defined in vcsPaths.
1104 func repoRootFromVCSPaths(importPath string, security web.SecurityMode, vcsPaths []*vcsPath) (*RepoRoot, error) {
1105 if str.HasPathPrefix(importPath, "example.net") {
1106 // TODO(rsc): This should not be necessary, but it's required to keep
1107 // tests like ../../testdata/script/mod_get_extra.txt from using the network.
1108 // That script has everything it needs in the replacement set, but it is still
1109 // doing network calls.
1110 return nil, fmt.Errorf("no modules on example.net")
1112 if importPath == "rsc.io" {
1113 // This special case allows tests like ../../testdata/script/govcs.txt
1114 // to avoid making any network calls. The module lookup for a path
1115 // like rsc.io/nonexist.svn/foo needs to not make a network call for
1116 // a lookup on rsc.io.
1117 return nil, fmt.Errorf("rsc.io is not a module")
1119 // A common error is to use https://packagepath because that's what
1120 // hg and git require. Diagnose this helpfully.
1121 if prefix := httpPrefix(importPath); prefix != "" {
1122 // The importPath has been cleaned, so has only one slash. The pattern
1123 // ignores the slashes; the error message puts them back on the RHS at least.
1124 return nil, fmt.Errorf("%q not allowed in import path", prefix+"//")
1126 for _, srv := range vcsPaths {
1127 if !str.HasPathPrefix(importPath, srv.pathPrefix) {
1130 m := srv.regexp.FindStringSubmatch(importPath)
1132 if srv.pathPrefix != "" {
1133 return nil, importErrorf(importPath, "invalid %s import path %q", srv.pathPrefix, importPath)
1138 // Build map of named subexpression matches for expand.
1139 match := map[string]string{
1140 "prefix": srv.pathPrefix + "/",
1141 "import": importPath,
1143 for i, name := range srv.regexp.SubexpNames() {
1144 if name != "" && match[name] == "" {
1149 match["vcs"] = expand(match, srv.vcs)
1152 match["repo"] = expand(match, srv.repo)
1154 if srv.check != nil {
1155 if err := srv.check(match); err != nil {
1159 vcs := vcsByCmd(match["vcs"])
1161 return nil, fmt.Errorf("unknown version control system %q", match["vcs"])
1163 if err := checkGOVCS(vcs, match["root"]); err != nil {
1167 if !srv.schemelessRepo {
1168 repoURL = match["repo"]
1170 repo := match["repo"]
1172 repoURL, ok = interceptVCSTest(repo, vcs, security)
1174 scheme := vcs.Scheme[0] // default to first scheme
1175 if vcs.PingCmd != "" {
1176 // If we know how to test schemes, scan to find one.
1177 for _, s := range vcs.Scheme {
1178 if security == web.SecureOnly && !vcs.isSecureScheme(s) {
1181 if vcs.Ping(s, repo) == nil {
1187 repoURL = scheme + "://" + repo
1192 Root: match["root"],
1197 return nil, errUnknownSite
1200 func interceptVCSTest(repo string, vcs *Cmd, security web.SecurityMode) (repoURL string, ok bool) {
1201 if VCSTestRepoURL == "" {
1205 // Since the "mod" protocol is implemented internally,
1206 // requests will be intercepted at a lower level (in cmd/go/internal/web).
1210 if scheme, path, ok := strings.Cut(repo, "://"); ok {
1211 if security == web.SecureOnly && !vcs.isSecureScheme(scheme) {
1212 return "", false // Let the caller reject the original URL.
1214 repo = path // Remove leading URL scheme if present.
1216 for _, host := range VCSTestHosts {
1217 if !str.HasPathPrefix(repo, host) {
1221 httpURL := VCSTestRepoURL + strings.TrimPrefix(repo, host)
1224 // Ping the vcweb HTTP server to tell it to initialize the SVN repository
1225 // and get the SVN server URL.
1226 u, err := urlpkg.Parse(httpURL + "?vcwebsvn=1")
1228 panic(fmt.Sprintf("invalid vcs-test repo URL: %v", err))
1230 svnURL, err := web.GetBytes(u)
1231 svnURL = bytes.TrimSpace(svnURL)
1232 if err == nil && len(svnURL) > 0 {
1233 return string(svnURL) + strings.TrimPrefix(repo, host), true
1236 // vcs-test doesn't have a svn handler for the given path,
1237 // so resolve the repo to HTTPS instead.
1240 return httpURL, true
1245 // urlForImportPath returns a partially-populated URL for the given Go import path.
1247 // The URL leaves the Scheme field blank so that web.Get will try any scheme
1248 // allowed by the selected security mode.
1249 func urlForImportPath(importPath string) (*urlpkg.URL, error) {
1250 slash := strings.Index(importPath, "/")
1252 slash = len(importPath)
1254 host, path := importPath[:slash], importPath[slash:]
1255 if !strings.Contains(host, ".") {
1256 return nil, errors.New("import path does not begin with hostname")
1261 return &urlpkg.URL{Host: host, Path: path, RawQuery: "go-get=1"}, nil
1264 // repoRootForImportDynamic finds a *RepoRoot for a custom domain that's not
1265 // statically known by repoRootFromVCSPaths.
1267 // This handles custom import paths like "name.tld/pkg/foo" or just "name.tld".
1268 func repoRootForImportDynamic(importPath string, mod ModuleMode, security web.SecurityMode) (*RepoRoot, error) {
1269 url, err := urlForImportPath(importPath)
1273 resp, err := web.Get(security, url)
1275 msg := "https fetch: %v"
1276 if security == web.Insecure {
1279 return nil, fmt.Errorf(msg, err)
1283 imports, err := parseMetaGoImports(body, mod)
1284 if len(imports) == 0 {
1285 if respErr := resp.Err(); respErr != nil {
1286 // If the server's status was not OK, prefer to report that instead of
1287 // an XML parse error.
1292 return nil, fmt.Errorf("parsing %s: %v", importPath, err)
1294 // Find the matched meta import.
1295 mmi, err := matchGoImport(imports, importPath)
1297 if _, ok := err.(ImportMismatchError); !ok {
1298 return nil, fmt.Errorf("parse %s: %v", url, err)
1300 return nil, fmt.Errorf("parse %s: no go-import meta tags (%s)", resp.URL, err)
1303 log.Printf("get %q: found meta tag %#v at %s", importPath, mmi, url)
1305 // If the import was "uni.edu/bob/project", which said the
1306 // prefix was "uni.edu" and the RepoRoot was "evilroot.com",
1307 // make sure we don't trust Bob and check out evilroot.com to
1308 // "uni.edu" yet (possibly overwriting/preempting another
1309 // non-evil student). Instead, first verify the root and see
1310 // if it matches Bob's claim.
1311 if mmi.Prefix != importPath {
1313 log.Printf("get %q: verifying non-authoritative meta tag", importPath)
1315 var imports []metaImport
1316 url, imports, err = metaImportsForPrefix(mmi.Prefix, mod, security)
1320 metaImport2, err := matchGoImport(imports, importPath)
1321 if err != nil || mmi != metaImport2 {
1322 return nil, fmt.Errorf("%s and %s disagree about go-import for %s", resp.URL, url, mmi.Prefix)
1326 if err := validateRepoRoot(mmi.RepoRoot); err != nil {
1327 return nil, fmt.Errorf("%s: invalid repo root %q: %v", resp.URL, mmi.RepoRoot, err)
1330 if mmi.VCS == "mod" {
1333 vcs = vcsByCmd(mmi.VCS)
1335 return nil, fmt.Errorf("%s: unknown vcs %q", resp.URL, mmi.VCS)
1339 if err := checkGOVCS(vcs, mmi.Prefix); err != nil {
1343 repoURL, ok := interceptVCSTest(mmi.RepoRoot, vcs, security)
1345 repoURL = mmi.RepoRoot
1356 // validateRepoRoot returns an error if repoRoot does not seem to be
1357 // a valid URL with scheme.
1358 func validateRepoRoot(repoRoot string) error {
1359 url, err := urlpkg.Parse(repoRoot)
1363 if url.Scheme == "" {
1364 return errors.New("no scheme")
1366 if url.Scheme == "file" {
1367 return errors.New("file scheme disallowed")
1372 var fetchGroup singleflight.Group
1374 fetchCacheMu sync.Mutex
1375 fetchCache = map[string]fetchResult{} // key is metaImportsForPrefix's importPrefix
1378 // metaImportsForPrefix takes a package's root import path as declared in a <meta> tag
1379 // and returns its HTML discovery URL and the parsed metaImport lines
1380 // found on the page.
1382 // The importPath is of the form "golang.org/x/tools".
1383 // It is an error if no imports are found.
1384 // url will still be valid if err != nil.
1385 // The returned url will be of the form "https://golang.org/x/tools?go-get=1"
1386 func metaImportsForPrefix(importPrefix string, mod ModuleMode, security web.SecurityMode) (*urlpkg.URL, []metaImport, error) {
1387 setCache := func(res fetchResult) (fetchResult, error) {
1389 defer fetchCacheMu.Unlock()
1390 fetchCache[importPrefix] = res
1394 resi, _, _ := fetchGroup.Do(importPrefix, func() (resi any, err error) {
1396 if res, ok := fetchCache[importPrefix]; ok {
1397 fetchCacheMu.Unlock()
1400 fetchCacheMu.Unlock()
1402 url, err := urlForImportPath(importPrefix)
1404 return setCache(fetchResult{err: err})
1406 resp, err := web.Get(security, url)
1408 return setCache(fetchResult{url: url, err: fmt.Errorf("fetching %s: %v", importPrefix, err)})
1412 imports, err := parseMetaGoImports(body, mod)
1413 if len(imports) == 0 {
1414 if respErr := resp.Err(); respErr != nil {
1415 // If the server's status was not OK, prefer to report that instead of
1416 // an XML parse error.
1417 return setCache(fetchResult{url: url, err: respErr})
1421 return setCache(fetchResult{url: url, err: fmt.Errorf("parsing %s: %v", resp.URL, err)})
1423 if len(imports) == 0 {
1424 err = fmt.Errorf("fetching %s: no go-import meta tag found in %s", importPrefix, resp.URL)
1426 return setCache(fetchResult{url: url, imports: imports, err: err})
1428 res := resi.(fetchResult)
1429 return res.url, res.imports, res.err
1432 type fetchResult struct {
1434 imports []metaImport
1438 // metaImport represents the parsed <meta name="go-import"
1439 // content="prefix vcs reporoot" /> tags from HTML files.
1440 type metaImport struct {
1441 Prefix, VCS, RepoRoot string
1444 // An ImportMismatchError is returned where metaImport/s are present
1445 // but none match our import path.
1446 type ImportMismatchError struct {
1448 mismatches []string // the meta imports that were discarded for not matching our importPath
1451 func (m ImportMismatchError) Error() string {
1452 formattedStrings := make([]string, len(m.mismatches))
1453 for i, pre := range m.mismatches {
1454 formattedStrings[i] = fmt.Sprintf("meta tag %s did not match import path %s", pre, m.importPath)
1456 return strings.Join(formattedStrings, ", ")
1459 // matchGoImport returns the metaImport from imports matching importPath.
1460 // An error is returned if there are multiple matches.
1461 // An ImportMismatchError is returned if none match.
1462 func matchGoImport(imports []metaImport, importPath string) (metaImport, error) {
1465 errImportMismatch := ImportMismatchError{importPath: importPath}
1466 for i, im := range imports {
1467 if !str.HasPathPrefix(importPath, im.Prefix) {
1468 errImportMismatch.mismatches = append(errImportMismatch.mismatches, im.Prefix)
1473 if imports[match].VCS == "mod" && im.VCS != "mod" {
1474 // All the mod entries precede all the non-mod entries.
1475 // We have a mod entry and don't care about the rest,
1479 return metaImport{}, fmt.Errorf("multiple meta tags match import path %q", importPath)
1485 return metaImport{}, errImportMismatch
1487 return imports[match], nil
1490 // expand rewrites s to replace {k} with match[k] for each key k in match.
1491 func expand(match map[string]string, s string) string {
1492 // We want to replace each match exactly once, and the result of expansion
1493 // must not depend on the iteration order through the map.
1494 // A strings.Replacer has exactly the properties we're looking for.
1495 oldNew := make([]string, 0, 2*len(match))
1496 for k, v := range match {
1497 oldNew = append(oldNew, "{"+k+"}", v)
1499 return strings.NewReplacer(oldNew...).Replace(s)
1502 // vcsPaths defines the meaning of import paths referring to
1503 // commonly-used VCS hosting sites (github.com/user/dir)
1504 // and import paths referring to a fully-qualified importPath
1505 // containing a VCS type (foo.com/repo.git/dir)
1506 var vcsPaths = []*vcsPath{
1509 pathPrefix: "github.com",
1510 regexp: lazyregexp.New(`^(?P<root>github\.com/[\w.\-]+/[\w.\-]+)(/[\w.\-]+)*$`),
1512 repo: "https://{root}",
1518 pathPrefix: "bitbucket.org",
1519 regexp: lazyregexp.New(`^(?P<root>bitbucket\.org/(?P<bitname>[\w.\-]+/[\w.\-]+))(/[\w.\-]+)*$`),
1521 repo: "https://{root}",
1525 // IBM DevOps Services (JazzHub)
1527 pathPrefix: "hub.jazz.net/git",
1528 regexp: lazyregexp.New(`^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[\w.\-]+)(/[\w.\-]+)*$`),
1530 repo: "https://{root}",
1536 pathPrefix: "git.apache.org",
1537 regexp: lazyregexp.New(`^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/[\w.\-]+)*$`),
1539 repo: "https://{root}",
1544 pathPrefix: "git.openstack.org",
1545 regexp: lazyregexp.New(`^(?P<root>git\.openstack\.org/[\w.\-]+/[\w.\-]+)(\.git)?(/[\w.\-]+)*$`),
1547 repo: "https://{root}",
1550 // chiselapp.com for fossil
1552 pathPrefix: "chiselapp.com",
1553 regexp: lazyregexp.New(`^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[\w.\-]+)$`),
1555 repo: "https://{root}",
1558 // General syntax for any server.
1561 regexp: lazyregexp.New(`(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?(/~?[\w.\-]+)+?)\.(?P<vcs>bzr|fossil|git|hg|svn))(/~?[\w.\-]+)*$`),
1562 schemelessRepo: true,
1566 // vcsPathsAfterDynamic gives additional vcsPaths entries
1567 // to try after the dynamic HTML check.
1568 // This gives those sites a chance to introduce <meta> tags
1569 // as part of a graceful transition away from the hard-coded logic.
1570 var vcsPathsAfterDynamic = []*vcsPath{
1571 // Launchpad. See golang.org/issue/11436.
1573 pathPrefix: "launchpad.net",
1574 regexp: lazyregexp.New(`^(?P<root>launchpad\.net/((?P<project>[\w.\-]+)(?P<series>/[\w.\-]+)?|~[\w.\-]+/(\+junk|[\w.\-]+)/[\w.\-]+))(/[\w.\-]+)*$`),
1576 repo: "https://{root}",
1577 check: launchpadVCS,
1581 // noVCSSuffix checks that the repository name does not
1582 // end in .foo for any version control system foo.
1583 // The usual culprit is ".git".
1584 func noVCSSuffix(match map[string]string) error {
1585 repo := match["repo"]
1586 for _, vcs := range vcsList {
1587 if strings.HasSuffix(repo, "."+vcs.Cmd) {
1588 return fmt.Errorf("invalid version control suffix in %s path", match["prefix"])
1594 // launchpadVCS solves the ambiguity for "lp.net/project/foo". In this case,
1595 // "foo" could be a series name registered in Launchpad with its own branch,
1596 // and it could also be the name of a directory within the main project
1597 // branch one level up.
1598 func launchpadVCS(match map[string]string) error {
1599 if match["project"] == "" || match["series"] == "" {
1604 Host: "code.launchpad.net",
1605 Path: expand(match, "/{project}{series}/.bzr/branch-format"),
1607 _, err := web.GetBytes(url)
1609 match["root"] = expand(match, "launchpad.net/{project}")
1610 match["repo"] = expand(match, "https://{root}")
1615 // importError is a copy of load.importError, made to avoid a dependency cycle
1616 // on cmd/go/internal/load. It just needs to satisfy load.ImportPathError.
1617 type importError struct {
1622 func importErrorf(path, format string, args ...any) error {
1623 err := &importError{importPath: path, err: fmt.Errorf(format, args...)}
1624 if errStr := err.Error(); !strings.Contains(errStr, path) {
1625 panic(fmt.Sprintf("path %q not in error %q", path, errStr))
1630 func (e *importError) Error() string {
1631 return e.err.Error()
1634 func (e *importError) Unwrap() error {
1635 // Don't return e.err directly, since we're only wrapping an error if %w
1636 // was passed to ImportErrorf.
1637 return errors.Unwrap(e.err)
1640 func (e *importError) ImportPath() string {