1
2
3
4
5
6
7
8
9
10
11
12
13
14 package vcs
15
16 import (
17 "bytes"
18 "encoding/json"
19 "errors"
20 "fmt"
21 exec "golang.org/x/sys/execabs"
22 "log"
23 "net/url"
24 "os"
25 "path/filepath"
26 "regexp"
27 "strconv"
28 "strings"
29 )
30
31
32 var Verbose bool
33
34
35 var ShowCmd bool
36
37
38
39 type Cmd struct {
40 Name string
41 Cmd string
42
43 CreateCmd string
44 DownloadCmd string
45
46 TagCmd []TagCmd
47 TagLookupCmd []TagCmd
48 TagSyncCmd string
49 TagSyncDefault string
50
51 LogCmd string
52
53 Scheme []string
54 PingCmd string
55 }
56
57
58
59 type TagCmd struct {
60 Cmd string
61 Pattern string
62 }
63
64
65 var vcsList = []*Cmd{
66 vcsHg,
67 vcsGit,
68 vcsSvn,
69 vcsBzr,
70 }
71
72
73
74 func ByCmd(cmd string) *Cmd {
75 for _, vcs := range vcsList {
76 if vcs.Cmd == cmd {
77 return vcs
78 }
79 }
80 return nil
81 }
82
83
84 var vcsHg = &Cmd{
85 Name: "Mercurial",
86 Cmd: "hg",
87
88 CreateCmd: "clone -U {repo} {dir}",
89 DownloadCmd: "pull",
90
91
92
93
94
95
96 TagCmd: []TagCmd{
97 {"tags", `^(\S+)`},
98 {"branches", `^(\S+)`},
99 },
100 TagSyncCmd: "update -r {tag}",
101 TagSyncDefault: "update default",
102
103 LogCmd: "log --encoding=utf-8 --limit={limit} --template={template}",
104
105 Scheme: []string{"https", "http", "ssh"},
106 PingCmd: "identify {scheme}://{repo}",
107 }
108
109
110 var vcsGit = &Cmd{
111 Name: "Git",
112 Cmd: "git",
113
114 CreateCmd: "clone {repo} {dir}",
115 DownloadCmd: "pull --ff-only",
116
117 TagCmd: []TagCmd{
118
119
120 {"show-ref", `(?:tags|origin)/(\S+)$`},
121 },
122 TagLookupCmd: []TagCmd{
123 {"show-ref tags/{tag} origin/{tag}", `((?:tags|origin)/\S+)$`},
124 },
125 TagSyncCmd: "checkout {tag}",
126 TagSyncDefault: "checkout master",
127
128 Scheme: []string{"git", "https", "http", "git+ssh"},
129 PingCmd: "ls-remote {scheme}://{repo}",
130 }
131
132
133 var vcsBzr = &Cmd{
134 Name: "Bazaar",
135 Cmd: "bzr",
136
137 CreateCmd: "branch {repo} {dir}",
138
139
140
141 DownloadCmd: "pull --overwrite",
142
143 TagCmd: []TagCmd{{"tags", `^(\S+)`}},
144 TagSyncCmd: "update -r {tag}",
145 TagSyncDefault: "update -r revno:-1",
146
147 Scheme: []string{"https", "http", "bzr", "bzr+ssh"},
148 PingCmd: "info {scheme}://{repo}",
149 }
150
151
152 var vcsSvn = &Cmd{
153 Name: "Subversion",
154 Cmd: "svn",
155
156 CreateCmd: "checkout {repo} {dir}",
157 DownloadCmd: "update",
158
159
160
161
162 LogCmd: "log --xml --limit={limit}",
163
164 Scheme: []string{"https", "http", "svn", "svn+ssh"},
165 PingCmd: "info {scheme}://{repo}",
166 }
167
168 func (v *Cmd) String() string {
169 return v.Name
170 }
171
172
173
174
175
176
177
178
179 func (v *Cmd) run(dir string, cmd string, keyval ...string) error {
180 _, err := v.run1(dir, cmd, keyval, true)
181 return err
182 }
183
184
185 func (v *Cmd) runVerboseOnly(dir string, cmd string, keyval ...string) error {
186 _, err := v.run1(dir, cmd, keyval, false)
187 return err
188 }
189
190
191 func (v *Cmd) runOutput(dir string, cmd string, keyval ...string) ([]byte, error) {
192 return v.run1(dir, cmd, keyval, true)
193 }
194
195
196 func (v *Cmd) run1(dir string, cmdline string, keyval []string, verbose bool) ([]byte, error) {
197 m := make(map[string]string)
198 for i := 0; i < len(keyval); i += 2 {
199 m[keyval[i]] = keyval[i+1]
200 }
201 args := strings.Fields(cmdline)
202 for i, arg := range args {
203 args[i] = expand(m, arg)
204 }
205
206 _, err := exec.LookPath(v.Cmd)
207 if err != nil {
208 fmt.Fprintf(os.Stderr,
209 "go: missing %s command. See http://golang.org/s/gogetcmd\n",
210 v.Name)
211 return nil, err
212 }
213
214 cmd := exec.Command(v.Cmd, args...)
215 cmd.Dir = dir
216 cmd.Env = envForDir(cmd.Dir)
217 if ShowCmd {
218 fmt.Printf("cd %s\n", dir)
219 fmt.Printf("%s %s\n", v.Cmd, strings.Join(args, " "))
220 }
221 var buf bytes.Buffer
222 cmd.Stdout = &buf
223 cmd.Stderr = &buf
224 err = cmd.Run()
225 out := buf.Bytes()
226 if err != nil {
227 if verbose || Verbose {
228 fmt.Fprintf(os.Stderr, "# cd %s; %s %s\n", dir, v.Cmd, strings.Join(args, " "))
229 os.Stderr.Write(out)
230 }
231 return nil, err
232 }
233 return out, nil
234 }
235
236
237
238 func (v *Cmd) Ping(scheme, repo string) error {
239 return v.runVerboseOnly(".", v.PingCmd, "scheme", scheme, "repo", repo)
240 }
241
242
243
244 func (v *Cmd) Create(dir, repo string) error {
245 return v.run(".", v.CreateCmd, "dir", dir, "repo", repo)
246 }
247
248
249
250
251 func (v *Cmd) CreateAtRev(dir, repo, rev string) error {
252 if err := v.Create(dir, repo); err != nil {
253 return err
254 }
255 return v.run(dir, v.TagSyncCmd, "tag", rev)
256 }
257
258
259
260 func (v *Cmd) Download(dir string) error {
261 return v.run(dir, v.DownloadCmd)
262 }
263
264
265
266 func (v *Cmd) Tags(dir string) ([]string, error) {
267 var tags []string
268 for _, tc := range v.TagCmd {
269 out, err := v.runOutput(dir, tc.Cmd)
270 if err != nil {
271 return nil, err
272 }
273 re := regexp.MustCompile(`(?m-s)` + tc.Pattern)
274 for _, m := range re.FindAllStringSubmatch(string(out), -1) {
275 tags = append(tags, m[1])
276 }
277 }
278 return tags, nil
279 }
280
281
282
283
284 func (v *Cmd) TagSync(dir, tag string) error {
285 if v.TagSyncCmd == "" {
286 return nil
287 }
288 if tag != "" {
289 for _, tc := range v.TagLookupCmd {
290 out, err := v.runOutput(dir, tc.Cmd, "tag", tag)
291 if err != nil {
292 return err
293 }
294 re := regexp.MustCompile(`(?m-s)` + tc.Pattern)
295 m := re.FindStringSubmatch(string(out))
296 if len(m) > 1 {
297 tag = m[1]
298 break
299 }
300 }
301 }
302 if tag == "" && v.TagSyncDefault != "" {
303 return v.run(dir, v.TagSyncDefault)
304 }
305 return v.run(dir, v.TagSyncCmd, "tag", tag)
306 }
307
308
309
310 func (v *Cmd) Log(dir, logTemplate string) ([]byte, error) {
311 if err := v.Download(dir); err != nil {
312 return []byte{}, err
313 }
314
315 const N = 50
316 return v.runOutput(dir, v.LogCmd, "limit", strconv.Itoa(N), "template", logTemplate)
317 }
318
319
320
321
322 func (v *Cmd) LogAtRev(dir, rev, logTemplate string) ([]byte, error) {
323 if err := v.Download(dir); err != nil {
324 return []byte{}, err
325 }
326
327
328 logAtRevCmd := v.LogCmd + " --rev=" + rev
329 return v.runOutput(dir, logAtRevCmd, "limit", strconv.Itoa(1), "template", logTemplate)
330 }
331
332
333
334 type vcsPath struct {
335 prefix string
336 re string
337 repo string
338 vcs string
339 check func(match map[string]string) error
340 ping bool
341
342 regexp *regexp.Regexp
343 }
344
345
346
347
348
349 func FromDir(dir, srcRoot string) (vcs *Cmd, root string, err error) {
350
351 dir = filepath.Clean(dir)
352 srcRoot = filepath.Clean(srcRoot)
353 if len(dir) <= len(srcRoot) || dir[len(srcRoot)] != filepath.Separator {
354 return nil, "", fmt.Errorf("directory %q is outside source root %q", dir, srcRoot)
355 }
356
357 var vcsRet *Cmd
358 var rootRet string
359
360 origDir := dir
361 for len(dir) > len(srcRoot) {
362 for _, vcs := range vcsList {
363 if _, err := os.Stat(filepath.Join(dir, "."+vcs.Cmd)); err == nil {
364 root := filepath.ToSlash(dir[len(srcRoot)+1:])
365
366
367 if vcsRet == nil {
368 vcsRet = vcs
369 rootRet = root
370 continue
371 }
372
373 if vcsRet == vcs && vcs.Cmd == "git" {
374 continue
375 }
376
377 return nil, "", fmt.Errorf("directory %q uses %s, but parent %q uses %s",
378 filepath.Join(srcRoot, rootRet), vcsRet.Cmd, filepath.Join(srcRoot, root), vcs.Cmd)
379 }
380 }
381
382
383 ndir := filepath.Dir(dir)
384 if len(ndir) >= len(dir) {
385
386 break
387 }
388 dir = ndir
389 }
390
391 if vcsRet != nil {
392 return vcsRet, rootRet, nil
393 }
394
395 return nil, "", fmt.Errorf("directory %q is not using a known version control system", origDir)
396 }
397
398
399
400 type RepoRoot struct {
401 VCS *Cmd
402
403
404 Repo string
405
406
407
408 Root string
409 }
410
411
412
413 func RepoRootForImportPath(importPath string, verbose bool) (*RepoRoot, error) {
414 rr, err := RepoRootForImportPathStatic(importPath, "")
415 if err == errUnknownSite {
416 rr, err = RepoRootForImportDynamic(importPath, verbose)
417
418
419
420
421
422 if err != nil {
423 if Verbose {
424 log.Printf("import %q: %v", importPath, err)
425 }
426 err = fmt.Errorf("unrecognized import path %q", importPath)
427 }
428 }
429
430 if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.Root, "...") {
431
432 rr = nil
433 err = fmt.Errorf("cannot expand ... in %q", importPath)
434 }
435 return rr, err
436 }
437
438 var errUnknownSite = errors.New("dynamic lookup required to find mapping")
439
440
441
442
443
444
445
446 func RepoRootForImportPathStatic(importPath, scheme string) (*RepoRoot, error) {
447 if strings.Contains(importPath, "://") {
448 return nil, fmt.Errorf("invalid import path %q", importPath)
449 }
450 for _, srv := range vcsPaths {
451 if !strings.HasPrefix(importPath, srv.prefix) {
452 continue
453 }
454 m := srv.regexp.FindStringSubmatch(importPath)
455 if m == nil {
456 if srv.prefix != "" {
457 return nil, fmt.Errorf("invalid %s import path %q", srv.prefix, importPath)
458 }
459 continue
460 }
461
462
463 match := map[string]string{
464 "prefix": srv.prefix,
465 "import": importPath,
466 }
467 for i, name := range srv.regexp.SubexpNames() {
468 if name != "" && match[name] == "" {
469 match[name] = m[i]
470 }
471 }
472 if srv.vcs != "" {
473 match["vcs"] = expand(match, srv.vcs)
474 }
475 if srv.repo != "" {
476 match["repo"] = expand(match, srv.repo)
477 }
478 if srv.check != nil {
479 if err := srv.check(match); err != nil {
480 return nil, err
481 }
482 }
483 vcs := ByCmd(match["vcs"])
484 if vcs == nil {
485 return nil, fmt.Errorf("unknown version control system %q", match["vcs"])
486 }
487 if srv.ping {
488 if scheme != "" {
489 match["repo"] = scheme + "://" + match["repo"]
490 } else {
491 for _, scheme := range vcs.Scheme {
492 if vcs.Ping(scheme, match["repo"]) == nil {
493 match["repo"] = scheme + "://" + match["repo"]
494 break
495 }
496 }
497 }
498 }
499 rr := &RepoRoot{
500 VCS: vcs,
501 Repo: match["repo"],
502 Root: match["root"],
503 }
504 return rr, nil
505 }
506 return nil, errUnknownSite
507 }
508
509
510
511
512
513 func RepoRootForImportDynamic(importPath string, verbose bool) (*RepoRoot, error) {
514 slash := strings.Index(importPath, "/")
515 if slash < 0 {
516 slash = len(importPath)
517 }
518 host := importPath[:slash]
519 if !strings.Contains(host, ".") {
520 return nil, errors.New("import path doesn't contain a hostname")
521 }
522 urlStr, body, err := httpsOrHTTP(importPath)
523 if err != nil {
524 return nil, fmt.Errorf("http/https fetch: %v", err)
525 }
526 defer body.Close()
527 imports, err := parseMetaGoImports(body)
528 if err != nil {
529 return nil, fmt.Errorf("parsing %s: %v", importPath, err)
530 }
531 metaImport, err := matchGoImport(imports, importPath)
532 if err != nil {
533 if err != errNoMatch {
534 return nil, fmt.Errorf("parse %s: %v", urlStr, err)
535 }
536 return nil, fmt.Errorf("parse %s: no go-import meta tags", urlStr)
537 }
538 if verbose {
539 log.Printf("get %q: found meta tag %#v at %s", importPath, metaImport, urlStr)
540 }
541
542
543
544
545
546
547 if metaImport.Prefix != importPath {
548 if verbose {
549 log.Printf("get %q: verifying non-authoritative meta tag", importPath)
550 }
551 urlStr0 := urlStr
552 urlStr, body, err = httpsOrHTTP(metaImport.Prefix)
553 if err != nil {
554 return nil, fmt.Errorf("fetch %s: %v", urlStr, err)
555 }
556 imports, err := parseMetaGoImports(body)
557 if err != nil {
558 return nil, fmt.Errorf("parsing %s: %v", importPath, err)
559 }
560 if len(imports) == 0 {
561 return nil, fmt.Errorf("fetch %s: no go-import meta tag", urlStr)
562 }
563 metaImport2, err := matchGoImport(imports, importPath)
564 if err != nil || metaImport != metaImport2 {
565 return nil, fmt.Errorf("%s and %s disagree about go-import for %s", urlStr0, urlStr, metaImport.Prefix)
566 }
567 }
568
569 if err := validateRepoRoot(metaImport.RepoRoot); err != nil {
570 return nil, fmt.Errorf("%s: invalid repo root %q: %v", urlStr, metaImport.RepoRoot, err)
571 }
572 rr := &RepoRoot{
573 VCS: ByCmd(metaImport.VCS),
574 Repo: metaImport.RepoRoot,
575 Root: metaImport.Prefix,
576 }
577 if rr.VCS == nil {
578 return nil, fmt.Errorf("%s: unknown vcs %q", urlStr, metaImport.VCS)
579 }
580 return rr, nil
581 }
582
583
584
585 func validateRepoRoot(repoRoot string) error {
586 url, err := url.Parse(repoRoot)
587 if err != nil {
588 return err
589 }
590 if url.Scheme == "" {
591 return errors.New("no scheme")
592 }
593 return nil
594 }
595
596
597
598 type metaImport struct {
599 Prefix, VCS, RepoRoot string
600 }
601
602
603 var errNoMatch = errors.New("no import match")
604
605
606
607 func pathPrefix(s, sub string) bool {
608
609 if !strings.HasPrefix(s, sub) {
610 return false
611 }
612
613 rem := s[len(sub):]
614 return rem == "" || rem[0] == '/'
615 }
616
617
618
619
620 func matchGoImport(imports []metaImport, importPath string) (_ metaImport, err error) {
621 match := -1
622 for i, im := range imports {
623 if !pathPrefix(importPath, im.Prefix) {
624 continue
625 }
626
627 if match != -1 {
628 err = fmt.Errorf("multiple meta tags match import path %q", importPath)
629 return
630 }
631 match = i
632 }
633 if match == -1 {
634 err = errNoMatch
635 return
636 }
637 return imports[match], nil
638 }
639
640
641 func expand(match map[string]string, s string) string {
642 for k, v := range match {
643 s = strings.Replace(s, "{"+k+"}", v, -1)
644 }
645 return s
646 }
647
648
649 var vcsPaths = []*vcsPath{
650
651 {
652 prefix: "github.com/",
653 re: `^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[\p{L}0-9_.\-]+)*$`,
654 vcs: "git",
655 repo: "https://{root}",
656 check: noVCSSuffix,
657 },
658
659
660 {
661 prefix: "bitbucket.org/",
662 re: `^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`,
663 repo: "https://{root}",
664 check: bitbucketVCS,
665 },
666
667
668 {
669 prefix: "launchpad.net/",
670 re: `^(?P<root>launchpad\.net/((?P<project>[A-Za-z0-9_.\-]+)(?P<series>/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`,
671 vcs: "bzr",
672 repo: "https://{root}",
673 check: launchpadVCS,
674 },
675
676
677 {
678 prefix: "git.openstack.org",
679 re: `^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/[A-Za-z0-9_.\-]+)*$`,
680 vcs: "git",
681 repo: "https://{root}",
682 check: noVCSSuffix,
683 },
684
685
686 {
687 re: `^(?P<root>(?P<repo>([a-z0-9.\-]+\.)+[a-z0-9.\-]+(:[0-9]+)?/[A-Za-z0-9_.\-/]*?)\.(?P<vcs>bzr|git|hg|svn))(/[A-Za-z0-9_.\-]+)*$`,
688 ping: true,
689 },
690 }
691
692 func init() {
693
694
695
696 for _, srv := range vcsPaths {
697 srv.regexp = regexp.MustCompile(srv.re)
698 }
699 }
700
701
702
703
704 func noVCSSuffix(match map[string]string) error {
705 repo := match["repo"]
706 for _, vcs := range vcsList {
707 if strings.HasSuffix(repo, "."+vcs.Cmd) {
708 return fmt.Errorf("invalid version control suffix in %s path", match["prefix"])
709 }
710 }
711 return nil
712 }
713
714
715
716 func bitbucketVCS(match map[string]string) error {
717 if err := noVCSSuffix(match); err != nil {
718 return err
719 }
720
721 var resp struct {
722 SCM string `json:"scm"`
723 }
724 url := expand(match, "https://api.bitbucket.org/2.0/repositories/{bitname}?fields=scm")
725 data, err := httpGET(url)
726 if err != nil {
727 return err
728 }
729 if err := json.Unmarshal(data, &resp); err != nil {
730 return fmt.Errorf("decoding %s: %v", url, err)
731 }
732
733 if ByCmd(resp.SCM) != nil {
734 match["vcs"] = resp.SCM
735 if resp.SCM == "git" {
736 match["repo"] += ".git"
737 }
738 return nil
739 }
740
741 return fmt.Errorf("unable to detect version control system for bitbucket.org/ path")
742 }
743
744
745
746
747
748 func launchpadVCS(match map[string]string) error {
749 if match["project"] == "" || match["series"] == "" {
750 return nil
751 }
752 _, err := httpGET(expand(match, "https://code.launchpad.net/{project}{series}/.bzr/branch-format"))
753 if err != nil {
754 match["root"] = expand(match, "launchpad.net/{project}")
755 match["repo"] = expand(match, "https://{root}")
756 }
757 return nil
758 }
759
View as plain text