Black Lives Matter. Support the Equal Justice Initiative.

Source file src/cmd/go/internal/modfetch/codehost/vcs.go

Documentation: cmd/go/internal/modfetch/codehost

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package codehost
     6  
     7  import (
     8  	"errors"
     9  	"fmt"
    10  	"internal/lazyregexp"
    11  	"io"
    12  	"io/ioutil"
    13  	"os"
    14  	"path/filepath"
    15  	"sort"
    16  	"strconv"
    17  	"strings"
    18  	"sync"
    19  	"time"
    20  
    21  	"cmd/go/internal/lockedfile"
    22  	"cmd/go/internal/par"
    23  	"cmd/go/internal/str"
    24  )
    25  
    26  // A VCSError indicates an error using a version control system.
    27  // The implication of a VCSError is that we know definitively where
    28  // to get the code, but we can't access it due to the error.
    29  // The caller should report this error instead of continuing to probe
    30  // other possible module paths.
    31  //
    32  // TODO(golang.org/issue/31730): See if we can invert this. (Return a
    33  // distinguished error for “repo not found” and treat everything else
    34  // as terminal.)
    35  type VCSError struct {
    36  	Err error
    37  }
    38  
    39  func (e *VCSError) Error() string { return e.Err.Error() }
    40  
    41  func vcsErrorf(format string, a ...interface{}) error {
    42  	return &VCSError{Err: fmt.Errorf(format, a...)}
    43  }
    44  
    45  func NewRepo(vcs, remote string) (Repo, error) {
    46  	type key struct {
    47  		vcs    string
    48  		remote string
    49  	}
    50  	type cached struct {
    51  		repo Repo
    52  		err  error
    53  	}
    54  	c := vcsRepoCache.Do(key{vcs, remote}, func() interface{} {
    55  		repo, err := newVCSRepo(vcs, remote)
    56  		if err != nil {
    57  			err = &VCSError{err}
    58  		}
    59  		return cached{repo, err}
    60  	}).(cached)
    61  
    62  	return c.repo, c.err
    63  }
    64  
    65  var vcsRepoCache par.Cache
    66  
    67  type vcsRepo struct {
    68  	mu lockedfile.Mutex // protects all commands, so we don't have to decide which are safe on a per-VCS basis
    69  
    70  	remote string
    71  	cmd    *vcsCmd
    72  	dir    string
    73  
    74  	tagsOnce sync.Once
    75  	tags     map[string]bool
    76  
    77  	branchesOnce sync.Once
    78  	branches     map[string]bool
    79  
    80  	fetchOnce sync.Once
    81  	fetchErr  error
    82  }
    83  
    84  func newVCSRepo(vcs, remote string) (Repo, error) {
    85  	if vcs == "git" {
    86  		return newGitRepo(remote, false)
    87  	}
    88  	cmd := vcsCmds[vcs]
    89  	if cmd == nil {
    90  		return nil, fmt.Errorf("unknown vcs: %s %s", vcs, remote)
    91  	}
    92  	if !strings.Contains(remote, "://") {
    93  		return nil, fmt.Errorf("invalid vcs remote: %s %s", vcs, remote)
    94  	}
    95  
    96  	r := &vcsRepo{remote: remote, cmd: cmd}
    97  	var err error
    98  	r.dir, r.mu.Path, err = WorkDir(vcsWorkDirType+vcs, r.remote)
    99  	if err != nil {
   100  		return nil, err
   101  	}
   102  
   103  	if cmd.init == nil {
   104  		return r, nil
   105  	}
   106  
   107  	unlock, err := r.mu.Lock()
   108  	if err != nil {
   109  		return nil, err
   110  	}
   111  	defer unlock()
   112  
   113  	if _, err := os.Stat(filepath.Join(r.dir, "."+vcs)); err != nil {
   114  		if _, err := Run(r.dir, cmd.init(r.remote)); err != nil {
   115  			os.RemoveAll(r.dir)
   116  			return nil, err
   117  		}
   118  	}
   119  	return r, nil
   120  }
   121  
   122  const vcsWorkDirType = "vcs1."
   123  
   124  type vcsCmd struct {
   125  	vcs           string                                                         // vcs name "hg"
   126  	init          func(remote string) []string                                   // cmd to init repo to track remote
   127  	tags          func(remote string) []string                                   // cmd to list local tags
   128  	tagRE         *lazyregexp.Regexp                                             // regexp to extract tag names from output of tags cmd
   129  	branches      func(remote string) []string                                   // cmd to list local branches
   130  	branchRE      *lazyregexp.Regexp                                             // regexp to extract branch names from output of tags cmd
   131  	badLocalRevRE *lazyregexp.Regexp                                             // regexp of names that must not be served out of local cache without doing fetch first
   132  	statLocal     func(rev, remote string) []string                              // cmd to stat local rev
   133  	parseStat     func(rev, out string) (*RevInfo, error)                        // cmd to parse output of statLocal
   134  	fetch         []string                                                       // cmd to fetch everything from remote
   135  	latest        string                                                         // name of latest commit on remote (tip, HEAD, etc)
   136  	readFile      func(rev, file, remote string) []string                        // cmd to read rev's file
   137  	readZip       func(rev, subdir, remote, target string) []string              // cmd to read rev's subdir as zip file
   138  	doReadZip     func(dst io.Writer, workDir, rev, subdir, remote string) error // arbitrary function to read rev's subdir as zip file
   139  }
   140  
   141  var re = lazyregexp.New
   142  
   143  var vcsCmds = map[string]*vcsCmd{
   144  	"hg": {
   145  		vcs: "hg",
   146  		init: func(remote string) []string {
   147  			return []string{"hg", "clone", "-U", "--", remote, "."}
   148  		},
   149  		tags: func(remote string) []string {
   150  			return []string{"hg", "tags", "-q"}
   151  		},
   152  		tagRE: re(`(?m)^[^\n]+$`),
   153  		branches: func(remote string) []string {
   154  			return []string{"hg", "branches", "-c", "-q"}
   155  		},
   156  		branchRE:      re(`(?m)^[^\n]+$`),
   157  		badLocalRevRE: re(`(?m)^(tip)$`),
   158  		statLocal: func(rev, remote string) []string {
   159  			return []string{"hg", "log", "-l1", "-r", rev, "--template", "{node} {date|hgdate} {tags}"}
   160  		},
   161  		parseStat: hgParseStat,
   162  		fetch:     []string{"hg", "pull", "-f"},
   163  		latest:    "tip",
   164  		readFile: func(rev, file, remote string) []string {
   165  			return []string{"hg", "cat", "-r", rev, file}
   166  		},
   167  		readZip: func(rev, subdir, remote, target string) []string {
   168  			pattern := []string{}
   169  			if subdir != "" {
   170  				pattern = []string{"-I", subdir + "/**"}
   171  			}
   172  			return str.StringList("hg", "archive", "-t", "zip", "--no-decode", "-r", rev, "--prefix=prefix/", pattern, "--", target)
   173  		},
   174  	},
   175  
   176  	"svn": {
   177  		vcs:  "svn",
   178  		init: nil, // no local checkout
   179  		tags: func(remote string) []string {
   180  			return []string{"svn", "list", "--", strings.TrimSuffix(remote, "/trunk") + "/tags"}
   181  		},
   182  		tagRE: re(`(?m)^(.*?)/?$`),
   183  		statLocal: func(rev, remote string) []string {
   184  			suffix := "@" + rev
   185  			if rev == "latest" {
   186  				suffix = ""
   187  			}
   188  			return []string{"svn", "log", "-l1", "--xml", "--", remote + suffix}
   189  		},
   190  		parseStat: svnParseStat,
   191  		latest:    "latest",
   192  		readFile: func(rev, file, remote string) []string {
   193  			return []string{"svn", "cat", "--", remote + "/" + file + "@" + rev}
   194  		},
   195  		doReadZip: svnReadZip,
   196  	},
   197  
   198  	"bzr": {
   199  		vcs: "bzr",
   200  		init: func(remote string) []string {
   201  			return []string{"bzr", "branch", "--use-existing-dir", "--", remote, "."}
   202  		},
   203  		fetch: []string{
   204  			"bzr", "pull", "--overwrite-tags",
   205  		},
   206  		tags: func(remote string) []string {
   207  			return []string{"bzr", "tags"}
   208  		},
   209  		tagRE:         re(`(?m)^\S+`),
   210  		badLocalRevRE: re(`^revno:-`),
   211  		statLocal: func(rev, remote string) []string {
   212  			return []string{"bzr", "log", "-l1", "--long", "--show-ids", "-r", rev}
   213  		},
   214  		parseStat: bzrParseStat,
   215  		latest:    "revno:-1",
   216  		readFile: func(rev, file, remote string) []string {
   217  			return []string{"bzr", "cat", "-r", rev, file}
   218  		},
   219  		readZip: func(rev, subdir, remote, target string) []string {
   220  			extra := []string{}
   221  			if subdir != "" {
   222  				extra = []string{"./" + subdir}
   223  			}
   224  			return str.StringList("bzr", "export", "--format=zip", "-r", rev, "--root=prefix/", "--", target, extra)
   225  		},
   226  	},
   227  
   228  	"fossil": {
   229  		vcs: "fossil",
   230  		init: func(remote string) []string {
   231  			return []string{"fossil", "clone", "--", remote, ".fossil"}
   232  		},
   233  		fetch: []string{"fossil", "pull", "-R", ".fossil"},
   234  		tags: func(remote string) []string {
   235  			return []string{"fossil", "tag", "-R", ".fossil", "list"}
   236  		},
   237  		tagRE: re(`XXXTODO`),
   238  		statLocal: func(rev, remote string) []string {
   239  			return []string{"fossil", "info", "-R", ".fossil", rev}
   240  		},
   241  		parseStat: fossilParseStat,
   242  		latest:    "trunk",
   243  		readFile: func(rev, file, remote string) []string {
   244  			return []string{"fossil", "cat", "-R", ".fossil", "-r", rev, file}
   245  		},
   246  		readZip: func(rev, subdir, remote, target string) []string {
   247  			extra := []string{}
   248  			if subdir != "" && !strings.ContainsAny(subdir, "*?[],") {
   249  				extra = []string{"--include", subdir}
   250  			}
   251  			// Note that vcsRepo.ReadZip below rewrites this command
   252  			// to run in a different directory, to work around a fossil bug.
   253  			return str.StringList("fossil", "zip", "-R", ".fossil", "--name", "prefix", extra, "--", rev, target)
   254  		},
   255  	},
   256  }
   257  
   258  func (r *vcsRepo) loadTags() {
   259  	out, err := Run(r.dir, r.cmd.tags(r.remote))
   260  	if err != nil {
   261  		return
   262  	}
   263  
   264  	// Run tag-listing command and extract tags.
   265  	r.tags = make(map[string]bool)
   266  	for _, tag := range r.cmd.tagRE.FindAllString(string(out), -1) {
   267  		if r.cmd.badLocalRevRE != nil && r.cmd.badLocalRevRE.MatchString(tag) {
   268  			continue
   269  		}
   270  		r.tags[tag] = true
   271  	}
   272  }
   273  
   274  func (r *vcsRepo) loadBranches() {
   275  	if r.cmd.branches == nil {
   276  		return
   277  	}
   278  
   279  	out, err := Run(r.dir, r.cmd.branches(r.remote))
   280  	if err != nil {
   281  		return
   282  	}
   283  
   284  	r.branches = make(map[string]bool)
   285  	for _, branch := range r.cmd.branchRE.FindAllString(string(out), -1) {
   286  		if r.cmd.badLocalRevRE != nil && r.cmd.badLocalRevRE.MatchString(branch) {
   287  			continue
   288  		}
   289  		r.branches[branch] = true
   290  	}
   291  }
   292  
   293  func (r *vcsRepo) Tags(prefix string) ([]string, error) {
   294  	unlock, err := r.mu.Lock()
   295  	if err != nil {
   296  		return nil, err
   297  	}
   298  	defer unlock()
   299  
   300  	r.tagsOnce.Do(r.loadTags)
   301  
   302  	tags := []string{}
   303  	for tag := range r.tags {
   304  		if strings.HasPrefix(tag, prefix) {
   305  			tags = append(tags, tag)
   306  		}
   307  	}
   308  	sort.Strings(tags)
   309  	return tags, nil
   310  }
   311  
   312  func (r *vcsRepo) Stat(rev string) (*RevInfo, error) {
   313  	unlock, err := r.mu.Lock()
   314  	if err != nil {
   315  		return nil, err
   316  	}
   317  	defer unlock()
   318  
   319  	if rev == "latest" {
   320  		rev = r.cmd.latest
   321  	}
   322  	r.branchesOnce.Do(r.loadBranches)
   323  	revOK := (r.cmd.badLocalRevRE == nil || !r.cmd.badLocalRevRE.MatchString(rev)) && !r.branches[rev]
   324  	if revOK {
   325  		if info, err := r.statLocal(rev); err == nil {
   326  			return info, nil
   327  		}
   328  	}
   329  
   330  	r.fetchOnce.Do(r.fetch)
   331  	if r.fetchErr != nil {
   332  		return nil, r.fetchErr
   333  	}
   334  	info, err := r.statLocal(rev)
   335  	if err != nil {
   336  		return nil, err
   337  	}
   338  	if !revOK {
   339  		info.Version = info.Name
   340  	}
   341  	return info, nil
   342  }
   343  
   344  func (r *vcsRepo) fetch() {
   345  	if len(r.cmd.fetch) > 0 {
   346  		_, r.fetchErr = Run(r.dir, r.cmd.fetch)
   347  	}
   348  }
   349  
   350  func (r *vcsRepo) statLocal(rev string) (*RevInfo, error) {
   351  	out, err := Run(r.dir, r.cmd.statLocal(rev, r.remote))
   352  	if err != nil {
   353  		return nil, &UnknownRevisionError{Rev: rev}
   354  	}
   355  	return r.cmd.parseStat(rev, string(out))
   356  }
   357  
   358  func (r *vcsRepo) Latest() (*RevInfo, error) {
   359  	return r.Stat("latest")
   360  }
   361  
   362  func (r *vcsRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) {
   363  	if rev == "latest" {
   364  		rev = r.cmd.latest
   365  	}
   366  	_, err := r.Stat(rev) // download rev into local repo
   367  	if err != nil {
   368  		return nil, err
   369  	}
   370  
   371  	// r.Stat acquires r.mu, so lock after that.
   372  	unlock, err := r.mu.Lock()
   373  	if err != nil {
   374  		return nil, err
   375  	}
   376  	defer unlock()
   377  
   378  	out, err := Run(r.dir, r.cmd.readFile(rev, file, r.remote))
   379  	if err != nil {
   380  		return nil, os.ErrNotExist
   381  	}
   382  	return out, nil
   383  }
   384  
   385  func (r *vcsRepo) ReadFileRevs(revs []string, file string, maxSize int64) (map[string]*FileRev, error) {
   386  	// We don't technically need to lock here since we're returning an error
   387  	// uncondititonally, but doing so anyway will help to avoid baking in
   388  	// lock-inversion bugs.
   389  	unlock, err := r.mu.Lock()
   390  	if err != nil {
   391  		return nil, err
   392  	}
   393  	defer unlock()
   394  
   395  	return nil, vcsErrorf("ReadFileRevs not implemented")
   396  }
   397  
   398  func (r *vcsRepo) RecentTag(rev, prefix, major string) (tag string, err error) {
   399  	// We don't technically need to lock here since we're returning an error
   400  	// uncondititonally, but doing so anyway will help to avoid baking in
   401  	// lock-inversion bugs.
   402  	unlock, err := r.mu.Lock()
   403  	if err != nil {
   404  		return "", err
   405  	}
   406  	defer unlock()
   407  
   408  	return "", vcsErrorf("RecentTag not implemented")
   409  }
   410  
   411  func (r *vcsRepo) DescendsFrom(rev, tag string) (bool, error) {
   412  	unlock, err := r.mu.Lock()
   413  	if err != nil {
   414  		return false, err
   415  	}
   416  	defer unlock()
   417  
   418  	return false, vcsErrorf("DescendsFrom not implemented")
   419  }
   420  
   421  func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error) {
   422  	if r.cmd.readZip == nil && r.cmd.doReadZip == nil {
   423  		return nil, vcsErrorf("ReadZip not implemented for %s", r.cmd.vcs)
   424  	}
   425  
   426  	unlock, err := r.mu.Lock()
   427  	if err != nil {
   428  		return nil, err
   429  	}
   430  	defer unlock()
   431  
   432  	if rev == "latest" {
   433  		rev = r.cmd.latest
   434  	}
   435  	f, err := ioutil.TempFile("", "go-readzip-*.zip")
   436  	if err != nil {
   437  		return nil, err
   438  	}
   439  	if r.cmd.doReadZip != nil {
   440  		lw := &limitedWriter{
   441  			W:               f,
   442  			N:               maxSize,
   443  			ErrLimitReached: errors.New("ReadZip: encoded file exceeds allowed size"),
   444  		}
   445  		err = r.cmd.doReadZip(lw, r.dir, rev, subdir, r.remote)
   446  		if err == nil {
   447  			_, err = f.Seek(0, io.SeekStart)
   448  		}
   449  	} else if r.cmd.vcs == "fossil" {
   450  		// If you run
   451  		//	fossil zip -R .fossil --name prefix trunk /tmp/x.zip
   452  		// fossil fails with "unable to create directory /tmp" [sic].
   453  		// Change the command to run in /tmp instead,
   454  		// replacing the -R argument with an absolute path.
   455  		args := r.cmd.readZip(rev, subdir, r.remote, filepath.Base(f.Name()))
   456  		for i := range args {
   457  			if args[i] == ".fossil" {
   458  				args[i] = filepath.Join(r.dir, ".fossil")
   459  			}
   460  		}
   461  		_, err = Run(filepath.Dir(f.Name()), args)
   462  	} else {
   463  		_, err = Run(r.dir, r.cmd.readZip(rev, subdir, r.remote, f.Name()))
   464  	}
   465  	if err != nil {
   466  		f.Close()
   467  		os.Remove(f.Name())
   468  		return nil, err
   469  	}
   470  	return &deleteCloser{f}, nil
   471  }
   472  
   473  // deleteCloser is a file that gets deleted on Close.
   474  type deleteCloser struct {
   475  	*os.File
   476  }
   477  
   478  func (d *deleteCloser) Close() error {
   479  	defer os.Remove(d.File.Name())
   480  	return d.File.Close()
   481  }
   482  
   483  func hgParseStat(rev, out string) (*RevInfo, error) {
   484  	f := strings.Fields(string(out))
   485  	if len(f) < 3 {
   486  		return nil, vcsErrorf("unexpected response from hg log: %q", out)
   487  	}
   488  	hash := f[0]
   489  	version := rev
   490  	if strings.HasPrefix(hash, version) {
   491  		version = hash // extend to full hash
   492  	}
   493  	t, err := strconv.ParseInt(f[1], 10, 64)
   494  	if err != nil {
   495  		return nil, vcsErrorf("invalid time from hg log: %q", out)
   496  	}
   497  
   498  	var tags []string
   499  	for _, tag := range f[3:] {
   500  		if tag != "tip" {
   501  			tags = append(tags, tag)
   502  		}
   503  	}
   504  	sort.Strings(tags)
   505  
   506  	info := &RevInfo{
   507  		Name:    hash,
   508  		Short:   ShortenSHA1(hash),
   509  		Time:    time.Unix(t, 0).UTC(),
   510  		Version: version,
   511  		Tags:    tags,
   512  	}
   513  	return info, nil
   514  }
   515  
   516  func bzrParseStat(rev, out string) (*RevInfo, error) {
   517  	var revno int64
   518  	var tm time.Time
   519  	for _, line := range strings.Split(out, "\n") {
   520  		if line == "" || line[0] == ' ' || line[0] == '\t' {
   521  			// End of header, start of commit message.
   522  			break
   523  		}
   524  		if line[0] == '-' {
   525  			continue
   526  		}
   527  		i := strings.Index(line, ":")
   528  		if i < 0 {
   529  			// End of header, start of commit message.
   530  			break
   531  		}
   532  		key, val := line[:i], strings.TrimSpace(line[i+1:])
   533  		switch key {
   534  		case "revno":
   535  			if j := strings.Index(val, " "); j >= 0 {
   536  				val = val[:j]
   537  			}
   538  			i, err := strconv.ParseInt(val, 10, 64)
   539  			if err != nil {
   540  				return nil, vcsErrorf("unexpected revno from bzr log: %q", line)
   541  			}
   542  			revno = i
   543  		case "timestamp":
   544  			j := strings.Index(val, " ")
   545  			if j < 0 {
   546  				return nil, vcsErrorf("unexpected timestamp from bzr log: %q", line)
   547  			}
   548  			t, err := time.Parse("2006-01-02 15:04:05 -0700", val[j+1:])
   549  			if err != nil {
   550  				return nil, vcsErrorf("unexpected timestamp from bzr log: %q", line)
   551  			}
   552  			tm = t.UTC()
   553  		}
   554  	}
   555  	if revno == 0 || tm.IsZero() {
   556  		return nil, vcsErrorf("unexpected response from bzr log: %q", out)
   557  	}
   558  
   559  	info := &RevInfo{
   560  		Name:    fmt.Sprintf("%d", revno),
   561  		Short:   fmt.Sprintf("%012d", revno),
   562  		Time:    tm,
   563  		Version: rev,
   564  	}
   565  	return info, nil
   566  }
   567  
   568  func fossilParseStat(rev, out string) (*RevInfo, error) {
   569  	for _, line := range strings.Split(out, "\n") {
   570  		if strings.HasPrefix(line, "uuid:") {
   571  			f := strings.Fields(line)
   572  			if len(f) != 5 || len(f[1]) != 40 || f[4] != "UTC" {
   573  				return nil, vcsErrorf("unexpected response from fossil info: %q", line)
   574  			}
   575  			t, err := time.Parse("2006-01-02 15:04:05", f[2]+" "+f[3])
   576  			if err != nil {
   577  				return nil, vcsErrorf("unexpected response from fossil info: %q", line)
   578  			}
   579  			hash := f[1]
   580  			version := rev
   581  			if strings.HasPrefix(hash, version) {
   582  				version = hash // extend to full hash
   583  			}
   584  			info := &RevInfo{
   585  				Name:    hash,
   586  				Short:   ShortenSHA1(hash),
   587  				Time:    t,
   588  				Version: version,
   589  			}
   590  			return info, nil
   591  		}
   592  	}
   593  	return nil, vcsErrorf("unexpected response from fossil info: %q", out)
   594  }
   595  
   596  type limitedWriter struct {
   597  	W               io.Writer
   598  	N               int64
   599  	ErrLimitReached error
   600  }
   601  
   602  func (l *limitedWriter) Write(p []byte) (n int, err error) {
   603  	if l.N > 0 {
   604  		max := len(p)
   605  		if l.N < int64(max) {
   606  			max = int(l.N)
   607  		}
   608  		n, err = l.W.Write(p[:max])
   609  		l.N -= int64(n)
   610  		if err != nil || n >= len(p) {
   611  			return n, err
   612  		}
   613  	}
   614  
   615  	return n, l.ErrLimitReached
   616  }
   617  

View as plain text