Black Lives Matter. Support the Equal Justice Initiative.

Source file src/cmd/go/internal/modfetch/codehost/git.go

Documentation: cmd/go/internal/modfetch/codehost

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package codehost
     6  
     7  import (
     8  	"bytes"
     9  	"errors"
    10  	"fmt"
    11  	"io"
    12  	"io/ioutil"
    13  	"net/url"
    14  	"os"
    15  	"os/exec"
    16  	"path/filepath"
    17  	"sort"
    18  	"strconv"
    19  	"strings"
    20  	"sync"
    21  	"time"
    22  
    23  	"cmd/go/internal/lockedfile"
    24  	"cmd/go/internal/par"
    25  	"cmd/go/internal/web"
    26  
    27  	"golang.org/x/mod/semver"
    28  )
    29  
    30  // LocalGitRepo is like Repo but accepts both Git remote references
    31  // and paths to repositories on the local file system.
    32  func LocalGitRepo(remote string) (Repo, error) {
    33  	return newGitRepoCached(remote, true)
    34  }
    35  
    36  // A notExistError wraps another error to retain its original text
    37  // but makes it opaquely equivalent to os.ErrNotExist.
    38  type notExistError struct {
    39  	err error
    40  }
    41  
    42  func (e notExistError) Error() string   { return e.err.Error() }
    43  func (notExistError) Is(err error) bool { return err == os.ErrNotExist }
    44  
    45  const gitWorkDirType = "git3"
    46  
    47  var gitRepoCache par.Cache
    48  
    49  func newGitRepoCached(remote string, localOK bool) (Repo, error) {
    50  	type key struct {
    51  		remote  string
    52  		localOK bool
    53  	}
    54  	type cached struct {
    55  		repo Repo
    56  		err  error
    57  	}
    58  
    59  	c := gitRepoCache.Do(key{remote, localOK}, func() interface{} {
    60  		repo, err := newGitRepo(remote, localOK)
    61  		return cached{repo, err}
    62  	}).(cached)
    63  
    64  	return c.repo, c.err
    65  }
    66  
    67  func newGitRepo(remote string, localOK bool) (Repo, error) {
    68  	r := &gitRepo{remote: remote}
    69  	if strings.Contains(remote, "://") {
    70  		// This is a remote path.
    71  		var err error
    72  		r.dir, r.mu.Path, err = WorkDir(gitWorkDirType, r.remote)
    73  		if err != nil {
    74  			return nil, err
    75  		}
    76  
    77  		unlock, err := r.mu.Lock()
    78  		if err != nil {
    79  			return nil, err
    80  		}
    81  		defer unlock()
    82  
    83  		if _, err := os.Stat(filepath.Join(r.dir, "objects")); err != nil {
    84  			if _, err := Run(r.dir, "git", "init", "--bare"); err != nil {
    85  				os.RemoveAll(r.dir)
    86  				return nil, err
    87  			}
    88  			// We could just say git fetch https://whatever later,
    89  			// but this lets us say git fetch origin instead, which
    90  			// is a little nicer. More importantly, using a named remote
    91  			// avoids a problem with Git LFS. See golang.org/issue/25605.
    92  			if _, err := Run(r.dir, "git", "remote", "add", "origin", "--", r.remote); err != nil {
    93  				os.RemoveAll(r.dir)
    94  				return nil, err
    95  			}
    96  		}
    97  		r.remoteURL = r.remote
    98  		r.remote = "origin"
    99  	} else {
   100  		// Local path.
   101  		// Disallow colon (not in ://) because sometimes
   102  		// that's rcp-style host:path syntax and sometimes it's not (c:\work).
   103  		// The go command has always insisted on URL syntax for ssh.
   104  		if strings.Contains(remote, ":") {
   105  			return nil, fmt.Errorf("git remote cannot use host:path syntax")
   106  		}
   107  		if !localOK {
   108  			return nil, fmt.Errorf("git remote must not be local directory")
   109  		}
   110  		r.local = true
   111  		info, err := os.Stat(remote)
   112  		if err != nil {
   113  			return nil, err
   114  		}
   115  		if !info.IsDir() {
   116  			return nil, fmt.Errorf("%s exists but is not a directory", remote)
   117  		}
   118  		r.dir = remote
   119  		r.mu.Path = r.dir + ".lock"
   120  	}
   121  	return r, nil
   122  }
   123  
   124  type gitRepo struct {
   125  	remote, remoteURL string
   126  	local             bool
   127  	dir               string
   128  
   129  	mu lockedfile.Mutex // protects fetchLevel and git repo state
   130  
   131  	fetchLevel int
   132  
   133  	statCache par.Cache
   134  
   135  	refsOnce sync.Once
   136  	// refs maps branch and tag refs (e.g., "HEAD", "refs/heads/master")
   137  	// to commits (e.g., "37ffd2e798afde829a34e8955b716ab730b2a6d6")
   138  	refs    map[string]string
   139  	refsErr error
   140  
   141  	localTagsOnce sync.Once
   142  	localTags     map[string]bool
   143  }
   144  
   145  const (
   146  	// How much have we fetched into the git repo (in this process)?
   147  	fetchNone = iota // nothing yet
   148  	fetchSome        // shallow fetches of individual hashes
   149  	fetchAll         // "fetch -t origin": get all remote branches and tags
   150  )
   151  
   152  // loadLocalTags loads tag references from the local git cache
   153  // into the map r.localTags.
   154  // Should only be called as r.localTagsOnce.Do(r.loadLocalTags).
   155  func (r *gitRepo) loadLocalTags() {
   156  	// The git protocol sends all known refs and ls-remote filters them on the client side,
   157  	// so we might as well record both heads and tags in one shot.
   158  	// Most of the time we only care about tags but sometimes we care about heads too.
   159  	out, err := Run(r.dir, "git", "tag", "-l")
   160  	if err != nil {
   161  		return
   162  	}
   163  
   164  	r.localTags = make(map[string]bool)
   165  	for _, line := range strings.Split(string(out), "\n") {
   166  		if line != "" {
   167  			r.localTags[line] = true
   168  		}
   169  	}
   170  }
   171  
   172  // loadRefs loads heads and tags references from the remote into the map r.refs.
   173  // Should only be called as r.refsOnce.Do(r.loadRefs).
   174  func (r *gitRepo) loadRefs() {
   175  	// The git protocol sends all known refs and ls-remote filters them on the client side,
   176  	// so we might as well record both heads and tags in one shot.
   177  	// Most of the time we only care about tags but sometimes we care about heads too.
   178  	out, gitErr := Run(r.dir, "git", "ls-remote", "-q", r.remote)
   179  	if gitErr != nil {
   180  		if rerr, ok := gitErr.(*RunError); ok {
   181  			if bytes.Contains(rerr.Stderr, []byte("fatal: could not read Username")) {
   182  				rerr.HelpText = "Confirm the import path was entered correctly.\nIf this is a private repository, see https://golang.org/doc/faq#git_https for additional information."
   183  			}
   184  		}
   185  
   186  		// If the remote URL doesn't exist at all, ideally we should treat the whole
   187  		// repository as nonexistent by wrapping the error in a notExistError.
   188  		// For HTTP and HTTPS, that's easy to detect: we'll try to fetch the URL
   189  		// ourselves and see what code it serves.
   190  		if u, err := url.Parse(r.remoteURL); err == nil && (u.Scheme == "http" || u.Scheme == "https") {
   191  			if _, err := web.GetBytes(u); errors.Is(err, os.ErrNotExist) {
   192  				gitErr = notExistError{gitErr}
   193  			}
   194  		}
   195  
   196  		r.refsErr = gitErr
   197  		return
   198  	}
   199  
   200  	r.refs = make(map[string]string)
   201  	for _, line := range strings.Split(string(out), "\n") {
   202  		f := strings.Fields(line)
   203  		if len(f) != 2 {
   204  			continue
   205  		}
   206  		if f[1] == "HEAD" || strings.HasPrefix(f[1], "refs/heads/") || strings.HasPrefix(f[1], "refs/tags/") {
   207  			r.refs[f[1]] = f[0]
   208  		}
   209  	}
   210  	for ref, hash := range r.refs {
   211  		if strings.HasSuffix(ref, "^{}") { // record unwrapped annotated tag as value of tag
   212  			r.refs[strings.TrimSuffix(ref, "^{}")] = hash
   213  			delete(r.refs, ref)
   214  		}
   215  	}
   216  }
   217  
   218  func (r *gitRepo) Tags(prefix string) ([]string, error) {
   219  	r.refsOnce.Do(r.loadRefs)
   220  	if r.refsErr != nil {
   221  		return nil, r.refsErr
   222  	}
   223  
   224  	tags := []string{}
   225  	for ref := range r.refs {
   226  		if !strings.HasPrefix(ref, "refs/tags/") {
   227  			continue
   228  		}
   229  		tag := ref[len("refs/tags/"):]
   230  		if !strings.HasPrefix(tag, prefix) {
   231  			continue
   232  		}
   233  		tags = append(tags, tag)
   234  	}
   235  	sort.Strings(tags)
   236  	return tags, nil
   237  }
   238  
   239  func (r *gitRepo) Latest() (*RevInfo, error) {
   240  	r.refsOnce.Do(r.loadRefs)
   241  	if r.refsErr != nil {
   242  		return nil, r.refsErr
   243  	}
   244  	if r.refs["HEAD"] == "" {
   245  		return nil, ErrNoCommits
   246  	}
   247  	return r.Stat(r.refs["HEAD"])
   248  }
   249  
   250  // findRef finds some ref name for the given hash,
   251  // for use when the server requires giving a ref instead of a hash.
   252  // There may be multiple ref names for a given hash,
   253  // in which case this returns some name - it doesn't matter which.
   254  func (r *gitRepo) findRef(hash string) (ref string, ok bool) {
   255  	r.refsOnce.Do(r.loadRefs)
   256  	for ref, h := range r.refs {
   257  		if h == hash {
   258  			return ref, true
   259  		}
   260  	}
   261  	return "", false
   262  }
   263  
   264  // minHashDigits is the minimum number of digits to require
   265  // before accepting a hex digit sequence as potentially identifying
   266  // a specific commit in a git repo. (Of course, users can always
   267  // specify more digits, and many will paste in all 40 digits,
   268  // but many of git's commands default to printing short hashes
   269  // as 7 digits.)
   270  const minHashDigits = 7
   271  
   272  // stat stats the given rev in the local repository,
   273  // or else it fetches more info from the remote repository and tries again.
   274  func (r *gitRepo) stat(rev string) (*RevInfo, error) {
   275  	if r.local {
   276  		return r.statLocal(rev, rev)
   277  	}
   278  
   279  	// Fast path: maybe rev is a hash we already have locally.
   280  	didStatLocal := false
   281  	if len(rev) >= minHashDigits && len(rev) <= 40 && AllHex(rev) {
   282  		if info, err := r.statLocal(rev, rev); err == nil {
   283  			return info, nil
   284  		}
   285  		didStatLocal = true
   286  	}
   287  
   288  	// Maybe rev is a tag we already have locally.
   289  	// (Note that we're excluding branches, which can be stale.)
   290  	r.localTagsOnce.Do(r.loadLocalTags)
   291  	if r.localTags[rev] {
   292  		return r.statLocal(rev, "refs/tags/"+rev)
   293  	}
   294  
   295  	// Maybe rev is the name of a tag or branch on the remote server.
   296  	// Or maybe it's the prefix of a hash of a named ref.
   297  	// Try to resolve to both a ref (git name) and full (40-hex-digit) commit hash.
   298  	r.refsOnce.Do(r.loadRefs)
   299  	var ref, hash string
   300  	if r.refs["refs/tags/"+rev] != "" {
   301  		ref = "refs/tags/" + rev
   302  		hash = r.refs[ref]
   303  		// Keep rev as is: tags are assumed not to change meaning.
   304  	} else if r.refs["refs/heads/"+rev] != "" {
   305  		ref = "refs/heads/" + rev
   306  		hash = r.refs[ref]
   307  		rev = hash // Replace rev, because meaning of refs/heads/foo can change.
   308  	} else if rev == "HEAD" && r.refs["HEAD"] != "" {
   309  		ref = "HEAD"
   310  		hash = r.refs[ref]
   311  		rev = hash // Replace rev, because meaning of HEAD can change.
   312  	} else if len(rev) >= minHashDigits && len(rev) <= 40 && AllHex(rev) {
   313  		// At the least, we have a hash prefix we can look up after the fetch below.
   314  		// Maybe we can map it to a full hash using the known refs.
   315  		prefix := rev
   316  		// Check whether rev is prefix of known ref hash.
   317  		for k, h := range r.refs {
   318  			if strings.HasPrefix(h, prefix) {
   319  				if hash != "" && hash != h {
   320  					// Hash is an ambiguous hash prefix.
   321  					// More information will not change that.
   322  					return nil, fmt.Errorf("ambiguous revision %s", rev)
   323  				}
   324  				if ref == "" || ref > k { // Break ties deterministically when multiple refs point at same hash.
   325  					ref = k
   326  				}
   327  				rev = h
   328  				hash = h
   329  			}
   330  		}
   331  		if hash == "" && len(rev) == 40 { // Didn't find a ref, but rev is a full hash.
   332  			hash = rev
   333  		}
   334  	} else {
   335  		return nil, &UnknownRevisionError{Rev: rev}
   336  	}
   337  
   338  	// Protect r.fetchLevel and the "fetch more and more" sequence.
   339  	unlock, err := r.mu.Lock()
   340  	if err != nil {
   341  		return nil, err
   342  	}
   343  	defer unlock()
   344  
   345  	// Perhaps r.localTags did not have the ref when we loaded local tags,
   346  	// but we've since done fetches that pulled down the hash we need
   347  	// (or already have the hash we need, just without its tag).
   348  	// Either way, try a local stat before falling back to network I/O.
   349  	if !didStatLocal {
   350  		if info, err := r.statLocal(rev, hash); err == nil {
   351  			if strings.HasPrefix(ref, "refs/tags/") {
   352  				// Make sure tag exists, so it will be in localTags next time the go command is run.
   353  				Run(r.dir, "git", "tag", strings.TrimPrefix(ref, "refs/tags/"), hash)
   354  			}
   355  			return info, nil
   356  		}
   357  	}
   358  
   359  	// If we know a specific commit we need and its ref, fetch it.
   360  	// We do NOT fetch arbitrary hashes (when we don't know the ref)
   361  	// because we want to avoid ever importing a commit that isn't
   362  	// reachable from refs/tags/* or refs/heads/* or HEAD.
   363  	// Both Gerrit and GitHub expose every CL/PR as a named ref,
   364  	// and we don't want those commits masquerading as being real
   365  	// pseudo-versions in the main repo.
   366  	if r.fetchLevel <= fetchSome && ref != "" && hash != "" && !r.local {
   367  		r.fetchLevel = fetchSome
   368  		var refspec string
   369  		if ref != "" && ref != "HEAD" {
   370  			// If we do know the ref name, save the mapping locally
   371  			// so that (if it is a tag) it can show up in localTags
   372  			// on a future call. Also, some servers refuse to allow
   373  			// full hashes in ref specs, so prefer a ref name if known.
   374  			refspec = ref + ":" + ref
   375  		} else {
   376  			// Fetch the hash but give it a local name (refs/dummy),
   377  			// because that triggers the fetch behavior of creating any
   378  			// other known remote tags for the hash. We never use
   379  			// refs/dummy (it's not refs/tags/dummy) and it will be
   380  			// overwritten in the next command, and that's fine.
   381  			ref = hash
   382  			refspec = hash + ":refs/dummy"
   383  		}
   384  		_, err := Run(r.dir, "git", "fetch", "-f", "--depth=1", r.remote, refspec)
   385  		if err == nil {
   386  			return r.statLocal(rev, ref)
   387  		}
   388  		// Don't try to be smart about parsing the error.
   389  		// It's too complex and varies too much by git version.
   390  		// No matter what went wrong, fall back to a complete fetch.
   391  	}
   392  
   393  	// Last resort.
   394  	// Fetch all heads and tags and hope the hash we want is in the history.
   395  	if err := r.fetchRefsLocked(); err != nil {
   396  		return nil, err
   397  	}
   398  
   399  	return r.statLocal(rev, rev)
   400  }
   401  
   402  // fetchRefsLocked fetches all heads and tags from the origin, along with the
   403  // ancestors of those commits.
   404  //
   405  // We only fetch heads and tags, not arbitrary other commits: we don't want to
   406  // pull in off-branch commits (such as rejected GitHub pull requests) that the
   407  // server may be willing to provide. (See the comments within the stat method
   408  // for more detail.)
   409  //
   410  // fetchRefsLocked requires that r.mu remain locked for the duration of the call.
   411  func (r *gitRepo) fetchRefsLocked() error {
   412  	if r.fetchLevel < fetchAll {
   413  		// NOTE: To work around a bug affecting Git clients up to at least 2.23.0
   414  		// (2019-08-16), we must first expand the set of local refs, and only then
   415  		// unshallow the repository as a separate fetch operation. (See
   416  		// golang.org/issue/34266 and
   417  		// https://github.com/git/git/blob/4c86140027f4a0d2caaa3ab4bd8bfc5ce3c11c8a/transport.c#L1303-L1309.)
   418  
   419  		if _, err := Run(r.dir, "git", "fetch", "-f", r.remote, "refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"); err != nil {
   420  			return err
   421  		}
   422  
   423  		if _, err := os.Stat(filepath.Join(r.dir, "shallow")); err == nil {
   424  			if _, err := Run(r.dir, "git", "fetch", "--unshallow", "-f", r.remote); err != nil {
   425  				return err
   426  			}
   427  		}
   428  
   429  		r.fetchLevel = fetchAll
   430  	}
   431  	return nil
   432  }
   433  
   434  // statLocal returns a RevInfo describing rev in the local git repository.
   435  // It uses version as info.Version.
   436  func (r *gitRepo) statLocal(version, rev string) (*RevInfo, error) {
   437  	out, err := Run(r.dir, "git", "-c", "log.showsignature=false", "log", "-n1", "--format=format:%H %ct %D", rev, "--")
   438  	if err != nil {
   439  		return nil, &UnknownRevisionError{Rev: rev}
   440  	}
   441  	f := strings.Fields(string(out))
   442  	if len(f) < 2 {
   443  		return nil, fmt.Errorf("unexpected response from git log: %q", out)
   444  	}
   445  	hash := f[0]
   446  	if strings.HasPrefix(hash, version) {
   447  		version = hash // extend to full hash
   448  	}
   449  	t, err := strconv.ParseInt(f[1], 10, 64)
   450  	if err != nil {
   451  		return nil, fmt.Errorf("invalid time from git log: %q", out)
   452  	}
   453  
   454  	info := &RevInfo{
   455  		Name:    hash,
   456  		Short:   ShortenSHA1(hash),
   457  		Time:    time.Unix(t, 0).UTC(),
   458  		Version: hash,
   459  	}
   460  
   461  	// Add tags. Output looks like:
   462  	//	ede458df7cd0fdca520df19a33158086a8a68e81 1523994202 HEAD -> master, tag: v1.2.4-annotated, tag: v1.2.3, origin/master, origin/HEAD
   463  	for i := 2; i < len(f); i++ {
   464  		if f[i] == "tag:" {
   465  			i++
   466  			if i < len(f) {
   467  				info.Tags = append(info.Tags, strings.TrimSuffix(f[i], ","))
   468  			}
   469  		}
   470  	}
   471  	sort.Strings(info.Tags)
   472  
   473  	// Used hash as info.Version above.
   474  	// Use caller's suggested version if it appears in the tag list
   475  	// (filters out branch names, HEAD).
   476  	for _, tag := range info.Tags {
   477  		if version == tag {
   478  			info.Version = version
   479  		}
   480  	}
   481  
   482  	return info, nil
   483  }
   484  
   485  func (r *gitRepo) Stat(rev string) (*RevInfo, error) {
   486  	if rev == "latest" {
   487  		return r.Latest()
   488  	}
   489  	type cached struct {
   490  		info *RevInfo
   491  		err  error
   492  	}
   493  	c := r.statCache.Do(rev, func() interface{} {
   494  		info, err := r.stat(rev)
   495  		return cached{info, err}
   496  	}).(cached)
   497  	return c.info, c.err
   498  }
   499  
   500  func (r *gitRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) {
   501  	// TODO: Could use git cat-file --batch.
   502  	info, err := r.Stat(rev) // download rev into local git repo
   503  	if err != nil {
   504  		return nil, err
   505  	}
   506  	out, err := Run(r.dir, "git", "cat-file", "blob", info.Name+":"+file)
   507  	if err != nil {
   508  		return nil, os.ErrNotExist
   509  	}
   510  	return out, nil
   511  }
   512  
   513  func (r *gitRepo) ReadFileRevs(revs []string, file string, maxSize int64) (map[string]*FileRev, error) {
   514  	// Create space to hold results.
   515  	files := make(map[string]*FileRev)
   516  	for _, rev := range revs {
   517  		f := &FileRev{Rev: rev}
   518  		files[rev] = f
   519  	}
   520  
   521  	// Collect locally-known revs.
   522  	need, err := r.readFileRevs(revs, file, files)
   523  	if err != nil {
   524  		return nil, err
   525  	}
   526  	if len(need) == 0 {
   527  		return files, nil
   528  	}
   529  
   530  	// Build list of known remote refs that might help.
   531  	var redo []string
   532  	r.refsOnce.Do(r.loadRefs)
   533  	if r.refsErr != nil {
   534  		return nil, r.refsErr
   535  	}
   536  	for _, tag := range need {
   537  		if r.refs["refs/tags/"+tag] != "" {
   538  			redo = append(redo, tag)
   539  		}
   540  	}
   541  	if len(redo) == 0 {
   542  		return files, nil
   543  	}
   544  
   545  	// Protect r.fetchLevel and the "fetch more and more" sequence.
   546  	// See stat method above.
   547  	unlock, err := r.mu.Lock()
   548  	if err != nil {
   549  		return nil, err
   550  	}
   551  	defer unlock()
   552  
   553  	if err := r.fetchRefsLocked(); err != nil {
   554  		return nil, err
   555  	}
   556  
   557  	if _, err := r.readFileRevs(redo, file, files); err != nil {
   558  		return nil, err
   559  	}
   560  
   561  	return files, nil
   562  }
   563  
   564  func (r *gitRepo) readFileRevs(tags []string, file string, fileMap map[string]*FileRev) (missing []string, err error) {
   565  	var stdin bytes.Buffer
   566  	for _, tag := range tags {
   567  		fmt.Fprintf(&stdin, "refs/tags/%s\n", tag)
   568  		fmt.Fprintf(&stdin, "refs/tags/%s:%s\n", tag, file)
   569  	}
   570  
   571  	data, err := RunWithStdin(r.dir, &stdin, "git", "cat-file", "--batch")
   572  	if err != nil {
   573  		return nil, err
   574  	}
   575  
   576  	next := func() (typ string, body []byte, ok bool) {
   577  		var line string
   578  		i := bytes.IndexByte(data, '\n')
   579  		if i < 0 {
   580  			return "", nil, false
   581  		}
   582  		line, data = string(bytes.TrimSpace(data[:i])), data[i+1:]
   583  		if strings.HasSuffix(line, " missing") {
   584  			return "missing", nil, true
   585  		}
   586  		f := strings.Fields(line)
   587  		if len(f) != 3 {
   588  			return "", nil, false
   589  		}
   590  		n, err := strconv.Atoi(f[2])
   591  		if err != nil || n > len(data) {
   592  			return "", nil, false
   593  		}
   594  		body, data = data[:n], data[n:]
   595  		if len(data) > 0 && data[0] == '\r' {
   596  			data = data[1:]
   597  		}
   598  		if len(data) > 0 && data[0] == '\n' {
   599  			data = data[1:]
   600  		}
   601  		return f[1], body, true
   602  	}
   603  
   604  	badGit := func() ([]string, error) {
   605  		return nil, fmt.Errorf("malformed output from git cat-file --batch")
   606  	}
   607  
   608  	for _, tag := range tags {
   609  		commitType, _, ok := next()
   610  		if !ok {
   611  			return badGit()
   612  		}
   613  		fileType, fileData, ok := next()
   614  		if !ok {
   615  			return badGit()
   616  		}
   617  		f := fileMap[tag]
   618  		f.Data = nil
   619  		f.Err = nil
   620  		switch commitType {
   621  		default:
   622  			f.Err = fmt.Errorf("unexpected non-commit type %q for rev %s", commitType, tag)
   623  
   624  		case "missing":
   625  			// Note: f.Err must not satisfy os.IsNotExist. That's reserved for the file not existing in a valid commit.
   626  			f.Err = fmt.Errorf("no such rev %s", tag)
   627  			missing = append(missing, tag)
   628  
   629  		case "tag", "commit":
   630  			switch fileType {
   631  			default:
   632  				f.Err = &os.PathError{Path: tag + ":" + file, Op: "read", Err: fmt.Errorf("unexpected non-blob type %q", fileType)}
   633  			case "missing":
   634  				f.Err = &os.PathError{Path: tag + ":" + file, Op: "read", Err: os.ErrNotExist}
   635  			case "blob":
   636  				f.Data = fileData
   637  			}
   638  		}
   639  	}
   640  	if len(bytes.TrimSpace(data)) != 0 {
   641  		return badGit()
   642  	}
   643  
   644  	return missing, nil
   645  }
   646  
   647  func (r *gitRepo) RecentTag(rev, prefix, major string) (tag string, err error) {
   648  	info, err := r.Stat(rev)
   649  	if err != nil {
   650  		return "", err
   651  	}
   652  	rev = info.Name // expand hash prefixes
   653  
   654  	// describe sets tag and err using 'git for-each-ref' and reports whether the
   655  	// result is definitive.
   656  	describe := func() (definitive bool) {
   657  		var out []byte
   658  		out, err = Run(r.dir, "git", "for-each-ref", "--format", "%(refname)", "refs/tags", "--merged", rev)
   659  		if err != nil {
   660  			return true
   661  		}
   662  
   663  		// prefixed tags aren't valid semver tags so compare without prefix, but only tags with correct prefix
   664  		var highest string
   665  		for _, line := range strings.Split(string(out), "\n") {
   666  			line = strings.TrimSpace(line)
   667  			// git do support lstrip in for-each-ref format, but it was added in v2.13.0. Stripping here
   668  			// instead gives support for git v2.7.0.
   669  			if !strings.HasPrefix(line, "refs/tags/") {
   670  				continue
   671  			}
   672  			line = line[len("refs/tags/"):]
   673  
   674  			if !strings.HasPrefix(line, prefix) {
   675  				continue
   676  			}
   677  
   678  			semtag := line[len(prefix):]
   679  			// Consider only tags that are valid and complete (not just major.minor prefixes).
   680  			// NOTE: Do not replace the call to semver.Compare with semver.Max.
   681  			// We want to return the actual tag, not a canonicalized version of it,
   682  			// and semver.Max currently canonicalizes (see golang.org/issue/32700).
   683  			if c := semver.Canonical(semtag); c != "" && strings.HasPrefix(semtag, c) && (major == "" || semver.Major(c) == major) && semver.Compare(semtag, highest) > 0 {
   684  				highest = semtag
   685  			}
   686  		}
   687  
   688  		if highest != "" {
   689  			tag = prefix + highest
   690  		}
   691  
   692  		return tag != "" && !AllHex(tag)
   693  	}
   694  
   695  	if describe() {
   696  		return tag, err
   697  	}
   698  
   699  	// Git didn't find a version tag preceding the requested rev.
   700  	// See whether any plausible tag exists.
   701  	tags, err := r.Tags(prefix + "v")
   702  	if err != nil {
   703  		return "", err
   704  	}
   705  	if len(tags) == 0 {
   706  		return "", nil
   707  	}
   708  
   709  	// There are plausible tags, but we don't know if rev is a descendent of any of them.
   710  	// Fetch the history to find out.
   711  
   712  	unlock, err := r.mu.Lock()
   713  	if err != nil {
   714  		return "", err
   715  	}
   716  	defer unlock()
   717  
   718  	if err := r.fetchRefsLocked(); err != nil {
   719  		return "", err
   720  	}
   721  
   722  	// If we've reached this point, we have all of the commits that are reachable
   723  	// from all heads and tags.
   724  	//
   725  	// The only refs we should be missing are those that are no longer reachable
   726  	// (or never were reachable) from any branch or tag, including the master
   727  	// branch, and we don't want to resolve them anyway (they're probably
   728  	// unreachable for a reason).
   729  	//
   730  	// Try one last time in case some other goroutine fetched rev while we were
   731  	// waiting on the lock.
   732  	describe()
   733  	return tag, err
   734  }
   735  
   736  func (r *gitRepo) DescendsFrom(rev, tag string) (bool, error) {
   737  	// The "--is-ancestor" flag was added to "git merge-base" in version 1.8.0, so
   738  	// this won't work with Git 1.7.1. According to golang.org/issue/28550, cmd/go
   739  	// already doesn't work with Git 1.7.1, so at least it's not a regression.
   740  	//
   741  	// git merge-base --is-ancestor exits with status 0 if rev is an ancestor, or
   742  	// 1 if not.
   743  	_, err := Run(r.dir, "git", "merge-base", "--is-ancestor", "--", tag, rev)
   744  
   745  	// Git reports "is an ancestor" with exit code 0 and "not an ancestor" with
   746  	// exit code 1.
   747  	// Unfortunately, if we've already fetched rev with a shallow history, git
   748  	// merge-base has been observed to report a false-negative, so don't stop yet
   749  	// even if the exit code is 1!
   750  	if err == nil {
   751  		return true, nil
   752  	}
   753  
   754  	// See whether the tag and rev even exist.
   755  	tags, err := r.Tags(tag)
   756  	if err != nil {
   757  		return false, err
   758  	}
   759  	if len(tags) == 0 {
   760  		return false, nil
   761  	}
   762  
   763  	// NOTE: r.stat is very careful not to fetch commits that we shouldn't know
   764  	// about, like rejected GitHub pull requests, so don't try to short-circuit
   765  	// that here.
   766  	if _, err = r.stat(rev); err != nil {
   767  		return false, err
   768  	}
   769  
   770  	// Now fetch history so that git can search for a path.
   771  	unlock, err := r.mu.Lock()
   772  	if err != nil {
   773  		return false, err
   774  	}
   775  	defer unlock()
   776  
   777  	if r.fetchLevel < fetchAll {
   778  		// Fetch the complete history for all refs and heads. It would be more
   779  		// efficient to only fetch the history from rev to tag, but that's much more
   780  		// complicated, and any kind of shallow fetch is fairly likely to trigger
   781  		// bugs in JGit servers and/or the go command anyway.
   782  		if err := r.fetchRefsLocked(); err != nil {
   783  			return false, err
   784  		}
   785  	}
   786  
   787  	_, err = Run(r.dir, "git", "merge-base", "--is-ancestor", "--", tag, rev)
   788  	if err == nil {
   789  		return true, nil
   790  	}
   791  	if ee, ok := err.(*RunError).Err.(*exec.ExitError); ok && ee.ExitCode() == 1 {
   792  		return false, nil
   793  	}
   794  	return false, err
   795  }
   796  
   797  func (r *gitRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error) {
   798  	// TODO: Use maxSize or drop it.
   799  	args := []string{}
   800  	if subdir != "" {
   801  		args = append(args, "--", subdir)
   802  	}
   803  	info, err := r.Stat(rev) // download rev into local git repo
   804  	if err != nil {
   805  		return nil, err
   806  	}
   807  
   808  	unlock, err := r.mu.Lock()
   809  	if err != nil {
   810  		return nil, err
   811  	}
   812  	defer unlock()
   813  
   814  	if err := ensureGitAttributes(r.dir); err != nil {
   815  		return nil, err
   816  	}
   817  
   818  	// Incredibly, git produces different archives depending on whether
   819  	// it is running on a Windows system or not, in an attempt to normalize
   820  	// text file line endings. Setting -c core.autocrlf=input means only
   821  	// translate files on the way into the repo, not on the way out (archive).
   822  	// The -c core.eol=lf should be unnecessary but set it anyway.
   823  	archive, err := Run(r.dir, "git", "-c", "core.autocrlf=input", "-c", "core.eol=lf", "archive", "--format=zip", "--prefix=prefix/", info.Name, args)
   824  	if err != nil {
   825  		if bytes.Contains(err.(*RunError).Stderr, []byte("did not match any files")) {
   826  			return nil, os.ErrNotExist
   827  		}
   828  		return nil, err
   829  	}
   830  
   831  	return ioutil.NopCloser(bytes.NewReader(archive)), nil
   832  }
   833  
   834  // ensureGitAttributes makes sure export-subst and export-ignore features are
   835  // disabled for this repo. This is intended to be run prior to running git
   836  // archive so that zip files are generated that produce consistent ziphashes
   837  // for a given revision, independent of variables such as git version and the
   838  // size of the repo.
   839  //
   840  // See: https://github.com/golang/go/issues/27153
   841  func ensureGitAttributes(repoDir string) (err error) {
   842  	const attr = "\n* -export-subst -export-ignore\n"
   843  
   844  	d := repoDir + "/info"
   845  	p := d + "/attributes"
   846  
   847  	if err := os.MkdirAll(d, 0755); err != nil {
   848  		return err
   849  	}
   850  
   851  	f, err := os.OpenFile(p, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0666)
   852  	if err != nil {
   853  		return err
   854  	}
   855  	defer func() {
   856  		closeErr := f.Close()
   857  		if closeErr != nil {
   858  			err = closeErr
   859  		}
   860  	}()
   861  
   862  	b, err := ioutil.ReadAll(f)
   863  	if err != nil {
   864  		return err
   865  	}
   866  	if !bytes.HasSuffix(b, []byte(attr)) {
   867  		_, err := f.WriteString(attr)
   868  		return err
   869  	}
   870  
   871  	return nil
   872  }
   873  

View as plain text