Black Lives Matter. Support the Equal Justice Initiative.

Source file src/cmd/go/internal/modfetch/codehost/codehost.go

Documentation: cmd/go/internal/modfetch/codehost

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package codehost defines the interface implemented by a code hosting source,
     6  // along with support code for use by implementations.
     7  package codehost
     8  
     9  import (
    10  	"bytes"
    11  	"crypto/sha256"
    12  	"fmt"
    13  	"io"
    14  	"io/ioutil"
    15  	"os"
    16  	"os/exec"
    17  	"path/filepath"
    18  	"strings"
    19  	"sync"
    20  	"time"
    21  
    22  	"cmd/go/internal/cfg"
    23  	"cmd/go/internal/lockedfile"
    24  	"cmd/go/internal/str"
    25  )
    26  
    27  // Downloaded size limits.
    28  const (
    29  	MaxGoMod   = 16 << 20  // maximum size of go.mod file
    30  	MaxLICENSE = 16 << 20  // maximum size of LICENSE file
    31  	MaxZipFile = 500 << 20 // maximum size of downloaded zip file
    32  )
    33  
    34  // A Repo represents a code hosting source.
    35  // Typical implementations include local version control repositories,
    36  // remote version control servers, and code hosting sites.
    37  // A Repo must be safe for simultaneous use by multiple goroutines.
    38  type Repo interface {
    39  	// List lists all tags with the given prefix.
    40  	Tags(prefix string) (tags []string, err error)
    41  
    42  	// Stat returns information about the revision rev.
    43  	// A revision can be any identifier known to the underlying service:
    44  	// commit hash, branch, tag, and so on.
    45  	Stat(rev string) (*RevInfo, error)
    46  
    47  	// Latest returns the latest revision on the default branch,
    48  	// whatever that means in the underlying implementation.
    49  	Latest() (*RevInfo, error)
    50  
    51  	// ReadFile reads the given file in the file tree corresponding to revision rev.
    52  	// It should refuse to read more than maxSize bytes.
    53  	//
    54  	// If the requested file does not exist it should return an error for which
    55  	// os.IsNotExist(err) returns true.
    56  	ReadFile(rev, file string, maxSize int64) (data []byte, err error)
    57  
    58  	// ReadFileRevs reads a single file at multiple versions.
    59  	// It should refuse to read more than maxSize bytes.
    60  	// The result is a map from each requested rev strings
    61  	// to the associated FileRev. The map must have a non-nil
    62  	// entry for every requested rev (unless ReadFileRevs returned an error).
    63  	// A file simply being missing or even corrupted in revs[i]
    64  	// should be reported only in files[revs[i]].Err, not in the error result
    65  	// from ReadFileRevs.
    66  	// The overall call should return an error (and no map) only
    67  	// in the case of a problem with obtaining the data, such as
    68  	// a network failure.
    69  	// Implementations may assume that revs only contain tags,
    70  	// not direct commit hashes.
    71  	ReadFileRevs(revs []string, file string, maxSize int64) (files map[string]*FileRev, err error)
    72  
    73  	// ReadZip downloads a zip file for the subdir subdirectory
    74  	// of the given revision to a new file in a given temporary directory.
    75  	// It should refuse to read more than maxSize bytes.
    76  	// It returns a ReadCloser for a streamed copy of the zip file.
    77  	// All files in the zip file are expected to be
    78  	// nested in a single top-level directory, whose name is not specified.
    79  	ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, err error)
    80  
    81  	// RecentTag returns the most recent tag on rev or one of its predecessors
    82  	// with the given prefix and major version.
    83  	// An empty major string matches any major version.
    84  	RecentTag(rev, prefix, major string) (tag string, err error)
    85  
    86  	// DescendsFrom reports whether rev or any of its ancestors has the given tag.
    87  	//
    88  	// DescendsFrom must return true for any tag returned by RecentTag for the
    89  	// same revision.
    90  	DescendsFrom(rev, tag string) (bool, error)
    91  }
    92  
    93  // A Rev describes a single revision in a source code repository.
    94  type RevInfo struct {
    95  	Name    string    // complete ID in underlying repository
    96  	Short   string    // shortened ID, for use in pseudo-version
    97  	Version string    // version used in lookup
    98  	Time    time.Time // commit time
    99  	Tags    []string  // known tags for commit
   100  }
   101  
   102  // A FileRev describes the result of reading a file at a given revision.
   103  type FileRev struct {
   104  	Rev  string // requested revision
   105  	Data []byte // file data
   106  	Err  error  // error if any; os.IsNotExist(Err)==true if rev exists but file does not exist in that rev
   107  }
   108  
   109  // UnknownRevisionError is an error equivalent to os.ErrNotExist, but for a
   110  // revision rather than a file.
   111  type UnknownRevisionError struct {
   112  	Rev string
   113  }
   114  
   115  func (e *UnknownRevisionError) Error() string {
   116  	return "unknown revision " + e.Rev
   117  }
   118  func (UnknownRevisionError) Is(err error) bool {
   119  	return err == os.ErrNotExist
   120  }
   121  
   122  // ErrNoCommits is an error equivalent to os.ErrNotExist indicating that a given
   123  // repository or module contains no commits.
   124  var ErrNoCommits error = noCommitsError{}
   125  
   126  type noCommitsError struct{}
   127  
   128  func (noCommitsError) Error() string {
   129  	return "no commits"
   130  }
   131  func (noCommitsError) Is(err error) bool {
   132  	return err == os.ErrNotExist
   133  }
   134  
   135  // AllHex reports whether the revision rev is entirely lower-case hexadecimal digits.
   136  func AllHex(rev string) bool {
   137  	for i := 0; i < len(rev); i++ {
   138  		c := rev[i]
   139  		if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' {
   140  			continue
   141  		}
   142  		return false
   143  	}
   144  	return true
   145  }
   146  
   147  // ShortenSHA1 shortens a SHA1 hash (40 hex digits) to the canonical length
   148  // used in pseudo-versions (12 hex digits).
   149  func ShortenSHA1(rev string) string {
   150  	if AllHex(rev) && len(rev) == 40 {
   151  		return rev[:12]
   152  	}
   153  	return rev
   154  }
   155  
   156  // WorkDir returns the name of the cached work directory to use for the
   157  // given repository type and name.
   158  func WorkDir(typ, name string) (dir, lockfile string, err error) {
   159  	if cfg.GOMODCACHE == "" {
   160  		return "", "", fmt.Errorf("neither GOPATH nor GOMODCACHE are set")
   161  	}
   162  
   163  	// We name the work directory for the SHA256 hash of the type and name.
   164  	// We intentionally avoid the actual name both because of possible
   165  	// conflicts with valid file system paths and because we want to ensure
   166  	// that one checkout is never nested inside another. That nesting has
   167  	// led to security problems in the past.
   168  	if strings.Contains(typ, ":") {
   169  		return "", "", fmt.Errorf("codehost.WorkDir: type cannot contain colon")
   170  	}
   171  	key := typ + ":" + name
   172  	dir = filepath.Join(cfg.GOMODCACHE, "cache/vcs", fmt.Sprintf("%x", sha256.Sum256([]byte(key))))
   173  
   174  	if cfg.BuildX {
   175  		fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", filepath.Dir(dir), typ, name)
   176  	}
   177  	if err := os.MkdirAll(filepath.Dir(dir), 0777); err != nil {
   178  		return "", "", err
   179  	}
   180  
   181  	lockfile = dir + ".lock"
   182  	if cfg.BuildX {
   183  		fmt.Fprintf(os.Stderr, "# lock %s", lockfile)
   184  	}
   185  
   186  	unlock, err := lockedfile.MutexAt(lockfile).Lock()
   187  	if err != nil {
   188  		return "", "", fmt.Errorf("codehost.WorkDir: can't find or create lock file: %v", err)
   189  	}
   190  	defer unlock()
   191  
   192  	data, err := ioutil.ReadFile(dir + ".info")
   193  	info, err2 := os.Stat(dir)
   194  	if err == nil && err2 == nil && info.IsDir() {
   195  		// Info file and directory both already exist: reuse.
   196  		have := strings.TrimSuffix(string(data), "\n")
   197  		if have != key {
   198  			return "", "", fmt.Errorf("%s exists with wrong content (have %q want %q)", dir+".info", have, key)
   199  		}
   200  		if cfg.BuildX {
   201  			fmt.Fprintf(os.Stderr, "# %s for %s %s\n", dir, typ, name)
   202  		}
   203  		return dir, lockfile, nil
   204  	}
   205  
   206  	// Info file or directory missing. Start from scratch.
   207  	if cfg.BuildX {
   208  		fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", dir, typ, name)
   209  	}
   210  	os.RemoveAll(dir)
   211  	if err := os.MkdirAll(dir, 0777); err != nil {
   212  		return "", "", err
   213  	}
   214  	if err := ioutil.WriteFile(dir+".info", []byte(key), 0666); err != nil {
   215  		os.RemoveAll(dir)
   216  		return "", "", err
   217  	}
   218  	return dir, lockfile, nil
   219  }
   220  
   221  type RunError struct {
   222  	Cmd      string
   223  	Err      error
   224  	Stderr   []byte
   225  	HelpText string
   226  }
   227  
   228  func (e *RunError) Error() string {
   229  	text := e.Cmd + ": " + e.Err.Error()
   230  	stderr := bytes.TrimRight(e.Stderr, "\n")
   231  	if len(stderr) > 0 {
   232  		text += ":\n\t" + strings.ReplaceAll(string(stderr), "\n", "\n\t")
   233  	}
   234  	if len(e.HelpText) > 0 {
   235  		text += "\n" + e.HelpText
   236  	}
   237  	return text
   238  }
   239  
   240  var dirLock sync.Map
   241  
   242  // Run runs the command line in the given directory
   243  // (an empty dir means the current directory).
   244  // It returns the standard output and, for a non-zero exit,
   245  // a *RunError indicating the command, exit status, and standard error.
   246  // Standard error is unavailable for commands that exit successfully.
   247  func Run(dir string, cmdline ...interface{}) ([]byte, error) {
   248  	return RunWithStdin(dir, nil, cmdline...)
   249  }
   250  
   251  // bashQuoter escapes characters that have special meaning in double-quoted strings in the bash shell.
   252  // See https://www.gnu.org/software/bash/manual/html_node/Double-Quotes.html.
   253  var bashQuoter = strings.NewReplacer(`"`, `\"`, `$`, `\$`, "`", "\\`", `\`, `\\`)
   254  
   255  func RunWithStdin(dir string, stdin io.Reader, cmdline ...interface{}) ([]byte, error) {
   256  	if dir != "" {
   257  		muIface, ok := dirLock.Load(dir)
   258  		if !ok {
   259  			muIface, _ = dirLock.LoadOrStore(dir, new(sync.Mutex))
   260  		}
   261  		mu := muIface.(*sync.Mutex)
   262  		mu.Lock()
   263  		defer mu.Unlock()
   264  	}
   265  
   266  	cmd := str.StringList(cmdline...)
   267  	if cfg.BuildX {
   268  		text := new(strings.Builder)
   269  		if dir != "" {
   270  			text.WriteString("cd ")
   271  			text.WriteString(dir)
   272  			text.WriteString("; ")
   273  		}
   274  		for i, arg := range cmd {
   275  			if i > 0 {
   276  				text.WriteByte(' ')
   277  			}
   278  			switch {
   279  			case strings.ContainsAny(arg, "'"):
   280  				// Quote args that could be mistaken for quoted args.
   281  				text.WriteByte('"')
   282  				text.WriteString(bashQuoter.Replace(arg))
   283  				text.WriteByte('"')
   284  			case strings.ContainsAny(arg, "$`\\*?[\"\t\n\v\f\r \u0085\u00a0"):
   285  				// Quote args that contain special characters, glob patterns, or spaces.
   286  				text.WriteByte('\'')
   287  				text.WriteString(arg)
   288  				text.WriteByte('\'')
   289  			default:
   290  				text.WriteString(arg)
   291  			}
   292  		}
   293  		fmt.Fprintf(os.Stderr, "%s\n", text)
   294  		start := time.Now()
   295  		defer func() {
   296  			fmt.Fprintf(os.Stderr, "%.3fs # %s\n", time.Since(start).Seconds(), text)
   297  		}()
   298  	}
   299  	// TODO: Impose limits on command output size.
   300  	// TODO: Set environment to get English error messages.
   301  	var stderr bytes.Buffer
   302  	var stdout bytes.Buffer
   303  	c := exec.Command(cmd[0], cmd[1:]...)
   304  	c.Dir = dir
   305  	c.Stdin = stdin
   306  	c.Stderr = &stderr
   307  	c.Stdout = &stdout
   308  	err := c.Run()
   309  	if err != nil {
   310  		err = &RunError{Cmd: strings.Join(cmd, " ") + " in " + dir, Stderr: stderr.Bytes(), Err: err}
   311  	}
   312  	return stdout.Bytes(), err
   313  }
   314  

View as plain text