Source file src/cmd/go/internal/get/get.go

Documentation: cmd/go/internal/get

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package get implements the ``go get'' command.
     6  package get
     7  
     8  import (
     9  	"fmt"
    10  	"go/build"
    11  	"os"
    12  	"path/filepath"
    13  	"runtime"
    14  	"strings"
    15  
    16  	"cmd/go/internal/base"
    17  	"cmd/go/internal/cfg"
    18  	"cmd/go/internal/load"
    19  	"cmd/go/internal/search"
    20  	"cmd/go/internal/str"
    21  	"cmd/go/internal/web"
    22  	"cmd/go/internal/work"
    23  )
    24  
    25  var CmdGet = &base.Command{
    26  	UsageLine: "go get [-d] [-f] [-t] [-u] [-v] [-fix] [-insecure] [build flags] [packages]",
    27  	Short:     "download and install packages and dependencies",
    28  	Long: `
    29  Get downloads the packages named by the import paths, along with their
    30  dependencies. It then installs the named packages, like 'go install'.
    31  
    32  The -d flag instructs get to stop after downloading the packages; that is,
    33  it instructs get not to install the packages.
    34  
    35  The -f flag, valid only when -u is set, forces get -u not to verify that
    36  each package has been checked out from the source control repository
    37  implied by its import path. This can be useful if the source is a local fork
    38  of the original.
    39  
    40  The -fix flag instructs get to run the fix tool on the downloaded packages
    41  before resolving dependencies or building the code.
    42  
    43  The -insecure flag permits fetching from repositories and resolving
    44  custom domains using insecure schemes such as HTTP. Use with caution.
    45  
    46  The -t flag instructs get to also download the packages required to build
    47  the tests for the specified packages.
    48  
    49  The -u flag instructs get to use the network to update the named packages
    50  and their dependencies. By default, get uses the network to check out
    51  missing packages but does not use it to look for updates to existing packages.
    52  
    53  The -v flag enables verbose progress and debug output.
    54  
    55  Get also accepts build flags to control the installation. See 'go help build'.
    56  
    57  When checking out a new package, get creates the target directory
    58  GOPATH/src/<import-path>. If the GOPATH contains multiple entries,
    59  get uses the first one. For more details see: 'go help gopath'.
    60  
    61  When checking out or updating a package, get looks for a branch or tag
    62  that matches the locally installed version of Go. The most important
    63  rule is that if the local installation is running version "go1", get
    64  searches for a branch or tag named "go1". If no such version exists
    65  it retrieves the default branch of the package.
    66  
    67  When go get checks out or updates a Git repository,
    68  it also updates any git submodules referenced by the repository.
    69  
    70  Get never checks out or updates code stored in vendor directories.
    71  
    72  For more about specifying packages, see 'go help packages'.
    73  
    74  For more about how 'go get' finds source code to
    75  download, see 'go help importpath'.
    76  
    77  This text describes the behavior of get when using GOPATH
    78  to manage source code and dependencies.
    79  If instead the go command is running in module-aware mode,
    80  the details of get's flags and effects change, as does 'go help get'.
    81  See 'go help modules' and 'go help module-get'.
    82  
    83  See also: go build, go install, go clean.
    84  	`,
    85  }
    86  
    87  var HelpGopathGet = &base.Command{
    88  	UsageLine: "gopath-get",
    89  	Short:     "legacy GOPATH go get",
    90  	Long: `
    91  The 'go get' command changes behavior depending on whether the
    92  go command is running in module-aware mode or legacy GOPATH mode.
    93  This help text, accessible as 'go help gopath-get' even in module-aware mode,
    94  describes 'go get' as it operates in legacy GOPATH mode.
    95  
    96  Usage: ` + CmdGet.UsageLine + `
    97  ` + CmdGet.Long,
    98  }
    99  
   100  var (
   101  	getD   = CmdGet.Flag.Bool("d", false, "")
   102  	getF   = CmdGet.Flag.Bool("f", false, "")
   103  	getT   = CmdGet.Flag.Bool("t", false, "")
   104  	getU   = CmdGet.Flag.Bool("u", false, "")
   105  	getFix = CmdGet.Flag.Bool("fix", false, "")
   106  
   107  	Insecure bool
   108  )
   109  
   110  func init() {
   111  	work.AddBuildFlags(CmdGet)
   112  	CmdGet.Run = runGet // break init loop
   113  	CmdGet.Flag.BoolVar(&Insecure, "insecure", Insecure, "")
   114  }
   115  
   116  func runGet(cmd *base.Command, args []string) {
   117  	if cfg.ModulesEnabled {
   118  		// Should not happen: main.go should install the separate module-enabled get code.
   119  		base.Fatalf("go get: modules not implemented")
   120  	}
   121  
   122  	work.BuildInit()
   123  
   124  	if *getF && !*getU {
   125  		base.Fatalf("go get: cannot use -f flag without -u")
   126  	}
   127  
   128  	// Disable any prompting for passwords by Git.
   129  	// Only has an effect for 2.3.0 or later, but avoiding
   130  	// the prompt in earlier versions is just too hard.
   131  	// If user has explicitly set GIT_TERMINAL_PROMPT=1, keep
   132  	// prompting.
   133  	// See golang.org/issue/9341 and golang.org/issue/12706.
   134  	if os.Getenv("GIT_TERMINAL_PROMPT") == "" {
   135  		os.Setenv("GIT_TERMINAL_PROMPT", "0")
   136  	}
   137  
   138  	// Disable any ssh connection pooling by Git.
   139  	// If a Git subprocess forks a child into the background to cache a new connection,
   140  	// that child keeps stdout/stderr open. After the Git subprocess exits,
   141  	// os /exec expects to be able to read from the stdout/stderr pipe
   142  	// until EOF to get all the data that the Git subprocess wrote before exiting.
   143  	// The EOF doesn't come until the child exits too, because the child
   144  	// is holding the write end of the pipe.
   145  	// This is unfortunate, but it has come up at least twice
   146  	// (see golang.org/issue/13453 and golang.org/issue/16104)
   147  	// and confuses users when it does.
   148  	// If the user has explicitly set GIT_SSH or GIT_SSH_COMMAND,
   149  	// assume they know what they are doing and don't step on it.
   150  	// But default to turning off ControlMaster.
   151  	if os.Getenv("GIT_SSH") == "" && os.Getenv("GIT_SSH_COMMAND") == "" {
   152  		os.Setenv("GIT_SSH_COMMAND", "ssh -o ControlMaster=no")
   153  	}
   154  
   155  	// Phase 1. Download/update.
   156  	var stk load.ImportStack
   157  	mode := 0
   158  	if *getT {
   159  		mode |= load.GetTestDeps
   160  	}
   161  	for _, pkg := range downloadPaths(args) {
   162  		download(pkg, nil, &stk, mode)
   163  	}
   164  	base.ExitIfErrors()
   165  
   166  	// Phase 2. Rescan packages and re-evaluate args list.
   167  
   168  	// Code we downloaded and all code that depends on it
   169  	// needs to be evicted from the package cache so that
   170  	// the information will be recomputed. Instead of keeping
   171  	// track of the reverse dependency information, evict
   172  	// everything.
   173  	load.ClearPackageCache()
   174  
   175  	pkgs := load.PackagesForBuild(args)
   176  
   177  	// Phase 3. Install.
   178  	if *getD {
   179  		// Download only.
   180  		// Check delayed until now so that importPaths
   181  		// and packagesForBuild have a chance to print errors.
   182  		return
   183  	}
   184  
   185  	work.InstallPackages(args, pkgs)
   186  }
   187  
   188  // downloadPaths prepares the list of paths to pass to download.
   189  // It expands ... patterns that can be expanded. If there is no match
   190  // for a particular pattern, downloadPaths leaves it in the result list,
   191  // in the hope that we can figure out the repository from the
   192  // initial ...-free prefix.
   193  func downloadPaths(patterns []string) []string {
   194  	for _, arg := range patterns {
   195  		if strings.Contains(arg, "@") {
   196  			base.Fatalf("go: cannot use path@version syntax in GOPATH mode")
   197  		}
   198  	}
   199  	var pkgs []string
   200  	for _, m := range search.ImportPathsQuiet(patterns) {
   201  		if len(m.Pkgs) == 0 && strings.Contains(m.Pattern, "...") {
   202  			pkgs = append(pkgs, m.Pattern)
   203  		} else {
   204  			pkgs = append(pkgs, m.Pkgs...)
   205  		}
   206  	}
   207  	return pkgs
   208  }
   209  
   210  // downloadCache records the import paths we have already
   211  // considered during the download, to avoid duplicate work when
   212  // there is more than one dependency sequence leading to
   213  // a particular package.
   214  var downloadCache = map[string]bool{}
   215  
   216  // downloadRootCache records the version control repository
   217  // root directories we have already considered during the download.
   218  // For example, all the packages in the github.com/google/codesearch repo
   219  // share the same root (the directory for that path), and we only need
   220  // to run the hg commands to consider each repository once.
   221  var downloadRootCache = map[string]bool{}
   222  
   223  // download runs the download half of the get command
   224  // for the package or pattern named by the argument.
   225  func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) {
   226  	if mode&load.ResolveImport != 0 {
   227  		// Caller is responsible for expanding vendor paths.
   228  		panic("internal error: download mode has useVendor set")
   229  	}
   230  	load1 := func(path string, mode int) *load.Package {
   231  		if parent == nil {
   232  			mode := 0 // don't do module or vendor resolution
   233  			return load.LoadImport(path, base.Cwd, nil, stk, nil, mode)
   234  		}
   235  		return load.LoadImport(path, parent.Dir, parent, stk, nil, mode|load.ResolveModule)
   236  	}
   237  
   238  	p := load1(arg, mode)
   239  	if p.Error != nil && p.Error.Hard {
   240  		base.Errorf("%s", p.Error)
   241  		return
   242  	}
   243  
   244  	// loadPackage inferred the canonical ImportPath from arg.
   245  	// Use that in the following to prevent hysteresis effects
   246  	// in e.g. downloadCache and packageCache.
   247  	// This allows invocations such as:
   248  	//   mkdir -p $GOPATH/src/github.com/user
   249  	//   cd $GOPATH/src/github.com/user
   250  	//   go get ./foo
   251  	// see: golang.org/issue/9767
   252  	arg = p.ImportPath
   253  
   254  	// There's nothing to do if this is a package in the standard library.
   255  	if p.Standard {
   256  		return
   257  	}
   258  
   259  	// Only process each package once.
   260  	// (Unless we're fetching test dependencies for this package,
   261  	// in which case we want to process it again.)
   262  	if downloadCache[arg] && mode&load.GetTestDeps == 0 {
   263  		return
   264  	}
   265  	downloadCache[arg] = true
   266  
   267  	pkgs := []*load.Package{p}
   268  	wildcardOkay := len(*stk) == 0
   269  	isWildcard := false
   270  
   271  	// Download if the package is missing, or update if we're using -u.
   272  	if p.Dir == "" || *getU {
   273  		// The actual download.
   274  		stk.Push(arg)
   275  		err := downloadPackage(p)
   276  		if err != nil {
   277  			base.Errorf("%s", &load.PackageError{ImportStack: stk.Copy(), Err: err.Error()})
   278  			stk.Pop()
   279  			return
   280  		}
   281  		stk.Pop()
   282  
   283  		args := []string{arg}
   284  		// If the argument has a wildcard in it, re-evaluate the wildcard.
   285  		// We delay this until after reloadPackage so that the old entry
   286  		// for p has been replaced in the package cache.
   287  		if wildcardOkay && strings.Contains(arg, "...") {
   288  			if build.IsLocalImport(arg) {
   289  				args = search.MatchPackagesInFS(arg).Pkgs
   290  			} else {
   291  				args = search.MatchPackages(arg).Pkgs
   292  			}
   293  			isWildcard = true
   294  		}
   295  
   296  		// Clear all relevant package cache entries before
   297  		// doing any new loads.
   298  		load.ClearPackageCachePartial(args)
   299  
   300  		pkgs = pkgs[:0]
   301  		for _, arg := range args {
   302  			// Note: load calls loadPackage or loadImport,
   303  			// which push arg onto stk already.
   304  			// Do not push here too, or else stk will say arg imports arg.
   305  			p := load1(arg, mode)
   306  			if p.Error != nil {
   307  				base.Errorf("%s", p.Error)
   308  				continue
   309  			}
   310  			pkgs = append(pkgs, p)
   311  		}
   312  	}
   313  
   314  	// Process package, which might now be multiple packages
   315  	// due to wildcard expansion.
   316  	for _, p := range pkgs {
   317  		if *getFix {
   318  			files := base.RelPaths(p.InternalAllGoFiles())
   319  			base.Run(cfg.BuildToolexec, str.StringList(base.Tool("fix"), files))
   320  
   321  			// The imports might have changed, so reload again.
   322  			p = load.ReloadPackageNoFlags(arg, stk)
   323  			if p.Error != nil {
   324  				base.Errorf("%s", p.Error)
   325  				return
   326  			}
   327  		}
   328  
   329  		if isWildcard {
   330  			// Report both the real package and the
   331  			// wildcard in any error message.
   332  			stk.Push(p.ImportPath)
   333  		}
   334  
   335  		// Process dependencies, now that we know what they are.
   336  		imports := p.Imports
   337  		if mode&load.GetTestDeps != 0 {
   338  			// Process test dependencies when -t is specified.
   339  			// (But don't get test dependencies for test dependencies:
   340  			// we always pass mode 0 to the recursive calls below.)
   341  			imports = str.StringList(imports, p.TestImports, p.XTestImports)
   342  		}
   343  		for i, path := range imports {
   344  			if path == "C" {
   345  				continue
   346  			}
   347  			// Fail fast on import naming full vendor path.
   348  			// Otherwise expand path as needed for test imports.
   349  			// Note that p.Imports can have additional entries beyond p.Internal.Build.Imports.
   350  			orig := path
   351  			if i < len(p.Internal.Build.Imports) {
   352  				orig = p.Internal.Build.Imports[i]
   353  			}
   354  			if j, ok := load.FindVendor(orig); ok {
   355  				stk.Push(path)
   356  				err := &load.PackageError{
   357  					ImportStack: stk.Copy(),
   358  					Err:         "must be imported as " + path[j+len("vendor/"):],
   359  				}
   360  				stk.Pop()
   361  				base.Errorf("%s", err)
   362  				continue
   363  			}
   364  			// If this is a test import, apply module and vendor lookup now.
   365  			// We cannot pass ResolveImport to download, because
   366  			// download does caching based on the value of path,
   367  			// so it must be the fully qualified path already.
   368  			if i >= len(p.Imports) {
   369  				path = load.ResolveImportPath(p, path)
   370  			}
   371  			download(path, p, stk, 0)
   372  		}
   373  
   374  		if isWildcard {
   375  			stk.Pop()
   376  		}
   377  	}
   378  }
   379  
   380  // downloadPackage runs the create or download command
   381  // to make the first copy of or update a copy of the given package.
   382  func downloadPackage(p *load.Package) error {
   383  	var (
   384  		vcs            *vcsCmd
   385  		repo, rootPath string
   386  		err            error
   387  		blindRepo      bool // set if the repo has unusual configuration
   388  	)
   389  
   390  	security := web.SecureOnly
   391  	if Insecure {
   392  		security = web.Insecure
   393  	}
   394  
   395  	// p can be either a real package, or a pseudo-package whose “import path” is
   396  	// actually a wildcard pattern.
   397  	// Trim the path at the element containing the first wildcard,
   398  	// and hope that it applies to the wildcarded parts too.
   399  	// This makes 'go get rsc.io/pdf/...' work in a fresh GOPATH.
   400  	importPrefix := p.ImportPath
   401  	if i := strings.Index(importPrefix, "..."); i >= 0 {
   402  		slash := strings.LastIndexByte(importPrefix[:i], '/')
   403  		if slash < 0 {
   404  			return fmt.Errorf("cannot expand ... in %q", p.ImportPath)
   405  		}
   406  		importPrefix = importPrefix[:slash]
   407  	}
   408  	if err := CheckImportPath(importPrefix); err != nil {
   409  		return fmt.Errorf("%s: invalid import path: %v", p.ImportPath, err)
   410  	}
   411  
   412  	if p.Internal.Build.SrcRoot != "" {
   413  		// Directory exists. Look for checkout along path to src.
   414  		vcs, rootPath, err = vcsFromDir(p.Dir, p.Internal.Build.SrcRoot)
   415  		if err != nil {
   416  			return err
   417  		}
   418  		repo = "<local>" // should be unused; make distinctive
   419  
   420  		// Double-check where it came from.
   421  		if *getU && vcs.remoteRepo != nil {
   422  			dir := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath))
   423  			remote, err := vcs.remoteRepo(vcs, dir)
   424  			if err != nil {
   425  				// Proceed anyway. The package is present; we likely just don't understand
   426  				// the repo configuration (e.g. unusual remote protocol).
   427  				blindRepo = true
   428  			}
   429  			repo = remote
   430  			if !*getF && err == nil {
   431  				if rr, err := RepoRootForImportPath(importPrefix, IgnoreMod, security); err == nil {
   432  					repo := rr.Repo
   433  					if rr.vcs.resolveRepo != nil {
   434  						resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo)
   435  						if err == nil {
   436  							repo = resolved
   437  						}
   438  					}
   439  					if remote != repo && rr.IsCustom {
   440  						return fmt.Errorf("%s is a custom import path for %s, but %s is checked out from %s", rr.Root, repo, dir, remote)
   441  					}
   442  				}
   443  			}
   444  		}
   445  	} else {
   446  		// Analyze the import path to determine the version control system,
   447  		// repository, and the import path for the root of the repository.
   448  		rr, err := RepoRootForImportPath(importPrefix, IgnoreMod, security)
   449  		if err != nil {
   450  			return err
   451  		}
   452  		vcs, repo, rootPath = rr.vcs, rr.Repo, rr.Root
   453  	}
   454  	if !blindRepo && !vcs.isSecure(repo) && !Insecure {
   455  		return fmt.Errorf("cannot download, %v uses insecure protocol", repo)
   456  	}
   457  
   458  	if p.Internal.Build.SrcRoot == "" {
   459  		// Package not found. Put in first directory of $GOPATH.
   460  		list := filepath.SplitList(cfg.BuildContext.GOPATH)
   461  		if len(list) == 0 {
   462  			return fmt.Errorf("cannot download, $GOPATH not set. For more details see: 'go help gopath'")
   463  		}
   464  		// Guard against people setting GOPATH=$GOROOT.
   465  		if filepath.Clean(list[0]) == filepath.Clean(cfg.GOROOT) {
   466  			return fmt.Errorf("cannot download, $GOPATH must not be set to $GOROOT. For more details see: 'go help gopath'")
   467  		}
   468  		if _, err := os.Stat(filepath.Join(list[0], "src/cmd/go/alldocs.go")); err == nil {
   469  			return fmt.Errorf("cannot download, %s is a GOROOT, not a GOPATH. For more details see: 'go help gopath'", list[0])
   470  		}
   471  		p.Internal.Build.Root = list[0]
   472  		p.Internal.Build.SrcRoot = filepath.Join(list[0], "src")
   473  		p.Internal.Build.PkgRoot = filepath.Join(list[0], "pkg")
   474  	}
   475  	root := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath))
   476  
   477  	if err := checkNestedVCS(vcs, root, p.Internal.Build.SrcRoot); err != nil {
   478  		return err
   479  	}
   480  
   481  	// If we've considered this repository already, don't do it again.
   482  	if downloadRootCache[root] {
   483  		return nil
   484  	}
   485  	downloadRootCache[root] = true
   486  
   487  	if cfg.BuildV {
   488  		fmt.Fprintf(os.Stderr, "%s (download)\n", rootPath)
   489  	}
   490  
   491  	// Check that this is an appropriate place for the repo to be checked out.
   492  	// The target directory must either not exist or have a repo checked out already.
   493  	meta := filepath.Join(root, "."+vcs.cmd)
   494  	if _, err := os.Stat(meta); err != nil {
   495  		// Metadata file or directory does not exist. Prepare to checkout new copy.
   496  		// Some version control tools require the target directory not to exist.
   497  		// We require that too, just to avoid stepping on existing work.
   498  		if _, err := os.Stat(root); err == nil {
   499  			return fmt.Errorf("%s exists but %s does not - stale checkout?", root, meta)
   500  		}
   501  
   502  		_, err := os.Stat(p.Internal.Build.Root)
   503  		gopathExisted := err == nil
   504  
   505  		// Some version control tools require the parent of the target to exist.
   506  		parent, _ := filepath.Split(root)
   507  		if err = os.MkdirAll(parent, 0777); err != nil {
   508  			return err
   509  		}
   510  		if cfg.BuildV && !gopathExisted && p.Internal.Build.Root == cfg.BuildContext.GOPATH {
   511  			fmt.Fprintf(os.Stderr, "created GOPATH=%s; see 'go help gopath'\n", p.Internal.Build.Root)
   512  		}
   513  
   514  		if err = vcs.create(root, repo); err != nil {
   515  			return err
   516  		}
   517  	} else {
   518  		// Metadata directory does exist; download incremental updates.
   519  		if err = vcs.download(root); err != nil {
   520  			return err
   521  		}
   522  	}
   523  
   524  	if cfg.BuildN {
   525  		// Do not show tag sync in -n; it's noise more than anything,
   526  		// and since we're not running commands, no tag will be found.
   527  		// But avoid printing nothing.
   528  		fmt.Fprintf(os.Stderr, "# cd %s; %s sync/update\n", root, vcs.cmd)
   529  		return nil
   530  	}
   531  
   532  	// Select and sync to appropriate version of the repository.
   533  	tags, err := vcs.tags(root)
   534  	if err != nil {
   535  		return err
   536  	}
   537  	vers := runtime.Version()
   538  	if i := strings.Index(vers, " "); i >= 0 {
   539  		vers = vers[:i]
   540  	}
   541  	if err := vcs.tagSync(root, selectTag(vers, tags)); err != nil {
   542  		return err
   543  	}
   544  
   545  	return nil
   546  }
   547  
   548  // selectTag returns the closest matching tag for a given version.
   549  // Closest means the latest one that is not after the current release.
   550  // Version "goX" (or "goX.Y" or "goX.Y.Z") matches tags of the same form.
   551  // Version "release.rN" matches tags of the form "go.rN" (N being a floating-point number).
   552  // Version "weekly.YYYY-MM-DD" matches tags like "go.weekly.YYYY-MM-DD".
   553  //
   554  // NOTE(rsc): Eventually we will need to decide on some logic here.
   555  // For now, there is only "go1". This matches the docs in go help get.
   556  func selectTag(goVersion string, tags []string) (match string) {
   557  	for _, t := range tags {
   558  		if t == "go1" {
   559  			return "go1"
   560  		}
   561  	}
   562  	return ""
   563  }
   564  

View as plain text