...
Run Format

Source file src/cmd/go/internal/get/get.go

Documentation: cmd/go/internal/get

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package get implements the ``go get'' command.
     6  package get
     7  
     8  import (
     9  	"fmt"
    10  	"go/build"
    11  	"os"
    12  	"path/filepath"
    13  	"runtime"
    14  	"strings"
    15  
    16  	"cmd/go/internal/base"
    17  	"cmd/go/internal/cfg"
    18  	"cmd/go/internal/load"
    19  	"cmd/go/internal/search"
    20  	"cmd/go/internal/str"
    21  	"cmd/go/internal/web"
    22  	"cmd/go/internal/work"
    23  )
    24  
    25  var CmdGet = &base.Command{
    26  	UsageLine: "go get [-d] [-f] [-t] [-u] [-v] [-fix] [-insecure] [build flags] [packages]",
    27  	Short:     "download and install packages and dependencies",
    28  	Long: `
    29  Get downloads the packages named by the import paths, along with their
    30  dependencies. It then installs the named packages, like 'go install'.
    31  
    32  The -d flag instructs get to stop after downloading the packages; that is,
    33  it instructs get not to install the packages.
    34  
    35  The -f flag, valid only when -u is set, forces get -u not to verify that
    36  each package has been checked out from the source control repository
    37  implied by its import path. This can be useful if the source is a local fork
    38  of the original.
    39  
    40  The -fix flag instructs get to run the fix tool on the downloaded packages
    41  before resolving dependencies or building the code.
    42  
    43  The -insecure flag permits fetching from repositories and resolving
    44  custom domains using insecure schemes such as HTTP. Use with caution.
    45  
    46  The -t flag instructs get to also download the packages required to build
    47  the tests for the specified packages.
    48  
    49  The -u flag instructs get to use the network to update the named packages
    50  and their dependencies. By default, get uses the network to check out
    51  missing packages but does not use it to look for updates to existing packages.
    52  
    53  The -v flag enables verbose progress and debug output.
    54  
    55  Get also accepts build flags to control the installation. See 'go help build'.
    56  
    57  When checking out a new package, get creates the target directory
    58  GOPATH/src/<import-path>. If the GOPATH contains multiple entries,
    59  get uses the first one. For more details see: 'go help gopath'.
    60  
    61  When checking out or updating a package, get looks for a branch or tag
    62  that matches the locally installed version of Go. The most important
    63  rule is that if the local installation is running version "go1", get
    64  searches for a branch or tag named "go1". If no such version exists
    65  it retrieves the default branch of the package.
    66  
    67  When go get checks out or updates a Git repository,
    68  it also updates any git submodules referenced by the repository.
    69  
    70  Get never checks out or updates code stored in vendor directories.
    71  
    72  For more about specifying packages, see 'go help packages'.
    73  
    74  For more about how 'go get' finds source code to
    75  download, see 'go help importpath'.
    76  
    77  This text describes the behavior of get when using GOPATH
    78  to manage source code and dependencies.
    79  If instead the go command is running in module-aware mode,
    80  the details of get's flags and effects change, as does 'go help get'.
    81  See 'go help modules' and 'go help module-get'.
    82  
    83  See also: go build, go install, go clean.
    84  	`,
    85  }
    86  
    87  var HelpGopathGet = &base.Command{
    88  	UsageLine: "gopath-get",
    89  	Short:     "legacy GOPATH go get",
    90  	Long: `
    91  The 'go get' command changes behavior depending on whether the
    92  go command is running in module-aware mode or legacy GOPATH mode.
    93  This help text, accessible as 'go help gopath-get' even in module-aware mode,
    94  describes 'go get' as it operates in legacy GOPATH mode.
    95  
    96  Usage: ` + CmdGet.UsageLine + `
    97  ` + CmdGet.Long,
    98  }
    99  
   100  var (
   101  	getD   = CmdGet.Flag.Bool("d", false, "")
   102  	getF   = CmdGet.Flag.Bool("f", false, "")
   103  	getT   = CmdGet.Flag.Bool("t", false, "")
   104  	getU   = CmdGet.Flag.Bool("u", false, "")
   105  	getFix = CmdGet.Flag.Bool("fix", false, "")
   106  
   107  	Insecure bool
   108  )
   109  
   110  func init() {
   111  	work.AddBuildFlags(CmdGet)
   112  	CmdGet.Run = runGet // break init loop
   113  	CmdGet.Flag.BoolVar(&Insecure, "insecure", Insecure, "")
   114  }
   115  
   116  func runGet(cmd *base.Command, args []string) {
   117  	if cfg.ModulesEnabled {
   118  		// Should not happen: main.go should install the separate module-enabled get code.
   119  		base.Fatalf("go get: modules not implemented")
   120  	}
   121  	if cfg.GoModInGOPATH != "" {
   122  		// Warn about not using modules with GO111MODULE=auto when go.mod exists.
   123  		// To silence the warning, users can set GO111MODULE=off.
   124  		fmt.Fprintf(os.Stderr, "go get: warning: modules disabled by GO111MODULE=auto in GOPATH/src;\n\tignoring %s;\n\tsee 'go help modules'\n", base.ShortPath(cfg.GoModInGOPATH))
   125  	}
   126  
   127  	work.BuildInit()
   128  
   129  	if *getF && !*getU {
   130  		base.Fatalf("go get: cannot use -f flag without -u")
   131  	}
   132  
   133  	// Disable any prompting for passwords by Git.
   134  	// Only has an effect for 2.3.0 or later, but avoiding
   135  	// the prompt in earlier versions is just too hard.
   136  	// If user has explicitly set GIT_TERMINAL_PROMPT=1, keep
   137  	// prompting.
   138  	// See golang.org/issue/9341 and golang.org/issue/12706.
   139  	if os.Getenv("GIT_TERMINAL_PROMPT") == "" {
   140  		os.Setenv("GIT_TERMINAL_PROMPT", "0")
   141  	}
   142  
   143  	// Disable any ssh connection pooling by Git.
   144  	// If a Git subprocess forks a child into the background to cache a new connection,
   145  	// that child keeps stdout/stderr open. After the Git subprocess exits,
   146  	// os /exec expects to be able to read from the stdout/stderr pipe
   147  	// until EOF to get all the data that the Git subprocess wrote before exiting.
   148  	// The EOF doesn't come until the child exits too, because the child
   149  	// is holding the write end of the pipe.
   150  	// This is unfortunate, but it has come up at least twice
   151  	// (see golang.org/issue/13453 and golang.org/issue/16104)
   152  	// and confuses users when it does.
   153  	// If the user has explicitly set GIT_SSH or GIT_SSH_COMMAND,
   154  	// assume they know what they are doing and don't step on it.
   155  	// But default to turning off ControlMaster.
   156  	if os.Getenv("GIT_SSH") == "" && os.Getenv("GIT_SSH_COMMAND") == "" {
   157  		os.Setenv("GIT_SSH_COMMAND", "ssh -o ControlMaster=no")
   158  	}
   159  
   160  	// Phase 1. Download/update.
   161  	var stk load.ImportStack
   162  	mode := 0
   163  	if *getT {
   164  		mode |= load.GetTestDeps
   165  	}
   166  	for _, pkg := range downloadPaths(args) {
   167  		download(pkg, nil, &stk, mode)
   168  	}
   169  	base.ExitIfErrors()
   170  
   171  	// Phase 2. Rescan packages and re-evaluate args list.
   172  
   173  	// Code we downloaded and all code that depends on it
   174  	// needs to be evicted from the package cache so that
   175  	// the information will be recomputed. Instead of keeping
   176  	// track of the reverse dependency information, evict
   177  	// everything.
   178  	load.ClearPackageCache()
   179  
   180  	// In order to rebuild packages information completely,
   181  	// we need to clear commands cache. Command packages are
   182  	// referring to evicted packages from the package cache.
   183  	// This leads to duplicated loads of the standard packages.
   184  	load.ClearCmdCache()
   185  
   186  	pkgs := load.PackagesForBuild(args)
   187  
   188  	// Phase 3. Install.
   189  	if *getD {
   190  		// Download only.
   191  		// Check delayed until now so that importPaths
   192  		// and packagesForBuild have a chance to print errors.
   193  		return
   194  	}
   195  
   196  	work.InstallPackages(args, pkgs)
   197  }
   198  
   199  // downloadPaths prepares the list of paths to pass to download.
   200  // It expands ... patterns that can be expanded. If there is no match
   201  // for a particular pattern, downloadPaths leaves it in the result list,
   202  // in the hope that we can figure out the repository from the
   203  // initial ...-free prefix.
   204  func downloadPaths(patterns []string) []string {
   205  	for _, arg := range patterns {
   206  		if strings.Contains(arg, "@") {
   207  			base.Fatalf("go: cannot use path@version syntax in GOPATH mode")
   208  		}
   209  	}
   210  	var pkgs []string
   211  	for _, m := range search.ImportPathsQuiet(patterns) {
   212  		if len(m.Pkgs) == 0 && strings.Contains(m.Pattern, "...") {
   213  			pkgs = append(pkgs, m.Pattern)
   214  		} else {
   215  			pkgs = append(pkgs, m.Pkgs...)
   216  		}
   217  	}
   218  	return pkgs
   219  }
   220  
   221  // downloadCache records the import paths we have already
   222  // considered during the download, to avoid duplicate work when
   223  // there is more than one dependency sequence leading to
   224  // a particular package.
   225  var downloadCache = map[string]bool{}
   226  
   227  // downloadRootCache records the version control repository
   228  // root directories we have already considered during the download.
   229  // For example, all the packages in the github.com/google/codesearch repo
   230  // share the same root (the directory for that path), and we only need
   231  // to run the hg commands to consider each repository once.
   232  var downloadRootCache = map[string]bool{}
   233  
   234  // download runs the download half of the get command
   235  // for the package or pattern named by the argument.
   236  func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) {
   237  	if mode&load.ResolveImport != 0 {
   238  		// Caller is responsible for expanding vendor paths.
   239  		panic("internal error: download mode has useVendor set")
   240  	}
   241  	load1 := func(path string, mode int) *load.Package {
   242  		if parent == nil {
   243  			return load.LoadPackageNoFlags(path, stk)
   244  		}
   245  		return load.LoadImport(path, parent.Dir, parent, stk, nil, mode|load.ResolveModule)
   246  	}
   247  
   248  	p := load1(arg, mode)
   249  	if p.Error != nil && p.Error.Hard {
   250  		base.Errorf("%s", p.Error)
   251  		return
   252  	}
   253  
   254  	// loadPackage inferred the canonical ImportPath from arg.
   255  	// Use that in the following to prevent hysteresis effects
   256  	// in e.g. downloadCache and packageCache.
   257  	// This allows invocations such as:
   258  	//   mkdir -p $GOPATH/src/github.com/user
   259  	//   cd $GOPATH/src/github.com/user
   260  	//   go get ./foo
   261  	// see: golang.org/issue/9767
   262  	arg = p.ImportPath
   263  
   264  	// There's nothing to do if this is a package in the standard library.
   265  	if p.Standard {
   266  		return
   267  	}
   268  
   269  	// Only process each package once.
   270  	// (Unless we're fetching test dependencies for this package,
   271  	// in which case we want to process it again.)
   272  	if downloadCache[arg] && mode&load.GetTestDeps == 0 {
   273  		return
   274  	}
   275  	downloadCache[arg] = true
   276  
   277  	pkgs := []*load.Package{p}
   278  	wildcardOkay := len(*stk) == 0
   279  	isWildcard := false
   280  
   281  	// Download if the package is missing, or update if we're using -u.
   282  	if p.Dir == "" || *getU {
   283  		// The actual download.
   284  		stk.Push(arg)
   285  		err := downloadPackage(p)
   286  		if err != nil {
   287  			base.Errorf("%s", &load.PackageError{ImportStack: stk.Copy(), Err: err.Error()})
   288  			stk.Pop()
   289  			return
   290  		}
   291  		stk.Pop()
   292  
   293  		args := []string{arg}
   294  		// If the argument has a wildcard in it, re-evaluate the wildcard.
   295  		// We delay this until after reloadPackage so that the old entry
   296  		// for p has been replaced in the package cache.
   297  		if wildcardOkay && strings.Contains(arg, "...") {
   298  			if build.IsLocalImport(arg) {
   299  				args = search.MatchPackagesInFS(arg).Pkgs
   300  			} else {
   301  				args = search.MatchPackages(arg).Pkgs
   302  			}
   303  			isWildcard = true
   304  		}
   305  
   306  		// Clear all relevant package cache entries before
   307  		// doing any new loads.
   308  		load.ClearPackageCachePartial(args)
   309  
   310  		pkgs = pkgs[:0]
   311  		for _, arg := range args {
   312  			// Note: load calls loadPackage or loadImport,
   313  			// which push arg onto stk already.
   314  			// Do not push here too, or else stk will say arg imports arg.
   315  			p := load1(arg, mode)
   316  			if p.Error != nil {
   317  				base.Errorf("%s", p.Error)
   318  				continue
   319  			}
   320  			pkgs = append(pkgs, p)
   321  		}
   322  	}
   323  
   324  	// Process package, which might now be multiple packages
   325  	// due to wildcard expansion.
   326  	for _, p := range pkgs {
   327  		if *getFix {
   328  			files := base.RelPaths(p.InternalAllGoFiles())
   329  			base.Run(cfg.BuildToolexec, str.StringList(base.Tool("fix"), files))
   330  
   331  			// The imports might have changed, so reload again.
   332  			p = load.ReloadPackageNoFlags(arg, stk)
   333  			if p.Error != nil {
   334  				base.Errorf("%s", p.Error)
   335  				return
   336  			}
   337  		}
   338  
   339  		if isWildcard {
   340  			// Report both the real package and the
   341  			// wildcard in any error message.
   342  			stk.Push(p.ImportPath)
   343  		}
   344  
   345  		// Process dependencies, now that we know what they are.
   346  		imports := p.Imports
   347  		if mode&load.GetTestDeps != 0 {
   348  			// Process test dependencies when -t is specified.
   349  			// (But don't get test dependencies for test dependencies:
   350  			// we always pass mode 0 to the recursive calls below.)
   351  			imports = str.StringList(imports, p.TestImports, p.XTestImports)
   352  		}
   353  		for i, path := range imports {
   354  			if path == "C" {
   355  				continue
   356  			}
   357  			// Fail fast on import naming full vendor path.
   358  			// Otherwise expand path as needed for test imports.
   359  			// Note that p.Imports can have additional entries beyond p.Internal.Build.Imports.
   360  			orig := path
   361  			if i < len(p.Internal.Build.Imports) {
   362  				orig = p.Internal.Build.Imports[i]
   363  			}
   364  			if j, ok := load.FindVendor(orig); ok {
   365  				stk.Push(path)
   366  				err := &load.PackageError{
   367  					ImportStack: stk.Copy(),
   368  					Err:         "must be imported as " + path[j+len("vendor/"):],
   369  				}
   370  				stk.Pop()
   371  				base.Errorf("%s", err)
   372  				continue
   373  			}
   374  			// If this is a test import, apply module and vendor lookup now.
   375  			// We cannot pass ResolveImport to download, because
   376  			// download does caching based on the value of path,
   377  			// so it must be the fully qualified path already.
   378  			if i >= len(p.Imports) {
   379  				path = load.ResolveImportPath(p, path)
   380  			}
   381  			download(path, p, stk, 0)
   382  		}
   383  
   384  		if isWildcard {
   385  			stk.Pop()
   386  		}
   387  	}
   388  }
   389  
   390  // downloadPackage runs the create or download command
   391  // to make the first copy of or update a copy of the given package.
   392  func downloadPackage(p *load.Package) error {
   393  	var (
   394  		vcs            *vcsCmd
   395  		repo, rootPath string
   396  		err            error
   397  		blindRepo      bool // set if the repo has unusual configuration
   398  	)
   399  
   400  	security := web.Secure
   401  	if Insecure {
   402  		security = web.Insecure
   403  	}
   404  
   405  	// p can be either a real package, or a pseudo-package whose “import path” is
   406  	// actually a wildcard pattern.
   407  	// Trim the path at the element containing the first wildcard,
   408  	// and hope that it applies to the wildcarded parts too.
   409  	// This makes 'go get rsc.io/pdf/...' work in a fresh GOPATH.
   410  	importPrefix := p.ImportPath
   411  	if i := strings.Index(importPrefix, "..."); i >= 0 {
   412  		slash := strings.LastIndexByte(importPrefix[:i], '/')
   413  		if slash < 0 {
   414  			return fmt.Errorf("cannot expand ... in %q", p.ImportPath)
   415  		}
   416  		importPrefix = importPrefix[:slash]
   417  	}
   418  	if err := CheckImportPath(importPrefix); err != nil {
   419  		return fmt.Errorf("%s: invalid import path: %v", p.ImportPath, err)
   420  	}
   421  
   422  	if p.Internal.Build.SrcRoot != "" {
   423  		// Directory exists. Look for checkout along path to src.
   424  		vcs, rootPath, err = vcsFromDir(p.Dir, p.Internal.Build.SrcRoot)
   425  		if err != nil {
   426  			return err
   427  		}
   428  		repo = "<local>" // should be unused; make distinctive
   429  
   430  		// Double-check where it came from.
   431  		if *getU && vcs.remoteRepo != nil {
   432  			dir := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath))
   433  			remote, err := vcs.remoteRepo(vcs, dir)
   434  			if err != nil {
   435  				// Proceed anyway. The package is present; we likely just don't understand
   436  				// the repo configuration (e.g. unusual remote protocol).
   437  				blindRepo = true
   438  			}
   439  			repo = remote
   440  			if !*getF && err == nil {
   441  				if rr, err := RepoRootForImportPath(importPrefix, IgnoreMod, security); err == nil {
   442  					repo := rr.Repo
   443  					if rr.vcs.resolveRepo != nil {
   444  						resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo)
   445  						if err == nil {
   446  							repo = resolved
   447  						}
   448  					}
   449  					if remote != repo && rr.IsCustom {
   450  						return fmt.Errorf("%s is a custom import path for %s, but %s is checked out from %s", rr.Root, repo, dir, remote)
   451  					}
   452  				}
   453  			}
   454  		}
   455  	} else {
   456  		// Analyze the import path to determine the version control system,
   457  		// repository, and the import path for the root of the repository.
   458  		rr, err := RepoRootForImportPath(importPrefix, IgnoreMod, security)
   459  		if err != nil {
   460  			return err
   461  		}
   462  		vcs, repo, rootPath = rr.vcs, rr.Repo, rr.Root
   463  	}
   464  	if !blindRepo && !vcs.isSecure(repo) && !Insecure {
   465  		return fmt.Errorf("cannot download, %v uses insecure protocol", repo)
   466  	}
   467  
   468  	if p.Internal.Build.SrcRoot == "" {
   469  		// Package not found. Put in first directory of $GOPATH.
   470  		list := filepath.SplitList(cfg.BuildContext.GOPATH)
   471  		if len(list) == 0 {
   472  			return fmt.Errorf("cannot download, $GOPATH not set. For more details see: 'go help gopath'")
   473  		}
   474  		// Guard against people setting GOPATH=$GOROOT.
   475  		if filepath.Clean(list[0]) == filepath.Clean(cfg.GOROOT) {
   476  			return fmt.Errorf("cannot download, $GOPATH must not be set to $GOROOT. For more details see: 'go help gopath'")
   477  		}
   478  		if _, err := os.Stat(filepath.Join(list[0], "src/cmd/go/alldocs.go")); err == nil {
   479  			return fmt.Errorf("cannot download, %s is a GOROOT, not a GOPATH. For more details see: 'go help gopath'", list[0])
   480  		}
   481  		p.Internal.Build.Root = list[0]
   482  		p.Internal.Build.SrcRoot = filepath.Join(list[0], "src")
   483  		p.Internal.Build.PkgRoot = filepath.Join(list[0], "pkg")
   484  	}
   485  	root := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath))
   486  
   487  	if err := checkNestedVCS(vcs, root, p.Internal.Build.SrcRoot); err != nil {
   488  		return err
   489  	}
   490  
   491  	// If we've considered this repository already, don't do it again.
   492  	if downloadRootCache[root] {
   493  		return nil
   494  	}
   495  	downloadRootCache[root] = true
   496  
   497  	if cfg.BuildV {
   498  		fmt.Fprintf(os.Stderr, "%s (download)\n", rootPath)
   499  	}
   500  
   501  	// Check that this is an appropriate place for the repo to be checked out.
   502  	// The target directory must either not exist or have a repo checked out already.
   503  	meta := filepath.Join(root, "."+vcs.cmd)
   504  	if _, err := os.Stat(meta); err != nil {
   505  		// Metadata file or directory does not exist. Prepare to checkout new copy.
   506  		// Some version control tools require the target directory not to exist.
   507  		// We require that too, just to avoid stepping on existing work.
   508  		if _, err := os.Stat(root); err == nil {
   509  			return fmt.Errorf("%s exists but %s does not - stale checkout?", root, meta)
   510  		}
   511  
   512  		_, err := os.Stat(p.Internal.Build.Root)
   513  		gopathExisted := err == nil
   514  
   515  		// Some version control tools require the parent of the target to exist.
   516  		parent, _ := filepath.Split(root)
   517  		if err = os.MkdirAll(parent, 0777); err != nil {
   518  			return err
   519  		}
   520  		if cfg.BuildV && !gopathExisted && p.Internal.Build.Root == cfg.BuildContext.GOPATH {
   521  			fmt.Fprintf(os.Stderr, "created GOPATH=%s; see 'go help gopath'\n", p.Internal.Build.Root)
   522  		}
   523  
   524  		if err = vcs.create(root, repo); err != nil {
   525  			return err
   526  		}
   527  	} else {
   528  		// Metadata directory does exist; download incremental updates.
   529  		if err = vcs.download(root); err != nil {
   530  			return err
   531  		}
   532  	}
   533  
   534  	if cfg.BuildN {
   535  		// Do not show tag sync in -n; it's noise more than anything,
   536  		// and since we're not running commands, no tag will be found.
   537  		// But avoid printing nothing.
   538  		fmt.Fprintf(os.Stderr, "# cd %s; %s sync/update\n", root, vcs.cmd)
   539  		return nil
   540  	}
   541  
   542  	// Select and sync to appropriate version of the repository.
   543  	tags, err := vcs.tags(root)
   544  	if err != nil {
   545  		return err
   546  	}
   547  	vers := runtime.Version()
   548  	if i := strings.Index(vers, " "); i >= 0 {
   549  		vers = vers[:i]
   550  	}
   551  	if err := vcs.tagSync(root, selectTag(vers, tags)); err != nil {
   552  		return err
   553  	}
   554  
   555  	return nil
   556  }
   557  
   558  // selectTag returns the closest matching tag for a given version.
   559  // Closest means the latest one that is not after the current release.
   560  // Version "goX" (or "goX.Y" or "goX.Y.Z") matches tags of the same form.
   561  // Version "release.rN" matches tags of the form "go.rN" (N being a floating-point number).
   562  // Version "weekly.YYYY-MM-DD" matches tags like "go.weekly.YYYY-MM-DD".
   563  //
   564  // NOTE(rsc): Eventually we will need to decide on some logic here.
   565  // For now, there is only "go1". This matches the docs in go help get.
   566  func selectTag(goVersion string, tags []string) (match string) {
   567  	for _, t := range tags {
   568  		if t == "go1" {
   569  			return "go1"
   570  		}
   571  	}
   572  	return ""
   573  }
   574  

View as plain text