...
Run Format

Source file src/cmd/go/internal/get/get.go

Documentation: cmd/go/internal/get

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package get implements the ``go get'' command.
     6  package get
     7  
     8  import (
     9  	"fmt"
    10  	"go/build"
    11  	"os"
    12  	"path/filepath"
    13  	"runtime"
    14  	"strings"
    15  
    16  	"cmd/go/internal/base"
    17  	"cmd/go/internal/cfg"
    18  	"cmd/go/internal/load"
    19  	"cmd/go/internal/str"
    20  	"cmd/go/internal/web"
    21  	"cmd/go/internal/work"
    22  )
    23  
    24  var CmdGet = &base.Command{
    25  	UsageLine: "get [-d] [-f] [-fix] [-insecure] [-t] [-u] [-v] [build flags] [packages]",
    26  	Short:     "download and install packages and dependencies",
    27  	Long: `
    28  Get downloads the packages named by the import paths, along with their
    29  dependencies. It then installs the named packages, like 'go install'.
    30  
    31  The -d flag instructs get to stop after downloading the packages; that is,
    32  it instructs get not to install the packages.
    33  
    34  The -f flag, valid only when -u is set, forces get -u not to verify that
    35  each package has been checked out from the source control repository
    36  implied by its import path. This can be useful if the source is a local fork
    37  of the original.
    38  
    39  The -fix flag instructs get to run the fix tool on the downloaded packages
    40  before resolving dependencies or building the code.
    41  
    42  The -insecure flag permits fetching from repositories and resolving
    43  custom domains using insecure schemes such as HTTP. Use with caution.
    44  
    45  The -t flag instructs get to also download the packages required to build
    46  the tests for the specified packages.
    47  
    48  The -u flag instructs get to use the network to update the named packages
    49  and their dependencies. By default, get uses the network to check out
    50  missing packages but does not use it to look for updates to existing packages.
    51  
    52  The -v flag enables verbose progress and debug output.
    53  
    54  Get also accepts build flags to control the installation. See 'go help build'.
    55  
    56  When checking out a new package, get creates the target directory
    57  GOPATH/src/<import-path>. If the GOPATH contains multiple entries,
    58  get uses the first one. For more details see: 'go help gopath'.
    59  
    60  When checking out or updating a package, get looks for a branch or tag
    61  that matches the locally installed version of Go. The most important
    62  rule is that if the local installation is running version "go1", get
    63  searches for a branch or tag named "go1". If no such version exists
    64  it retrieves the default branch of the package.
    65  
    66  When go get checks out or updates a Git repository,
    67  it also updates any git submodules referenced by the repository.
    68  
    69  Get never checks out or updates code stored in vendor directories.
    70  
    71  For more about specifying packages, see 'go help packages'.
    72  
    73  For more about how 'go get' finds source code to
    74  download, see 'go help importpath'.
    75  
    76  See also: go build, go install, go clean.
    77  	`,
    78  }
    79  
    80  var getD = CmdGet.Flag.Bool("d", false, "")
    81  var getF = CmdGet.Flag.Bool("f", false, "")
    82  var getT = CmdGet.Flag.Bool("t", false, "")
    83  var getU = CmdGet.Flag.Bool("u", false, "")
    84  var getFix = CmdGet.Flag.Bool("fix", false, "")
    85  var getInsecure = CmdGet.Flag.Bool("insecure", false, "")
    86  
    87  func init() {
    88  	work.AddBuildFlags(CmdGet)
    89  	CmdGet.Run = runGet // break init loop
    90  }
    91  
    92  func runGet(cmd *base.Command, args []string) {
    93  	work.BuildInit()
    94  
    95  	if *getF && !*getU {
    96  		base.Fatalf("go get: cannot use -f flag without -u")
    97  	}
    98  
    99  	// Disable any prompting for passwords by Git.
   100  	// Only has an effect for 2.3.0 or later, but avoiding
   101  	// the prompt in earlier versions is just too hard.
   102  	// If user has explicitly set GIT_TERMINAL_PROMPT=1, keep
   103  	// prompting.
   104  	// See golang.org/issue/9341 and golang.org/issue/12706.
   105  	if os.Getenv("GIT_TERMINAL_PROMPT") == "" {
   106  		os.Setenv("GIT_TERMINAL_PROMPT", "0")
   107  	}
   108  
   109  	// Disable any ssh connection pooling by Git.
   110  	// If a Git subprocess forks a child into the background to cache a new connection,
   111  	// that child keeps stdout/stderr open. After the Git subprocess exits,
   112  	// os /exec expects to be able to read from the stdout/stderr pipe
   113  	// until EOF to get all the data that the Git subprocess wrote before exiting.
   114  	// The EOF doesn't come until the child exits too, because the child
   115  	// is holding the write end of the pipe.
   116  	// This is unfortunate, but it has come up at least twice
   117  	// (see golang.org/issue/13453 and golang.org/issue/16104)
   118  	// and confuses users when it does.
   119  	// If the user has explicitly set GIT_SSH or GIT_SSH_COMMAND,
   120  	// assume they know what they are doing and don't step on it.
   121  	// But default to turning off ControlMaster.
   122  	if os.Getenv("GIT_SSH") == "" && os.Getenv("GIT_SSH_COMMAND") == "" {
   123  		os.Setenv("GIT_SSH_COMMAND", "ssh -o ControlMaster=no")
   124  	}
   125  
   126  	// Phase 1. Download/update.
   127  	var stk load.ImportStack
   128  	mode := 0
   129  	if *getT {
   130  		mode |= load.GetTestDeps
   131  	}
   132  	args = downloadPaths(args)
   133  	for _, arg := range args {
   134  		download(arg, nil, &stk, mode)
   135  	}
   136  	base.ExitIfErrors()
   137  
   138  	// Phase 2. Rescan packages and re-evaluate args list.
   139  
   140  	// Code we downloaded and all code that depends on it
   141  	// needs to be evicted from the package cache so that
   142  	// the information will be recomputed. Instead of keeping
   143  	// track of the reverse dependency information, evict
   144  	// everything.
   145  	load.ClearPackageCache()
   146  
   147  	// In order to rebuild packages information completely,
   148  	// we need to clear commands cache. Command packages are
   149  	// referring to evicted packages from the package cache.
   150  	// This leads to duplicated loads of the standard packages.
   151  	load.ClearCmdCache()
   152  
   153  	args = load.ImportPaths(args)
   154  	load.PackagesForBuild(args)
   155  
   156  	// Phase 3. Install.
   157  	if *getD {
   158  		// Download only.
   159  		// Check delayed until now so that importPaths
   160  		// and packagesForBuild have a chance to print errors.
   161  		return
   162  	}
   163  
   164  	work.InstallPackages(args, true)
   165  }
   166  
   167  // downloadPaths prepares the list of paths to pass to download.
   168  // It expands ... patterns that can be expanded. If there is no match
   169  // for a particular pattern, downloadPaths leaves it in the result list,
   170  // in the hope that we can figure out the repository from the
   171  // initial ...-free prefix.
   172  func downloadPaths(args []string) []string {
   173  	args = load.ImportPathsNoDotExpansion(args)
   174  	var out []string
   175  	for _, a := range args {
   176  		if strings.Contains(a, "...") {
   177  			var expand []string
   178  			// Use matchPackagesInFS to avoid printing
   179  			// warnings. They will be printed by the
   180  			// eventual call to importPaths instead.
   181  			if build.IsLocalImport(a) {
   182  				expand = load.MatchPackagesInFS(a)
   183  			} else {
   184  				expand = load.MatchPackages(a)
   185  			}
   186  			if len(expand) > 0 {
   187  				out = append(out, expand...)
   188  				continue
   189  			}
   190  		}
   191  		out = append(out, a)
   192  	}
   193  	return out
   194  }
   195  
   196  // downloadCache records the import paths we have already
   197  // considered during the download, to avoid duplicate work when
   198  // there is more than one dependency sequence leading to
   199  // a particular package.
   200  var downloadCache = map[string]bool{}
   201  
   202  // downloadRootCache records the version control repository
   203  // root directories we have already considered during the download.
   204  // For example, all the packages in the github.com/google/codesearch repo
   205  // share the same root (the directory for that path), and we only need
   206  // to run the hg commands to consider each repository once.
   207  var downloadRootCache = map[string]bool{}
   208  
   209  // download runs the download half of the get command
   210  // for the package named by the argument.
   211  func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) {
   212  	if mode&load.UseVendor != 0 {
   213  		// Caller is responsible for expanding vendor paths.
   214  		panic("internal error: download mode has useVendor set")
   215  	}
   216  	load1 := func(path string, mode int) *load.Package {
   217  		if parent == nil {
   218  			return load.LoadPackage(path, stk)
   219  		}
   220  		return load.LoadImport(path, parent.Dir, parent, stk, nil, mode)
   221  	}
   222  
   223  	p := load1(arg, mode)
   224  	if p.Error != nil && p.Error.Hard {
   225  		base.Errorf("%s", p.Error)
   226  		return
   227  	}
   228  
   229  	// loadPackage inferred the canonical ImportPath from arg.
   230  	// Use that in the following to prevent hysteresis effects
   231  	// in e.g. downloadCache and packageCache.
   232  	// This allows invocations such as:
   233  	//   mkdir -p $GOPATH/src/github.com/user
   234  	//   cd $GOPATH/src/github.com/user
   235  	//   go get ./foo
   236  	// see: golang.org/issue/9767
   237  	arg = p.ImportPath
   238  
   239  	// There's nothing to do if this is a package in the standard library.
   240  	if p.Standard {
   241  		return
   242  	}
   243  
   244  	// Only process each package once.
   245  	// (Unless we're fetching test dependencies for this package,
   246  	// in which case we want to process it again.)
   247  	if downloadCache[arg] && mode&load.GetTestDeps == 0 {
   248  		return
   249  	}
   250  	downloadCache[arg] = true
   251  
   252  	pkgs := []*load.Package{p}
   253  	wildcardOkay := len(*stk) == 0
   254  	isWildcard := false
   255  
   256  	// Download if the package is missing, or update if we're using -u.
   257  	if p.Dir == "" || *getU {
   258  		// The actual download.
   259  		stk.Push(arg)
   260  		err := downloadPackage(p)
   261  		if err != nil {
   262  			base.Errorf("%s", &load.PackageError{ImportStack: stk.Copy(), Err: err.Error()})
   263  			stk.Pop()
   264  			return
   265  		}
   266  		stk.Pop()
   267  
   268  		args := []string{arg}
   269  		// If the argument has a wildcard in it, re-evaluate the wildcard.
   270  		// We delay this until after reloadPackage so that the old entry
   271  		// for p has been replaced in the package cache.
   272  		if wildcardOkay && strings.Contains(arg, "...") {
   273  			if build.IsLocalImport(arg) {
   274  				args = load.MatchPackagesInFS(arg)
   275  			} else {
   276  				args = load.MatchPackages(arg)
   277  			}
   278  			isWildcard = true
   279  		}
   280  
   281  		// Clear all relevant package cache entries before
   282  		// doing any new loads.
   283  		load.ClearPackageCachePartial(args)
   284  
   285  		pkgs = pkgs[:0]
   286  		for _, arg := range args {
   287  			// Note: load calls loadPackage or loadImport,
   288  			// which push arg onto stk already.
   289  			// Do not push here too, or else stk will say arg imports arg.
   290  			p := load1(arg, mode)
   291  			if p.Error != nil {
   292  				base.Errorf("%s", p.Error)
   293  				continue
   294  			}
   295  			pkgs = append(pkgs, p)
   296  		}
   297  	}
   298  
   299  	// Process package, which might now be multiple packages
   300  	// due to wildcard expansion.
   301  	for _, p := range pkgs {
   302  		if *getFix {
   303  			files := base.RelPaths(p.InternalAllGoFiles())
   304  			base.Run(cfg.BuildToolexec, str.StringList(base.Tool("fix"), files))
   305  
   306  			// The imports might have changed, so reload again.
   307  			p = load.ReloadPackage(arg, stk)
   308  			if p.Error != nil {
   309  				base.Errorf("%s", p.Error)
   310  				return
   311  			}
   312  		}
   313  
   314  		if isWildcard {
   315  			// Report both the real package and the
   316  			// wildcard in any error message.
   317  			stk.Push(p.ImportPath)
   318  		}
   319  
   320  		// Process dependencies, now that we know what they are.
   321  		imports := p.Imports
   322  		if mode&load.GetTestDeps != 0 {
   323  			// Process test dependencies when -t is specified.
   324  			// (But don't get test dependencies for test dependencies:
   325  			// we always pass mode 0 to the recursive calls below.)
   326  			imports = str.StringList(imports, p.TestImports, p.XTestImports)
   327  		}
   328  		for i, path := range imports {
   329  			if path == "C" {
   330  				continue
   331  			}
   332  			// Fail fast on import naming full vendor path.
   333  			// Otherwise expand path as needed for test imports.
   334  			// Note that p.Imports can have additional entries beyond p.Internal.Build.Imports.
   335  			orig := path
   336  			if i < len(p.Internal.Build.Imports) {
   337  				orig = p.Internal.Build.Imports[i]
   338  			}
   339  			if j, ok := load.FindVendor(orig); ok {
   340  				stk.Push(path)
   341  				err := &load.PackageError{
   342  					ImportStack: stk.Copy(),
   343  					Err:         "must be imported as " + path[j+len("vendor/"):],
   344  				}
   345  				stk.Pop()
   346  				base.Errorf("%s", err)
   347  				continue
   348  			}
   349  			// If this is a test import, apply vendor lookup now.
   350  			// We cannot pass useVendor to download, because
   351  			// download does caching based on the value of path,
   352  			// so it must be the fully qualified path already.
   353  			if i >= len(p.Imports) {
   354  				path = load.VendoredImportPath(p, path)
   355  			}
   356  			download(path, p, stk, 0)
   357  		}
   358  
   359  		if isWildcard {
   360  			stk.Pop()
   361  		}
   362  	}
   363  }
   364  
   365  // downloadPackage runs the create or download command
   366  // to make the first copy of or update a copy of the given package.
   367  func downloadPackage(p *load.Package) error {
   368  	var (
   369  		vcs            *vcsCmd
   370  		repo, rootPath string
   371  		err            error
   372  	)
   373  
   374  	security := web.Secure
   375  	if *getInsecure {
   376  		security = web.Insecure
   377  	}
   378  
   379  	if p.Internal.Build.SrcRoot != "" {
   380  		// Directory exists. Look for checkout along path to src.
   381  		vcs, rootPath, err = vcsFromDir(p.Dir, p.Internal.Build.SrcRoot)
   382  		if err != nil {
   383  			return err
   384  		}
   385  		repo = "<local>" // should be unused; make distinctive
   386  
   387  		// Double-check where it came from.
   388  		if *getU && vcs.remoteRepo != nil {
   389  			dir := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath))
   390  			remote, err := vcs.remoteRepo(vcs, dir)
   391  			if err != nil {
   392  				return err
   393  			}
   394  			repo = remote
   395  			if !*getF {
   396  				if rr, err := repoRootForImportPath(p.ImportPath, security); err == nil {
   397  					repo := rr.repo
   398  					if rr.vcs.resolveRepo != nil {
   399  						resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo)
   400  						if err == nil {
   401  							repo = resolved
   402  						}
   403  					}
   404  					if remote != repo && rr.isCustom {
   405  						return fmt.Errorf("%s is a custom import path for %s, but %s is checked out from %s", rr.root, repo, dir, remote)
   406  					}
   407  				}
   408  			}
   409  		}
   410  	} else {
   411  		// Analyze the import path to determine the version control system,
   412  		// repository, and the import path for the root of the repository.
   413  		rr, err := repoRootForImportPath(p.ImportPath, security)
   414  		if err != nil {
   415  			return err
   416  		}
   417  		vcs, repo, rootPath = rr.vcs, rr.repo, rr.root
   418  	}
   419  	if !vcs.isSecure(repo) && !*getInsecure {
   420  		return fmt.Errorf("cannot download, %v uses insecure protocol", repo)
   421  	}
   422  
   423  	if p.Internal.Build.SrcRoot == "" {
   424  		// Package not found. Put in first directory of $GOPATH.
   425  		list := filepath.SplitList(cfg.BuildContext.GOPATH)
   426  		if len(list) == 0 {
   427  			return fmt.Errorf("cannot download, $GOPATH not set. For more details see: 'go help gopath'")
   428  		}
   429  		// Guard against people setting GOPATH=$GOROOT.
   430  		if filepath.Clean(list[0]) == filepath.Clean(cfg.GOROOT) {
   431  			return fmt.Errorf("cannot download, $GOPATH must not be set to $GOROOT. For more details see: 'go help gopath'")
   432  		}
   433  		if _, err := os.Stat(filepath.Join(list[0], "src/cmd/go/alldocs.go")); err == nil {
   434  			return fmt.Errorf("cannot download, %s is a GOROOT, not a GOPATH. For more details see: 'go help gopath'", list[0])
   435  		}
   436  		p.Internal.Build.Root = list[0]
   437  		p.Internal.Build.SrcRoot = filepath.Join(list[0], "src")
   438  		p.Internal.Build.PkgRoot = filepath.Join(list[0], "pkg")
   439  	}
   440  	root := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath))
   441  
   442  	if err := checkNestedVCS(vcs, root, p.Internal.Build.SrcRoot); err != nil {
   443  		return err
   444  	}
   445  
   446  	// If we've considered this repository already, don't do it again.
   447  	if downloadRootCache[root] {
   448  		return nil
   449  	}
   450  	downloadRootCache[root] = true
   451  
   452  	if cfg.BuildV {
   453  		fmt.Fprintf(os.Stderr, "%s (download)\n", rootPath)
   454  	}
   455  
   456  	// Check that this is an appropriate place for the repo to be checked out.
   457  	// The target directory must either not exist or have a repo checked out already.
   458  	meta := filepath.Join(root, "."+vcs.cmd)
   459  	if _, err := os.Stat(meta); err != nil {
   460  		// Metadata file or directory does not exist. Prepare to checkout new copy.
   461  		// Some version control tools require the target directory not to exist.
   462  		// We require that too, just to avoid stepping on existing work.
   463  		if _, err := os.Stat(root); err == nil {
   464  			return fmt.Errorf("%s exists but %s does not - stale checkout?", root, meta)
   465  		}
   466  
   467  		_, err := os.Stat(p.Internal.Build.Root)
   468  		gopathExisted := err == nil
   469  
   470  		// Some version control tools require the parent of the target to exist.
   471  		parent, _ := filepath.Split(root)
   472  		if err = os.MkdirAll(parent, 0777); err != nil {
   473  			return err
   474  		}
   475  		if cfg.BuildV && !gopathExisted && p.Internal.Build.Root == cfg.BuildContext.GOPATH {
   476  			fmt.Fprintf(os.Stderr, "created GOPATH=%s; see 'go help gopath'\n", p.Internal.Build.Root)
   477  		}
   478  
   479  		if err = vcs.create(root, repo); err != nil {
   480  			return err
   481  		}
   482  	} else {
   483  		// Metadata directory does exist; download incremental updates.
   484  		if err = vcs.download(root); err != nil {
   485  			return err
   486  		}
   487  	}
   488  
   489  	if cfg.BuildN {
   490  		// Do not show tag sync in -n; it's noise more than anything,
   491  		// and since we're not running commands, no tag will be found.
   492  		// But avoid printing nothing.
   493  		fmt.Fprintf(os.Stderr, "# cd %s; %s sync/update\n", root, vcs.cmd)
   494  		return nil
   495  	}
   496  
   497  	// Select and sync to appropriate version of the repository.
   498  	tags, err := vcs.tags(root)
   499  	if err != nil {
   500  		return err
   501  	}
   502  	vers := runtime.Version()
   503  	if i := strings.Index(vers, " "); i >= 0 {
   504  		vers = vers[:i]
   505  	}
   506  	if err := vcs.tagSync(root, selectTag(vers, tags)); err != nil {
   507  		return err
   508  	}
   509  
   510  	return nil
   511  }
   512  
   513  // selectTag returns the closest matching tag for a given version.
   514  // Closest means the latest one that is not after the current release.
   515  // Version "goX" (or "goX.Y" or "goX.Y.Z") matches tags of the same form.
   516  // Version "release.rN" matches tags of the form "go.rN" (N being a floating-point number).
   517  // Version "weekly.YYYY-MM-DD" matches tags like "go.weekly.YYYY-MM-DD".
   518  //
   519  // NOTE(rsc): Eventually we will need to decide on some logic here.
   520  // For now, there is only "go1". This matches the docs in go help get.
   521  func selectTag(goVersion string, tags []string) (match string) {
   522  	for _, t := range tags {
   523  		if t == "go1" {
   524  			return "go1"
   525  		}
   526  	}
   527  	return ""
   528  }
   529  

View as plain text