vendor.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modcmd
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"errors"
    11  	"fmt"
    12  	"go/build"
    13  	"io"
    14  	"io/fs"
    15  	"os"
    16  	"path"
    17  	"path/filepath"
    18  	"sort"
    19  	"strings"
    20  
    21  	"cmd/go/internal/base"
    22  	"cmd/go/internal/cfg"
    23  	"cmd/go/internal/fsys"
    24  	"cmd/go/internal/gover"
    25  	"cmd/go/internal/imports"
    26  	"cmd/go/internal/load"
    27  	"cmd/go/internal/modload"
    28  	"cmd/go/internal/str"
    29  
    30  	"golang.org/x/mod/module"
    31  )
    32  
    33  var cmdVendor = &base.Command{
    34  	UsageLine: "go mod vendor [-e] [-v] [-o outdir]",
    35  	Short:     "make vendored copy of dependencies",
    36  	Long: `
    37  Vendor resets the main module's vendor directory to include all packages
    38  needed to build and test all the main module's packages.
    39  It does not include test code for vendored packages.
    40  
    41  The -v flag causes vendor to print the names of vendored
    42  modules and packages to standard error.
    43  
    44  The -e flag causes vendor to attempt to proceed despite errors
    45  encountered while loading packages.
    46  
    47  The -o flag causes vendor to create the vendor directory at the given
    48  path instead of "vendor". The go command can only use a vendor directory
    49  named "vendor" within the module root directory, so this flag is
    50  primarily useful for other tools.
    51  
    52  See https://golang.org/ref/mod#go-mod-vendor for more about 'go mod vendor'.
    53  	`,
    54  	Run: runVendor,
    55  }
    56  
    57  var vendorE bool   // if true, report errors but proceed anyway
    58  var vendorO string // if set, overrides the default output directory
    59  
    60  func init() {
    61  	cmdVendor.Flag.BoolVar(&cfg.BuildV, "v", false, "")
    62  	cmdVendor.Flag.BoolVar(&vendorE, "e", false, "")
    63  	cmdVendor.Flag.StringVar(&vendorO, "o", "", "")
    64  	base.AddChdirFlag(&cmdVendor.Flag)
    65  	base.AddModCommonFlags(&cmdVendor.Flag)
    66  }
    67  
    68  func runVendor(ctx context.Context, cmd *base.Command, args []string) {
    69  	modload.InitWorkfile()
    70  	if modload.WorkFilePath() != "" {
    71  		base.Fatalf("go: 'go mod vendor' cannot be run in workspace mode. Run 'go work vendor' to vendor the workspace or set 'GOWORK=off' to exit workspace mode.")
    72  	}
    73  	RunVendor(ctx, vendorE, vendorO, args)
    74  }
    75  
    76  func RunVendor(ctx context.Context, vendorE bool, vendorO string, args []string) {
    77  	if len(args) != 0 {
    78  		base.Fatalf("go: 'go mod vendor' accepts no arguments")
    79  	}
    80  	modload.ForceUseModules = true
    81  	modload.RootMode = modload.NeedRoot
    82  
    83  	loadOpts := modload.PackageOpts{
    84  		Tags:                     imports.AnyTags(),
    85  		VendorModulesInGOROOTSrc: true,
    86  		ResolveMissingImports:    true,
    87  		UseVendorAll:             true,
    88  		AllowErrors:              vendorE,
    89  		SilenceMissingStdImports: true,
    90  	}
    91  	_, pkgs := modload.LoadPackages(ctx, loadOpts, "all")
    92  
    93  	var vdir string
    94  	switch {
    95  	case filepath.IsAbs(vendorO):
    96  		vdir = vendorO
    97  	case vendorO != "":
    98  		vdir = filepath.Join(base.Cwd(), vendorO)
    99  	default:
   100  		vdir = filepath.Join(modload.VendorDir())
   101  	}
   102  	if err := os.RemoveAll(vdir); err != nil {
   103  		base.Fatal(err)
   104  	}
   105  
   106  	modpkgs := make(map[module.Version][]string)
   107  	for _, pkg := range pkgs {
   108  		m := modload.PackageModule(pkg)
   109  		if m.Path == "" || modload.MainModules.Contains(m.Path) {
   110  			continue
   111  		}
   112  		modpkgs[m] = append(modpkgs[m], pkg)
   113  	}
   114  
   115  	includeAllReplacements := false
   116  	includeGoVersions := false
   117  	isExplicit := map[module.Version]bool{}
   118  	gv := modload.MainModules.GoVersion()
   119  	if gover.Compare(gv, "1.14") >= 0 && (modload.FindGoWork(base.Cwd()) != "" || modload.ModFile().Go != nil) {
   120  		// If the Go version is at least 1.14, annotate all explicit 'require' and
   121  		// 'replace' targets found in the go.mod file so that we can perform a
   122  		// stronger consistency check when -mod=vendor is set.
   123  		for _, m := range modload.MainModules.Versions() {
   124  			if modFile := modload.MainModules.ModFile(m); modFile != nil {
   125  				for _, r := range modFile.Require {
   126  					isExplicit[r.Mod] = true
   127  				}
   128  			}
   129  
   130  		}
   131  		includeAllReplacements = true
   132  	}
   133  	if gover.Compare(gv, "1.17") >= 0 {
   134  		// If the Go version is at least 1.17, annotate all modules with their
   135  		// 'go' version directives.
   136  		includeGoVersions = true
   137  	}
   138  
   139  	var vendorMods []module.Version
   140  	for m := range isExplicit {
   141  		vendorMods = append(vendorMods, m)
   142  	}
   143  	for m := range modpkgs {
   144  		if !isExplicit[m] {
   145  			vendorMods = append(vendorMods, m)
   146  		}
   147  	}
   148  	gover.ModSort(vendorMods)
   149  
   150  	var (
   151  		buf bytes.Buffer
   152  		w   io.Writer = &buf
   153  	)
   154  	if cfg.BuildV {
   155  		w = io.MultiWriter(&buf, os.Stderr)
   156  	}
   157  
   158  	if modload.MainModules.WorkFile() != nil {
   159  		fmt.Fprintf(w, "## workspace\n")
   160  	}
   161  
   162  	replacementWritten := make(map[module.Version]bool)
   163  	for _, m := range vendorMods {
   164  		replacement := modload.Replacement(m)
   165  		line := moduleLine(m, replacement)
   166  		replacementWritten[m] = true
   167  		io.WriteString(w, line)
   168  
   169  		goVersion := ""
   170  		if includeGoVersions {
   171  			goVersion = modload.ModuleInfo(ctx, m.Path).GoVersion
   172  		}
   173  		switch {
   174  		case isExplicit[m] && goVersion != "":
   175  			fmt.Fprintf(w, "## explicit; go %s\n", goVersion)
   176  		case isExplicit[m]:
   177  			io.WriteString(w, "## explicit\n")
   178  		case goVersion != "":
   179  			fmt.Fprintf(w, "## go %s\n", goVersion)
   180  		}
   181  
   182  		pkgs := modpkgs[m]
   183  		sort.Strings(pkgs)
   184  		for _, pkg := range pkgs {
   185  			fmt.Fprintf(w, "%s\n", pkg)
   186  			vendorPkg(vdir, pkg)
   187  		}
   188  	}
   189  
   190  	if includeAllReplacements {
   191  		// Record unused and wildcard replacements at the end of the modules.txt file:
   192  		// without access to the complete build list, the consumer of the vendor
   193  		// directory can't otherwise determine that those replacements had no effect.
   194  		for _, m := range modload.MainModules.Versions() {
   195  			if workFile := modload.MainModules.WorkFile(); workFile != nil {
   196  				for _, r := range workFile.Replace {
   197  					if replacementWritten[r.Old] {
   198  						// We already recorded this replacement.
   199  						continue
   200  					}
   201  					replacementWritten[r.Old] = true
   202  
   203  					line := moduleLine(r.Old, r.New)
   204  					buf.WriteString(line)
   205  					if cfg.BuildV {
   206  						os.Stderr.WriteString(line)
   207  					}
   208  				}
   209  			}
   210  			if modFile := modload.MainModules.ModFile(m); modFile != nil {
   211  				for _, r := range modFile.Replace {
   212  					if replacementWritten[r.Old] {
   213  						// We already recorded this replacement.
   214  						continue
   215  					}
   216  					replacementWritten[r.Old] = true
   217  					rNew := modload.Replacement(r.Old)
   218  					if rNew == (module.Version{}) {
   219  						// There is no replacement. Don't try to write it.
   220  						continue
   221  					}
   222  
   223  					line := moduleLine(r.Old, rNew)
   224  					buf.WriteString(line)
   225  					if cfg.BuildV {
   226  						os.Stderr.WriteString(line)
   227  					}
   228  				}
   229  			}
   230  		}
   231  	}
   232  
   233  	if buf.Len() == 0 {
   234  		fmt.Fprintf(os.Stderr, "go: no dependencies to vendor\n")
   235  		return
   236  	}
   237  
   238  	if err := os.MkdirAll(vdir, 0777); err != nil {
   239  		base.Fatal(err)
   240  	}
   241  
   242  	if err := os.WriteFile(filepath.Join(vdir, "modules.txt"), buf.Bytes(), 0666); err != nil {
   243  		base.Fatal(err)
   244  	}
   245  }
   246  
   247  func moduleLine(m, r module.Version) string {
   248  	b := new(strings.Builder)
   249  	b.WriteString("# ")
   250  	b.WriteString(m.Path)
   251  	if m.Version != "" {
   252  		b.WriteString(" ")
   253  		b.WriteString(m.Version)
   254  	}
   255  	if r.Path != "" {
   256  		if str.HasFilePathPrefix(filepath.Clean(r.Path), "vendor") {
   257  			base.Fatalf("go: replacement path %s inside vendor directory", r.Path)
   258  		}
   259  		b.WriteString(" => ")
   260  		b.WriteString(r.Path)
   261  		if r.Version != "" {
   262  			b.WriteString(" ")
   263  			b.WriteString(r.Version)
   264  		}
   265  	}
   266  	b.WriteString("\n")
   267  	return b.String()
   268  }
   269  
   270  func vendorPkg(vdir, pkg string) {
   271  	src, realPath, _ := modload.Lookup("", false, pkg)
   272  	if src == "" {
   273  		base.Errorf("internal error: no pkg for %s\n", pkg)
   274  		return
   275  	}
   276  	if realPath != pkg {
   277  		// TODO(#26904): Revisit whether this behavior still makes sense.
   278  		// This should actually be impossible today, because the import map is the
   279  		// identity function for packages outside of the standard library.
   280  		//
   281  		// Part of the purpose of the vendor directory is to allow the packages in
   282  		// the module to continue to build in GOPATH mode, and GOPATH-mode users
   283  		// won't know about replacement aliasing. How important is it to maintain
   284  		// compatibility?
   285  		fmt.Fprintf(os.Stderr, "warning: %s imported as both %s and %s; making two copies.\n", realPath, realPath, pkg)
   286  	}
   287  
   288  	copiedFiles := make(map[string]bool)
   289  	dst := filepath.Join(vdir, pkg)
   290  	copyDir(dst, src, matchPotentialSourceFile, copiedFiles)
   291  	if m := modload.PackageModule(realPath); m.Path != "" {
   292  		copyMetadata(m.Path, realPath, dst, src, copiedFiles)
   293  	}
   294  
   295  	ctx := build.Default
   296  	ctx.UseAllFiles = true
   297  	bp, err := ctx.ImportDir(src, build.IgnoreVendor)
   298  	// Because UseAllFiles is set on the build.Context, it's possible ta get
   299  	// a MultiplePackageError on an otherwise valid package: the package could
   300  	// have different names for GOOS=windows and GOOS=mac for example. On the
   301  	// other hand if there's a NoGoError, the package might have source files
   302  	// specifying "//go:build ignore" those packages should be skipped because
   303  	// embeds from ignored files can't be used.
   304  	// TODO(#42504): Find a better way to avoid errors from ImportDir. We'll
   305  	// need to figure this out when we switch to PackagesAndErrors as per the
   306  	// TODO above.
   307  	var multiplePackageError *build.MultiplePackageError
   308  	var noGoError *build.NoGoError
   309  	if err != nil {
   310  		if errors.As(err, &noGoError) {
   311  			return // No source files in this package are built. Skip embeds in ignored files.
   312  		} else if !errors.As(err, &multiplePackageError) { // multiplePackageErrors are OK, but others are not.
   313  			base.Fatalf("internal error: failed to find embedded files of %s: %v\n", pkg, err)
   314  		}
   315  	}
   316  	var embedPatterns []string
   317  	if gover.Compare(modload.MainModules.GoVersion(), "1.22") >= 0 {
   318  		embedPatterns = bp.EmbedPatterns
   319  	} else {
   320  		// Maintain the behavior of https://github.com/golang/go/issues/63473
   321  		// so that we continue to agree with older versions of the go command
   322  		// about the contents of vendor directories in existing modules
   323  		embedPatterns = str.StringList(bp.EmbedPatterns, bp.TestEmbedPatterns, bp.XTestEmbedPatterns)
   324  	}
   325  	embeds, err := load.ResolveEmbed(bp.Dir, embedPatterns)
   326  	if err != nil {
   327  		base.Fatal(err)
   328  	}
   329  	for _, embed := range embeds {
   330  		embedDst := filepath.Join(dst, embed)
   331  		if copiedFiles[embedDst] {
   332  			continue
   333  		}
   334  
   335  		// Copy the file as is done by copyDir below.
   336  		r, err := os.Open(filepath.Join(src, embed))
   337  		if err != nil {
   338  			base.Fatal(err)
   339  		}
   340  		if err := os.MkdirAll(filepath.Dir(embedDst), 0777); err != nil {
   341  			base.Fatal(err)
   342  		}
   343  		w, err := os.Create(embedDst)
   344  		if err != nil {
   345  			base.Fatal(err)
   346  		}
   347  		if _, err := io.Copy(w, r); err != nil {
   348  			base.Fatal(err)
   349  		}
   350  		r.Close()
   351  		if err := w.Close(); err != nil {
   352  			base.Fatal(err)
   353  		}
   354  	}
   355  }
   356  
   357  type metakey struct {
   358  	modPath string
   359  	dst     string
   360  }
   361  
   362  var copiedMetadata = make(map[metakey]bool)
   363  
   364  // copyMetadata copies metadata files from parents of src to parents of dst,
   365  // stopping after processing the src parent for modPath.
   366  func copyMetadata(modPath, pkg, dst, src string, copiedFiles map[string]bool) {
   367  	for parent := 0; ; parent++ {
   368  		if copiedMetadata[metakey{modPath, dst}] {
   369  			break
   370  		}
   371  		copiedMetadata[metakey{modPath, dst}] = true
   372  		if parent > 0 {
   373  			copyDir(dst, src, matchMetadata, copiedFiles)
   374  		}
   375  		if modPath == pkg {
   376  			break
   377  		}
   378  		pkg = path.Dir(pkg)
   379  		dst = filepath.Dir(dst)
   380  		src = filepath.Dir(src)
   381  	}
   382  }
   383  
   384  // metaPrefixes is the list of metadata file prefixes.
   385  // Vendoring copies metadata files from parents of copied directories.
   386  // Note that this list could be arbitrarily extended, and it is longer
   387  // in other tools (such as godep or dep). By using this limited set of
   388  // prefixes and also insisting on capitalized file names, we are trying
   389  // to nudge people toward more agreement on the naming
   390  // and also trying to avoid false positives.
   391  var metaPrefixes = []string{
   392  	"AUTHORS",
   393  	"CONTRIBUTORS",
   394  	"COPYLEFT",
   395  	"COPYING",
   396  	"COPYRIGHT",
   397  	"LEGAL",
   398  	"LICENSE",
   399  	"NOTICE",
   400  	"PATENTS",
   401  }
   402  
   403  // matchMetadata reports whether info is a metadata file.
   404  func matchMetadata(dir string, info fs.DirEntry) bool {
   405  	name := info.Name()
   406  	for _, p := range metaPrefixes {
   407  		if strings.HasPrefix(name, p) {
   408  			return true
   409  		}
   410  	}
   411  	return false
   412  }
   413  
   414  // matchPotentialSourceFile reports whether info may be relevant to a build operation.
   415  func matchPotentialSourceFile(dir string, info fs.DirEntry) bool {
   416  	if strings.HasSuffix(info.Name(), "_test.go") {
   417  		return false
   418  	}
   419  	if info.Name() == "go.mod" || info.Name() == "go.sum" {
   420  		if gv := modload.MainModules.GoVersion(); gover.Compare(gv, "1.17") >= 0 {
   421  			// As of Go 1.17, we strip go.mod and go.sum files from dependency modules.
   422  			// Otherwise, 'go' commands invoked within the vendor subtree may misidentify
   423  			// an arbitrary directory within the vendor tree as a module root.
   424  			// (See https://golang.org/issue/42970.)
   425  			return false
   426  		}
   427  	}
   428  	if strings.HasSuffix(info.Name(), ".go") {
   429  		f, err := fsys.Open(filepath.Join(dir, info.Name()))
   430  		if err != nil {
   431  			base.Fatal(err)
   432  		}
   433  		defer f.Close()
   434  
   435  		content, err := imports.ReadImports(f, false, nil)
   436  		if err == nil && !imports.ShouldBuild(content, imports.AnyTags()) {
   437  			// The file is explicitly tagged "ignore", so it can't affect the build.
   438  			// Leave it out.
   439  			return false
   440  		}
   441  		return true
   442  	}
   443  
   444  	// We don't know anything about this file, so optimistically assume that it is
   445  	// needed.
   446  	return true
   447  }
   448  
   449  // copyDir copies all regular files satisfying match(info) from src to dst.
   450  func copyDir(dst, src string, match func(dir string, info fs.DirEntry) bool, copiedFiles map[string]bool) {
   451  	files, err := os.ReadDir(src)
   452  	if err != nil {
   453  		base.Fatal(err)
   454  	}
   455  	if err := os.MkdirAll(dst, 0777); err != nil {
   456  		base.Fatal(err)
   457  	}
   458  	for _, file := range files {
   459  		if file.IsDir() || !file.Type().IsRegular() || !match(src, file) {
   460  			continue
   461  		}
   462  		copiedFiles[file.Name()] = true
   463  		r, err := os.Open(filepath.Join(src, file.Name()))
   464  		if err != nil {
   465  			base.Fatal(err)
   466  		}
   467  		dstPath := filepath.Join(dst, file.Name())
   468  		copiedFiles[dstPath] = true
   469  		w, err := os.Create(dstPath)
   470  		if err != nil {
   471  			base.Fatal(err)
   472  		}
   473  		if _, err := io.Copy(w, r); err != nil {
   474  			base.Fatal(err)
   475  		}
   476  		r.Close()
   477  		if err := w.Close(); err != nil {
   478  			base.Fatal(err)
   479  		}
   480  	}
   481  }
   482
View as plain text