Black Lives Matter. Support the Equal Justice Initiative.

Source file src/cmd/go/internal/modcmd/vendor.go

Documentation: cmd/go/internal/modcmd

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modcmd
     6  
     7  import (
     8  	"bytes"
     9  	"context"
    10  	"errors"
    11  	"fmt"
    12  	"go/build"
    13  	"io"
    14  	"io/fs"
    15  	"os"
    16  	"path/filepath"
    17  	"sort"
    18  	"strings"
    19  
    20  	"cmd/go/internal/base"
    21  	"cmd/go/internal/cfg"
    22  	"cmd/go/internal/fsys"
    23  	"cmd/go/internal/imports"
    24  	"cmd/go/internal/load"
    25  	"cmd/go/internal/modload"
    26  	"cmd/go/internal/str"
    27  
    28  	"golang.org/x/mod/module"
    29  	"golang.org/x/mod/semver"
    30  )
    31  
    32  var cmdVendor = &base.Command{
    33  	UsageLine: "go mod vendor [-e] [-v]",
    34  	Short:     "make vendored copy of dependencies",
    35  	Long: `
    36  Vendor resets the main module's vendor directory to include all packages
    37  needed to build and test all the main module's packages.
    38  It does not include test code for vendored packages.
    39  
    40  The -v flag causes vendor to print the names of vendored
    41  modules and packages to standard error.
    42  
    43  The -e flag causes vendor to attempt to proceed despite errors
    44  encountered while loading packages.
    45  
    46  See https://golang.org/ref/mod#go-mod-vendor for more about 'go mod vendor'.
    47  	`,
    48  	Run: runVendor,
    49  }
    50  
    51  var vendorE bool // if true, report errors but proceed anyway
    52  
    53  func init() {
    54  	cmdVendor.Flag.BoolVar(&cfg.BuildV, "v", false, "")
    55  	cmdVendor.Flag.BoolVar(&vendorE, "e", false, "")
    56  	base.AddModCommonFlags(&cmdVendor.Flag)
    57  }
    58  
    59  func runVendor(ctx context.Context, cmd *base.Command, args []string) {
    60  	if len(args) != 0 {
    61  		base.Fatalf("go mod vendor: vendor takes no arguments")
    62  	}
    63  	modload.ForceUseModules = true
    64  	modload.RootMode = modload.NeedRoot
    65  
    66  	loadOpts := modload.PackageOpts{
    67  		Tags:                  imports.AnyTags(),
    68  		ResolveMissingImports: true,
    69  		UseVendorAll:          true,
    70  		AllowErrors:           vendorE,
    71  	}
    72  	_, pkgs := modload.LoadPackages(ctx, loadOpts, "all")
    73  
    74  	vdir := filepath.Join(modload.ModRoot(), "vendor")
    75  	if err := os.RemoveAll(vdir); err != nil {
    76  		base.Fatalf("go mod vendor: %v", err)
    77  	}
    78  
    79  	modpkgs := make(map[module.Version][]string)
    80  	for _, pkg := range pkgs {
    81  		m := modload.PackageModule(pkg)
    82  		if m.Path == "" || m == modload.Target {
    83  			continue
    84  		}
    85  		modpkgs[m] = append(modpkgs[m], pkg)
    86  	}
    87  
    88  	includeAllReplacements := false
    89  	isExplicit := map[module.Version]bool{}
    90  	if gv := modload.ModFile().Go; gv != nil && semver.Compare("v"+gv.Version, "v1.14") >= 0 {
    91  		// If the Go version is at least 1.14, annotate all explicit 'require' and
    92  		// 'replace' targets found in the go.mod file so that we can perform a
    93  		// stronger consistency check when -mod=vendor is set.
    94  		for _, r := range modload.ModFile().Require {
    95  			isExplicit[r.Mod] = true
    96  		}
    97  		includeAllReplacements = true
    98  	}
    99  
   100  	var vendorMods []module.Version
   101  	for m := range isExplicit {
   102  		vendorMods = append(vendorMods, m)
   103  	}
   104  	for m := range modpkgs {
   105  		if !isExplicit[m] {
   106  			vendorMods = append(vendorMods, m)
   107  		}
   108  	}
   109  	module.Sort(vendorMods)
   110  
   111  	var buf bytes.Buffer
   112  	for _, m := range vendorMods {
   113  		line := moduleLine(m, modload.Replacement(m))
   114  		buf.WriteString(line)
   115  		if cfg.BuildV {
   116  			os.Stderr.WriteString(line)
   117  		}
   118  		if isExplicit[m] {
   119  			buf.WriteString("## explicit\n")
   120  			if cfg.BuildV {
   121  				os.Stderr.WriteString("## explicit\n")
   122  			}
   123  		}
   124  		pkgs := modpkgs[m]
   125  		sort.Strings(pkgs)
   126  		for _, pkg := range pkgs {
   127  			fmt.Fprintf(&buf, "%s\n", pkg)
   128  			if cfg.BuildV {
   129  				fmt.Fprintf(os.Stderr, "%s\n", pkg)
   130  			}
   131  			vendorPkg(vdir, pkg)
   132  		}
   133  	}
   134  
   135  	if includeAllReplacements {
   136  		// Record unused and wildcard replacements at the end of the modules.txt file:
   137  		// without access to the complete build list, the consumer of the vendor
   138  		// directory can't otherwise determine that those replacements had no effect.
   139  		for _, r := range modload.ModFile().Replace {
   140  			if len(modpkgs[r.Old]) > 0 {
   141  				// We we already recorded this replacement in the entry for the replaced
   142  				// module with the packages it provides.
   143  				continue
   144  			}
   145  
   146  			line := moduleLine(r.Old, r.New)
   147  			buf.WriteString(line)
   148  			if cfg.BuildV {
   149  				os.Stderr.WriteString(line)
   150  			}
   151  		}
   152  	}
   153  
   154  	if buf.Len() == 0 {
   155  		fmt.Fprintf(os.Stderr, "go: no dependencies to vendor\n")
   156  		return
   157  	}
   158  
   159  	if err := os.MkdirAll(vdir, 0777); err != nil {
   160  		base.Fatalf("go mod vendor: %v", err)
   161  	}
   162  
   163  	if err := os.WriteFile(filepath.Join(vdir, "modules.txt"), buf.Bytes(), 0666); err != nil {
   164  		base.Fatalf("go mod vendor: %v", err)
   165  	}
   166  }
   167  
   168  func moduleLine(m, r module.Version) string {
   169  	b := new(strings.Builder)
   170  	b.WriteString("# ")
   171  	b.WriteString(m.Path)
   172  	if m.Version != "" {
   173  		b.WriteString(" ")
   174  		b.WriteString(m.Version)
   175  	}
   176  	if r.Path != "" {
   177  		b.WriteString(" => ")
   178  		b.WriteString(r.Path)
   179  		if r.Version != "" {
   180  			b.WriteString(" ")
   181  			b.WriteString(r.Version)
   182  		}
   183  	}
   184  	b.WriteString("\n")
   185  	return b.String()
   186  }
   187  
   188  func vendorPkg(vdir, pkg string) {
   189  	// TODO(#42504): Instead of calling modload.ImportMap then build.ImportDir,
   190  	// just call load.PackagesAndErrors. To do that, we need to add a good way
   191  	// to ignore build constraints.
   192  	realPath := modload.ImportMap(pkg)
   193  	if realPath != pkg && modload.ImportMap(realPath) != "" {
   194  		fmt.Fprintf(os.Stderr, "warning: %s imported as both %s and %s; making two copies.\n", realPath, realPath, pkg)
   195  	}
   196  
   197  	copiedFiles := make(map[string]bool)
   198  	dst := filepath.Join(vdir, pkg)
   199  	src := modload.PackageDir(realPath)
   200  	if src == "" {
   201  		fmt.Fprintf(os.Stderr, "internal error: no pkg for %s -> %s\n", pkg, realPath)
   202  	}
   203  	copyDir(dst, src, matchPotentialSourceFile, copiedFiles)
   204  	if m := modload.PackageModule(realPath); m.Path != "" {
   205  		copyMetadata(m.Path, realPath, dst, src, copiedFiles)
   206  	}
   207  
   208  	ctx := build.Default
   209  	ctx.UseAllFiles = true
   210  	bp, err := ctx.ImportDir(src, build.IgnoreVendor)
   211  	// Because UseAllFiles is set on the build.Context, it's possible ta get
   212  	// a MultiplePackageError on an otherwise valid package: the package could
   213  	// have different names for GOOS=windows and GOOS=mac for example. On the
   214  	// other hand if there's a NoGoError, the package might have source files
   215  	// specifying "// +build ignore" those packages should be skipped because
   216  	// embeds from ignored files can't be used.
   217  	// TODO(#42504): Find a better way to avoid errors from ImportDir. We'll
   218  	// need to figure this out when we switch to PackagesAndErrors as per the
   219  	// TODO above.
   220  	var multiplePackageError *build.MultiplePackageError
   221  	var noGoError *build.NoGoError
   222  	if err != nil {
   223  		if errors.As(err, &noGoError) {
   224  			return // No source files in this package are built. Skip embeds in ignored files.
   225  		} else if !errors.As(err, &multiplePackageError) { // multiplePackgeErrors are okay, but others are not.
   226  			base.Fatalf("internal error: failed to find embedded files of %s: %v\n", pkg, err)
   227  		}
   228  	}
   229  	embedPatterns := str.StringList(bp.EmbedPatterns, bp.TestEmbedPatterns, bp.XTestEmbedPatterns)
   230  	embeds, err := load.ResolveEmbed(bp.Dir, embedPatterns)
   231  	if err != nil {
   232  		base.Fatalf("go mod vendor: %v", err)
   233  	}
   234  	for _, embed := range embeds {
   235  		embedDst := filepath.Join(dst, embed)
   236  		if copiedFiles[embedDst] {
   237  			continue
   238  		}
   239  
   240  		// Copy the file as is done by copyDir below.
   241  		r, err := os.Open(filepath.Join(src, embed))
   242  		if err != nil {
   243  			base.Fatalf("go mod vendor: %v", err)
   244  		}
   245  		if err := os.MkdirAll(filepath.Dir(embedDst), 0777); err != nil {
   246  			base.Fatalf("go mod vendor: %v", err)
   247  		}
   248  		w, err := os.Create(embedDst)
   249  		if err != nil {
   250  			base.Fatalf("go mod vendor: %v", err)
   251  		}
   252  		if _, err := io.Copy(w, r); err != nil {
   253  			base.Fatalf("go mod vendor: %v", err)
   254  		}
   255  		r.Close()
   256  		if err := w.Close(); err != nil {
   257  			base.Fatalf("go mod vendor: %v", err)
   258  		}
   259  	}
   260  }
   261  
   262  type metakey struct {
   263  	modPath string
   264  	dst     string
   265  }
   266  
   267  var copiedMetadata = make(map[metakey]bool)
   268  
   269  // copyMetadata copies metadata files from parents of src to parents of dst,
   270  // stopping after processing the src parent for modPath.
   271  func copyMetadata(modPath, pkg, dst, src string, copiedFiles map[string]bool) {
   272  	for parent := 0; ; parent++ {
   273  		if copiedMetadata[metakey{modPath, dst}] {
   274  			break
   275  		}
   276  		copiedMetadata[metakey{modPath, dst}] = true
   277  		if parent > 0 {
   278  			copyDir(dst, src, matchMetadata, copiedFiles)
   279  		}
   280  		if modPath == pkg {
   281  			break
   282  		}
   283  		pkg = filepath.Dir(pkg)
   284  		dst = filepath.Dir(dst)
   285  		src = filepath.Dir(src)
   286  	}
   287  }
   288  
   289  // metaPrefixes is the list of metadata file prefixes.
   290  // Vendoring copies metadata files from parents of copied directories.
   291  // Note that this list could be arbitrarily extended, and it is longer
   292  // in other tools (such as godep or dep). By using this limited set of
   293  // prefixes and also insisting on capitalized file names, we are trying
   294  // to nudge people toward more agreement on the naming
   295  // and also trying to avoid false positives.
   296  var metaPrefixes = []string{
   297  	"AUTHORS",
   298  	"CONTRIBUTORS",
   299  	"COPYLEFT",
   300  	"COPYING",
   301  	"COPYRIGHT",
   302  	"LEGAL",
   303  	"LICENSE",
   304  	"NOTICE",
   305  	"PATENTS",
   306  }
   307  
   308  // matchMetadata reports whether info is a metadata file.
   309  func matchMetadata(dir string, info fs.DirEntry) bool {
   310  	name := info.Name()
   311  	for _, p := range metaPrefixes {
   312  		if strings.HasPrefix(name, p) {
   313  			return true
   314  		}
   315  	}
   316  	return false
   317  }
   318  
   319  // matchPotentialSourceFile reports whether info may be relevant to a build operation.
   320  func matchPotentialSourceFile(dir string, info fs.DirEntry) bool {
   321  	if strings.HasSuffix(info.Name(), "_test.go") {
   322  		return false
   323  	}
   324  	if strings.HasSuffix(info.Name(), ".go") {
   325  		f, err := fsys.Open(filepath.Join(dir, info.Name()))
   326  		if err != nil {
   327  			base.Fatalf("go mod vendor: %v", err)
   328  		}
   329  		defer f.Close()
   330  
   331  		content, err := imports.ReadImports(f, false, nil)
   332  		if err == nil && !imports.ShouldBuild(content, imports.AnyTags()) {
   333  			// The file is explicitly tagged "ignore", so it can't affect the build.
   334  			// Leave it out.
   335  			return false
   336  		}
   337  		return true
   338  	}
   339  
   340  	// We don't know anything about this file, so optimistically assume that it is
   341  	// needed.
   342  	return true
   343  }
   344  
   345  // copyDir copies all regular files satisfying match(info) from src to dst.
   346  func copyDir(dst, src string, match func(dir string, info fs.DirEntry) bool, copiedFiles map[string]bool) {
   347  	files, err := os.ReadDir(src)
   348  	if err != nil {
   349  		base.Fatalf("go mod vendor: %v", err)
   350  	}
   351  	if err := os.MkdirAll(dst, 0777); err != nil {
   352  		base.Fatalf("go mod vendor: %v", err)
   353  	}
   354  	for _, file := range files {
   355  		if file.IsDir() || !file.Type().IsRegular() || !match(src, file) {
   356  			continue
   357  		}
   358  		copiedFiles[file.Name()] = true
   359  		r, err := os.Open(filepath.Join(src, file.Name()))
   360  		if err != nil {
   361  			base.Fatalf("go mod vendor: %v", err)
   362  		}
   363  		dstPath := filepath.Join(dst, file.Name())
   364  		copiedFiles[dstPath] = true
   365  		w, err := os.Create(dstPath)
   366  		if err != nil {
   367  			base.Fatalf("go mod vendor: %v", err)
   368  		}
   369  		if _, err := io.Copy(w, r); err != nil {
   370  			base.Fatalf("go mod vendor: %v", err)
   371  		}
   372  		r.Close()
   373  		if err := w.Close(); err != nil {
   374  			base.Fatalf("go mod vendor: %v", err)
   375  		}
   376  	}
   377  }
   378  

View as plain text