import.go

Documentation: cmd/go/internal/modload

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modload
     6  
     7  import (
     8  	"context"
     9  	"errors"
    10  	"fmt"
    11  	"go/build"
    12  	"internal/goroot"
    13  	"io/fs"
    14  	"os"
    15  	"path/filepath"
    16  	"sort"
    17  	"strings"
    18  
    19  	"cmd/go/internal/cfg"
    20  	"cmd/go/internal/fsys"
    21  	"cmd/go/internal/modfetch"
    22  	"cmd/go/internal/par"
    23  	"cmd/go/internal/search"
    24  
    25  	"golang.org/x/mod/module"
    26  	"golang.org/x/mod/semver"
    27  )
    28  
    29  type ImportMissingError struct {
    30  	Path     string
    31  	Module   module.Version
    32  	QueryErr error
    33  
    34  	// isStd indicates whether we would expect to find the package in the standard
    35  	// library. This is normally true for all dotless import paths, but replace
    36  	// directives can cause us to treat the replaced paths as also being in
    37  	// modules.
    38  	isStd bool
    39  
    40  	// replaced the highest replaced version of the module where the replacement
    41  	// contains the package. replaced is only set if the replacement is unused.
    42  	replaced module.Version
    43  
    44  	// newMissingVersion is set to a newer version of Module if one is present
    45  	// in the build list. When set, we can't automatically upgrade.
    46  	newMissingVersion string
    47  }
    48  
    49  func (e *ImportMissingError) Error() string {
    50  	if e.Module.Path == "" {
    51  		if e.isStd {
    52  			return fmt.Sprintf("package %s is not in GOROOT (%s)", e.Path, filepath.Join(cfg.GOROOT, "src", e.Path))
    53  		}
    54  		if e.QueryErr != nil && e.QueryErr != ErrNoModRoot {
    55  			return fmt.Sprintf("cannot find module providing package %s: %v", e.Path, e.QueryErr)
    56  		}
    57  		if cfg.BuildMod == "mod" || (cfg.BuildMod == "readonly" && allowMissingModuleImports) {
    58  			return "cannot find module providing package " + e.Path
    59  		}
    60  
    61  		if e.replaced.Path != "" {
    62  			suggestArg := e.replaced.Path
    63  			if !modfetch.IsZeroPseudoVersion(e.replaced.Version) {
    64  				suggestArg = e.replaced.String()
    65  			}
    66  			return fmt.Sprintf("module %s provides package %s and is replaced but not required; to add it:\n\tgo get %s", e.replaced.Path, e.Path, suggestArg)
    67  		}
    68  
    69  		message := fmt.Sprintf("no required module provides package %s", e.Path)
    70  		if e.QueryErr != nil {
    71  			return fmt.Sprintf("%s: %v", message, e.QueryErr)
    72  		}
    73  		return fmt.Sprintf("%s; to add it:\n\tgo get %s", message, e.Path)
    74  	}
    75  
    76  	if e.newMissingVersion != "" {
    77  		return fmt.Sprintf("package %s provided by %s at latest version %s but not at required version %s", e.Path, e.Module.Path, e.Module.Version, e.newMissingVersion)
    78  	}
    79  
    80  	return fmt.Sprintf("missing module for import: %s@%s provides %s", e.Module.Path, e.Module.Version, e.Path)
    81  }
    82  
    83  func (e *ImportMissingError) Unwrap() error {
    84  	return e.QueryErr
    85  }
    86  
    87  func (e *ImportMissingError) ImportPath() string {
    88  	return e.Path
    89  }
    90  
    91  // An AmbiguousImportError indicates an import of a package found in multiple
    92  // modules in the build list, or found in both the main module and its vendor
    93  // directory.
    94  type AmbiguousImportError struct {
    95  	importPath string
    96  	Dirs       []string
    97  	Modules    []module.Version // Either empty or 1:1 with Dirs.
    98  }
    99  
   100  func (e *AmbiguousImportError) ImportPath() string {
   101  	return e.importPath
   102  }
   103  
   104  func (e *AmbiguousImportError) Error() string {
   105  	locType := "modules"
   106  	if len(e.Modules) == 0 {
   107  		locType = "directories"
   108  	}
   109  
   110  	var buf strings.Builder
   111  	fmt.Fprintf(&buf, "ambiguous import: found package %s in multiple %s:", e.importPath, locType)
   112  
   113  	for i, dir := range e.Dirs {
   114  		buf.WriteString("\n\t")
   115  		if i < len(e.Modules) {
   116  			m := e.Modules[i]
   117  			buf.WriteString(m.Path)
   118  			if m.Version != "" {
   119  				fmt.Fprintf(&buf, " %s", m.Version)
   120  			}
   121  			fmt.Fprintf(&buf, " (%s)", dir)
   122  		} else {
   123  			buf.WriteString(dir)
   124  		}
   125  	}
   126  
   127  	return buf.String()
   128  }
   129  
   130  // ImportMissingSumError is reported in readonly mode when we need to check
   131  // if a module contains a package, but we don't have a sum for its .zip file.
   132  // We might need sums for multiple modules to verify the package is unique.
   133  //
   134  // TODO(#43653): consolidate multiple errors of this type into a single error
   135  // that suggests a 'go get' command for root packages that transtively import
   136  // packages from modules with missing sums. load.CheckPackageErrors would be
   137  // a good place to consolidate errors, but we'll need to attach the import
   138  // stack here.
   139  type ImportMissingSumError struct {
   140  	importPath                string
   141  	found                     bool
   142  	mods                      []module.Version
   143  	importer, importerVersion string // optional, but used for additional context
   144  	importerIsTest            bool
   145  }
   146  
   147  func (e *ImportMissingSumError) Error() string {
   148  	var importParen string
   149  	if e.importer != "" {
   150  		importParen = fmt.Sprintf(" (imported by %s)", e.importer)
   151  	}
   152  	var message string
   153  	if e.found {
   154  		message = fmt.Sprintf("missing go.sum entry needed to verify package %s%s is provided by exactly one module", e.importPath, importParen)
   155  	} else {
   156  		message = fmt.Sprintf("missing go.sum entry for module providing package %s%s", e.importPath, importParen)
   157  	}
   158  	var hint string
   159  	if e.importer == "" {
   160  		// Importing package is unknown, or the missing package was named on the
   161  		// command line. Recommend 'go mod download' for the modules that could
   162  		// provide the package, since that shouldn't change go.mod.
   163  		args := make([]string, len(e.mods))
   164  		for i, mod := range e.mods {
   165  			args[i] = mod.Path
   166  		}
   167  		hint = fmt.Sprintf("; to add:\n\tgo mod download %s", strings.Join(args, " "))
   168  	} else {
   169  		// Importing package is known (common case). Recommend 'go get' on the
   170  		// current version of the importing package.
   171  		tFlag := ""
   172  		if e.importerIsTest {
   173  			tFlag = " -t"
   174  		}
   175  		version := ""
   176  		if e.importerVersion != "" {
   177  			version = "@" + e.importerVersion
   178  		}
   179  		hint = fmt.Sprintf("; to add:\n\tgo get%s %s%s", tFlag, e.importer, version)
   180  	}
   181  	return message + hint
   182  }
   183  
   184  func (e *ImportMissingSumError) ImportPath() string {
   185  	return e.importPath
   186  }
   187  
   188  type invalidImportError struct {
   189  	importPath string
   190  	err        error
   191  }
   192  
   193  func (e *invalidImportError) ImportPath() string {
   194  	return e.importPath
   195  }
   196  
   197  func (e *invalidImportError) Error() string {
   198  	return e.err.Error()
   199  }
   200  
   201  func (e *invalidImportError) Unwrap() error {
   202  	return e.err
   203  }
   204  
   205  // importFromBuildList finds the module and directory in the build list
   206  // containing the package with the given import path. The answer must be unique:
   207  // importFromBuildList returns an error if multiple modules attempt to provide
   208  // the same package.
   209  //
   210  // importFromBuildList can return a module with an empty m.Path, for packages in
   211  // the standard library.
   212  //
   213  // importFromBuildList can return an empty directory string, for fake packages
   214  // like "C" and "unsafe".
   215  //
   216  // If the package cannot be found in buildList,
   217  // importFromBuildList returns an *ImportMissingError.
   218  func importFromBuildList(ctx context.Context, path string, buildList []module.Version) (m module.Version, dir string, err error) {
   219  	if strings.Contains(path, "@") {
   220  		return module.Version{}, "", fmt.Errorf("import path should not have @version")
   221  	}
   222  	if build.IsLocalImport(path) {
   223  		return module.Version{}, "", fmt.Errorf("relative import not supported")
   224  	}
   225  	if path == "C" || path == "unsafe" {
   226  		// There's no directory for import "C" or import "unsafe".
   227  		return module.Version{}, "", nil
   228  	}
   229  	// Before any further lookup, check that the path is valid.
   230  	if err := module.CheckImportPath(path); err != nil {
   231  		return module.Version{}, "", &invalidImportError{importPath: path, err: err}
   232  	}
   233  
   234  	// Is the package in the standard library?
   235  	pathIsStd := search.IsStandardImportPath(path)
   236  	if pathIsStd && goroot.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, path) {
   237  		if targetInGorootSrc {
   238  			if dir, ok, err := dirInModule(path, targetPrefix, ModRoot(), true); err != nil {
   239  				return module.Version{}, dir, err
   240  			} else if ok {
   241  				return Target, dir, nil
   242  			}
   243  		}
   244  		dir := filepath.Join(cfg.GOROOT, "src", path)
   245  		return module.Version{}, dir, nil
   246  	}
   247  
   248  	// -mod=vendor is special.
   249  	// Everything must be in the main module or the main module's vendor directory.
   250  	if cfg.BuildMod == "vendor" {
   251  		mainDir, mainOK, mainErr := dirInModule(path, targetPrefix, ModRoot(), true)
   252  		vendorDir, vendorOK, _ := dirInModule(path, "", filepath.Join(ModRoot(), "vendor"), false)
   253  		if mainOK && vendorOK {
   254  			return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: []string{mainDir, vendorDir}}
   255  		}
   256  		// Prefer to return main directory if there is one,
   257  		// Note that we're not checking that the package exists.
   258  		// We'll leave that for load.
   259  		if !vendorOK && mainDir != "" {
   260  			return Target, mainDir, nil
   261  		}
   262  		if mainErr != nil {
   263  			return module.Version{}, "", mainErr
   264  		}
   265  		readVendorList()
   266  		return vendorPkgModule[path], vendorDir, nil
   267  	}
   268  
   269  	// Check each module on the build list.
   270  	var dirs []string
   271  	var mods []module.Version
   272  	var sumErrMods []module.Version
   273  	for _, m := range buildList {
   274  		if !maybeInModule(path, m.Path) {
   275  			// Avoid possibly downloading irrelevant modules.
   276  			continue
   277  		}
   278  		needSum := true
   279  		root, isLocal, err := fetch(ctx, m, needSum)
   280  		if err != nil {
   281  			if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) {
   282  				// We are missing a sum needed to fetch a module in the build list.
   283  				// We can't verify that the package is unique, and we may not find
   284  				// the package at all. Keep checking other modules to decide which
   285  				// error to report. Multiple sums may be missing if we need to look in
   286  				// multiple nested modules to resolve the import.
   287  				sumErrMods = append(sumErrMods, m)
   288  				continue
   289  			}
   290  			// Report fetch error.
   291  			// Note that we don't know for sure this module is necessary,
   292  			// but it certainly _could_ provide the package, and even if we
   293  			// continue the loop and find the package in some other module,
   294  			// we need to look at this module to make sure the import is
   295  			// not ambiguous.
   296  			return module.Version{}, "", err
   297  		}
   298  		if dir, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil {
   299  			return module.Version{}, "", err
   300  		} else if ok {
   301  			mods = append(mods, m)
   302  			dirs = append(dirs, dir)
   303  		}
   304  	}
   305  	if len(mods) > 1 {
   306  		return module.Version{}, "", &AmbiguousImportError{importPath: path, Dirs: dirs, Modules: mods}
   307  	}
   308  	if len(sumErrMods) > 0 {
   309  		return module.Version{}, "", &ImportMissingSumError{
   310  			importPath: path,
   311  			mods:       sumErrMods,
   312  			found:      len(mods) > 0,
   313  		}
   314  	}
   315  	if len(mods) == 1 {
   316  		return mods[0], dirs[0], nil
   317  	}
   318  
   319  	var queryErr error
   320  	if !HasModRoot() {
   321  		queryErr = ErrNoModRoot
   322  	}
   323  	return module.Version{}, "", &ImportMissingError{Path: path, QueryErr: queryErr, isStd: pathIsStd}
   324  }
   325  
   326  // queryImport attempts to locate a module that can be added to the current
   327  // build list to provide the package with the given import path.
   328  //
   329  // Unlike QueryPattern, queryImport prefers to add a replaced version of a
   330  // module *before* checking the proxies for a version to add.
   331  func queryImport(ctx context.Context, path string) (module.Version, error) {
   332  	// To avoid spurious remote fetches, try the latest replacement for each
   333  	// module (golang.org/issue/26241).
   334  	if index != nil {
   335  		var mods []module.Version
   336  		for mp, mv := range index.highestReplaced {
   337  			if !maybeInModule(path, mp) {
   338  				continue
   339  			}
   340  			if mv == "" {
   341  				// The only replacement is a wildcard that doesn't specify a version, so
   342  				// synthesize a pseudo-version with an appropriate major version and a
   343  				// timestamp below any real timestamp. That way, if the main module is
   344  				// used from within some other module, the user will be able to upgrade
   345  				// the requirement to any real version they choose.
   346  				if _, pathMajor, ok := module.SplitPathVersion(mp); ok && len(pathMajor) > 0 {
   347  					mv = modfetch.ZeroPseudoVersion(pathMajor[1:])
   348  				} else {
   349  					mv = modfetch.ZeroPseudoVersion("v0")
   350  				}
   351  			}
   352  			mods = append(mods, module.Version{Path: mp, Version: mv})
   353  		}
   354  
   355  		// Every module path in mods is a prefix of the import path.
   356  		// As in QueryPattern, prefer the longest prefix that satisfies the import.
   357  		sort.Slice(mods, func(i, j int) bool {
   358  			return len(mods[i].Path) > len(mods[j].Path)
   359  		})
   360  		for _, m := range mods {
   361  			needSum := true
   362  			root, isLocal, err := fetch(ctx, m, needSum)
   363  			if err != nil {
   364  				if sumErr := (*sumMissingError)(nil); errors.As(err, &sumErr) {
   365  					return module.Version{}, &ImportMissingSumError{importPath: path}
   366  				}
   367  				return module.Version{}, err
   368  			}
   369  			if _, ok, err := dirInModule(path, m.Path, root, isLocal); err != nil {
   370  				return m, err
   371  			} else if ok {
   372  				if cfg.BuildMod == "readonly" {
   373  					return module.Version{}, &ImportMissingError{Path: path, replaced: m}
   374  				}
   375  				return m, nil
   376  			}
   377  		}
   378  		if len(mods) > 0 && module.CheckPath(path) != nil {
   379  			// The package path is not valid to fetch remotely,
   380  			// so it can only exist in a replaced module,
   381  			// and we know from the above loop that it is not.
   382  			return module.Version{}, &PackageNotInModuleError{
   383  				Mod:         mods[0],
   384  				Query:       "latest",
   385  				Pattern:     path,
   386  				Replacement: Replacement(mods[0]),
   387  			}
   388  		}
   389  	}
   390  
   391  	if search.IsStandardImportPath(path) {
   392  		// This package isn't in the standard library, isn't in any module already
   393  		// in the build list, and isn't in any other module that the user has
   394  		// shimmed in via a "replace" directive.
   395  		// Moreover, the import path is reserved for the standard library, so
   396  		// QueryPattern cannot possibly find a module containing this package.
   397  		//
   398  		// Instead of trying QueryPattern, report an ImportMissingError immediately.
   399  		return module.Version{}, &ImportMissingError{Path: path, isStd: true}
   400  	}
   401  
   402  	if cfg.BuildMod == "readonly" && !allowMissingModuleImports {
   403  		// In readonly mode, we can't write go.mod, so we shouldn't try to look up
   404  		// the module. If readonly mode was enabled explicitly, include that in
   405  		// the error message.
   406  		var queryErr error
   407  		if cfg.BuildModExplicit {
   408  			queryErr = fmt.Errorf("import lookup disabled by -mod=%s", cfg.BuildMod)
   409  		} else if cfg.BuildModReason != "" {
   410  			queryErr = fmt.Errorf("import lookup disabled by -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason)
   411  		}
   412  		return module.Version{}, &ImportMissingError{Path: path, QueryErr: queryErr}
   413  	}
   414  
   415  	// Look up module containing the package, for addition to the build list.
   416  	// Goal is to determine the module, download it to dir,
   417  	// and return m, dir, ImpportMissingError.
   418  	fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path)
   419  
   420  	candidates, err := QueryPackages(ctx, path, "latest", Selected, CheckAllowed)
   421  	if err != nil {
   422  		if errors.Is(err, fs.ErrNotExist) {
   423  			// Return "cannot find module providing package […]" instead of whatever
   424  			// low-level error QueryPattern produced.
   425  			return module.Version{}, &ImportMissingError{Path: path, QueryErr: err}
   426  		} else {
   427  			return module.Version{}, err
   428  		}
   429  	}
   430  
   431  	candidate0MissingVersion := ""
   432  	for i, c := range candidates {
   433  		cm := c.Mod
   434  		canAdd := true
   435  		for _, bm := range buildList {
   436  			if bm.Path == cm.Path && semver.Compare(bm.Version, cm.Version) > 0 {
   437  				// QueryPattern proposed that we add module cm to provide the package,
   438  				// but we already depend on a newer version of that module (and we don't
   439  				// have the package).
   440  				//
   441  				// This typically happens when a package is present at the "@latest"
   442  				// version (e.g., v1.0.0) of a module, but we have a newer version
   443  				// of the same module in the build list (e.g., v1.0.1-beta), and
   444  				// the package is not present there.
   445  				canAdd = false
   446  				if i == 0 {
   447  					candidate0MissingVersion = bm.Version
   448  				}
   449  				break
   450  			}
   451  		}
   452  		if canAdd {
   453  			return cm, nil
   454  		}
   455  	}
   456  	return module.Version{}, &ImportMissingError{
   457  		Path:              path,
   458  		Module:            candidates[0].Mod,
   459  		newMissingVersion: candidate0MissingVersion,
   460  	}
   461  }
   462  
   463  // maybeInModule reports whether, syntactically,
   464  // a package with the given import path could be supplied
   465  // by a module with the given module path (mpath).
   466  func maybeInModule(path, mpath string) bool {
   467  	return mpath == path ||
   468  		len(path) > len(mpath) && path[len(mpath)] == '/' && path[:len(mpath)] == mpath
   469  }
   470  
   471  var (
   472  	haveGoModCache   par.Cache // dir → bool
   473  	haveGoFilesCache par.Cache // dir → goFilesEntry
   474  )
   475  
   476  type goFilesEntry struct {
   477  	haveGoFiles bool
   478  	err         error
   479  }
   480  
   481  // dirInModule locates the directory that would hold the package named by the given path,
   482  // if it were in the module with module path mpath and root mdir.
   483  // If path is syntactically not within mpath,
   484  // or if mdir is a local file tree (isLocal == true) and the directory
   485  // that would hold path is in a sub-module (covered by a go.mod below mdir),
   486  // dirInModule returns "", false, nil.
   487  //
   488  // Otherwise, dirInModule returns the name of the directory where
   489  // Go source files would be expected, along with a boolean indicating
   490  // whether there are in fact Go source files in that directory.
   491  // A non-nil error indicates that the existence of the directory and/or
   492  // source files could not be determined, for example due to a permission error.
   493  func dirInModule(path, mpath, mdir string, isLocal bool) (dir string, haveGoFiles bool, err error) {
   494  	// Determine where to expect the package.
   495  	if path == mpath {
   496  		dir = mdir
   497  	} else if mpath == "" { // vendor directory
   498  		dir = filepath.Join(mdir, path)
   499  	} else if len(path) > len(mpath) && path[len(mpath)] == '/' && path[:len(mpath)] == mpath {
   500  		dir = filepath.Join(mdir, path[len(mpath)+1:])
   501  	} else {
   502  		return "", false, nil
   503  	}
   504  
   505  	// Check that there aren't other modules in the way.
   506  	// This check is unnecessary inside the module cache
   507  	// and important to skip in the vendor directory,
   508  	// where all the module trees have been overlaid.
   509  	// So we only check local module trees
   510  	// (the main module, and any directory trees pointed at by replace directives).
   511  	if isLocal {
   512  		for d := dir; d != mdir && len(d) > len(mdir); {
   513  			haveGoMod := haveGoModCache.Do(d, func() interface{} {
   514  				fi, err := fsys.Stat(filepath.Join(d, "go.mod"))
   515  				return err == nil && !fi.IsDir()
   516  			}).(bool)
   517  
   518  			if haveGoMod {
   519  				return "", false, nil
   520  			}
   521  			parent := filepath.Dir(d)
   522  			if parent == d {
   523  				// Break the loop, as otherwise we'd loop
   524  				// forever if d=="." and mdir=="".
   525  				break
   526  			}
   527  			d = parent
   528  		}
   529  	}
   530  
   531  	// Now committed to returning dir (not "").
   532  
   533  	// Are there Go source files in the directory?
   534  	// We don't care about build tags, not even "+build ignore".
   535  	// We're just looking for a plausible directory.
   536  	res := haveGoFilesCache.Do(dir, func() interface{} {
   537  		ok, err := fsys.IsDirWithGoFiles(dir)
   538  		return goFilesEntry{haveGoFiles: ok, err: err}
   539  	}).(goFilesEntry)
   540  
   541  	return dir, res.haveGoFiles, res.err
   542  }
   543  
   544  // fetch downloads the given module (or its replacement)
   545  // and returns its location.
   546  //
   547  // needSum indicates whether the module may be downloaded in readonly mode
   548  // without a go.sum entry. It should only be false for modules fetched
   549  // speculatively (for example, for incompatible version filtering). The sum
   550  // will still be verified normally.
   551  //
   552  // The isLocal return value reports whether the replacement,
   553  // if any, is local to the filesystem.
   554  func fetch(ctx context.Context, mod module.Version, needSum bool) (dir string, isLocal bool, err error) {
   555  	if mod == Target {
   556  		return ModRoot(), true, nil
   557  	}
   558  	if r := Replacement(mod); r.Path != "" {
   559  		if r.Version == "" {
   560  			dir = r.Path
   561  			if !filepath.IsAbs(dir) {
   562  				dir = filepath.Join(ModRoot(), dir)
   563  			}
   564  			// Ensure that the replacement directory actually exists:
   565  			// dirInModule does not report errors for missing modules,
   566  			// so if we don't report the error now, later failures will be
   567  			// very mysterious.
   568  			if _, err := fsys.Stat(dir); err != nil {
   569  				if os.IsNotExist(err) {
   570  					// Semantically the module version itself “exists” — we just don't
   571  					// have its source code. Remove the equivalence to os.ErrNotExist,
   572  					// and make the message more concise while we're at it.
   573  					err = fmt.Errorf("replacement directory %s does not exist", r.Path)
   574  				} else {
   575  					err = fmt.Errorf("replacement directory %s: %w", r.Path, err)
   576  				}
   577  				return dir, true, module.VersionError(mod, err)
   578  			}
   579  			return dir, true, nil
   580  		}
   581  		mod = r
   582  	}
   583  
   584  	if HasModRoot() && cfg.BuildMod == "readonly" && needSum && !modfetch.HaveSum(mod) {
   585  		return "", false, module.VersionError(mod, &sumMissingError{})
   586  	}
   587  
   588  	dir, err = modfetch.Download(ctx, mod)
   589  	return dir, false, err
   590  }
   591  
   592  type sumMissingError struct {
   593  	suggestion string
   594  }
   595  
   596  func (e *sumMissingError) Error() string {
   597  	return "missing go.sum entry" + e.suggestion
   598  }
   599
View as plain text