modfile.go

Documentation: cmd/go/internal/modload

     1  // Copyright 2020 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modload
     6  
     7  import (
     8  	"context"
     9  	"errors"
    10  	"fmt"
    11  	"path/filepath"
    12  	"strings"
    13  	"sync"
    14  	"unicode"
    15  
    16  	"cmd/go/internal/base"
    17  	"cmd/go/internal/cfg"
    18  	"cmd/go/internal/lockedfile"
    19  	"cmd/go/internal/modfetch"
    20  	"cmd/go/internal/par"
    21  	"cmd/go/internal/trace"
    22  
    23  	"golang.org/x/mod/modfile"
    24  	"golang.org/x/mod/module"
    25  	"golang.org/x/mod/semver"
    26  )
    27  
    28  // narrowAllVersionV is the Go version (plus leading "v") at which the
    29  // module-module "all" pattern no longer closes over the dependencies of
    30  // tests outside of the main module.
    31  const narrowAllVersionV = "v1.16"
    32  const go116EnableNarrowAll = true
    33  
    34  var modFile *modfile.File
    35  
    36  // A modFileIndex is an index of data corresponding to a modFile
    37  // at a specific point in time.
    38  type modFileIndex struct {
    39  	data            []byte
    40  	dataNeedsFix    bool // true if fixVersion applied a change while parsing data
    41  	module          module.Version
    42  	goVersionV      string // GoVersion with "v" prefix
    43  	require         map[module.Version]requireMeta
    44  	replace         map[module.Version]module.Version
    45  	highestReplaced map[string]string // highest replaced version of each module path; empty string for wildcard-only replacements
    46  	exclude         map[module.Version]bool
    47  }
    48  
    49  // index is the index of the go.mod file as of when it was last read or written.
    50  var index *modFileIndex
    51  
    52  type requireMeta struct {
    53  	indirect bool
    54  }
    55  
    56  // CheckAllowed returns an error equivalent to ErrDisallowed if m is excluded by
    57  // the main module's go.mod or retracted by its author. Most version queries use
    58  // this to filter out versions that should not be used.
    59  func CheckAllowed(ctx context.Context, m module.Version) error {
    60  	if err := CheckExclusions(ctx, m); err != nil {
    61  		return err
    62  	}
    63  	if err := CheckRetractions(ctx, m); err != nil {
    64  		return err
    65  	}
    66  	return nil
    67  }
    68  
    69  // ErrDisallowed is returned by version predicates passed to Query and similar
    70  // functions to indicate that a version should not be considered.
    71  var ErrDisallowed = errors.New("disallowed module version")
    72  
    73  // CheckExclusions returns an error equivalent to ErrDisallowed if module m is
    74  // excluded by the main module's go.mod file.
    75  func CheckExclusions(ctx context.Context, m module.Version) error {
    76  	if index != nil && index.exclude[m] {
    77  		return module.VersionError(m, errExcluded)
    78  	}
    79  	return nil
    80  }
    81  
    82  var errExcluded = &excludedError{}
    83  
    84  type excludedError struct{}
    85  
    86  func (e *excludedError) Error() string     { return "excluded by go.mod" }
    87  func (e *excludedError) Is(err error) bool { return err == ErrDisallowed }
    88  
    89  // CheckRetractions returns an error if module m has been retracted by
    90  // its author.
    91  func CheckRetractions(ctx context.Context, m module.Version) error {
    92  	if m.Version == "" {
    93  		// Main module, standard library, or file replacement module.
    94  		// Cannot be retracted.
    95  		return nil
    96  	}
    97  
    98  	// Look up retraction information from the latest available version of
    99  	// the module. Cache retraction information so we don't parse the go.mod
   100  	// file repeatedly.
   101  	type entry struct {
   102  		retract []retraction
   103  		err     error
   104  	}
   105  	path := m.Path
   106  	e := retractCache.Do(path, func() (v interface{}) {
   107  		ctx, span := trace.StartSpan(ctx, "checkRetractions "+path)
   108  		defer span.Done()
   109  
   110  		if repl := Replacement(module.Version{Path: m.Path}); repl.Path != "" {
   111  			// All versions of the module were replaced with a local directory.
   112  			// Don't load retractions.
   113  			return &entry{nil, nil}
   114  		}
   115  
   116  		// Find the latest version of the module.
   117  		// Ignore exclusions from the main module's go.mod.
   118  		const ignoreSelected = ""
   119  		var allowAll AllowedFunc
   120  		rev, err := Query(ctx, path, "latest", ignoreSelected, allowAll)
   121  		if err != nil {
   122  			return &entry{nil, err}
   123  		}
   124  
   125  		// Load go.mod for that version.
   126  		// If the version is replaced, we'll load retractions from the replacement.
   127  		//
   128  		// If there's an error loading the go.mod, we'll return it here.
   129  		// These errors should generally be ignored by callers of checkRetractions,
   130  		// since they happen frequently when we're offline. These errors are not
   131  		// equivalent to ErrDisallowed, so they may be distinguished from
   132  		// retraction errors.
   133  		//
   134  		// We load the raw file here: the go.mod file may have a different module
   135  		// path that we expect if the module or its repository was renamed.
   136  		// We still want to apply retractions to other aliases of the module.
   137  		rm := module.Version{Path: path, Version: rev.Version}
   138  		if repl := Replacement(rm); repl.Path != "" {
   139  			rm = repl
   140  		}
   141  		summary, err := rawGoModSummary(rm)
   142  		if err != nil {
   143  			return &entry{nil, err}
   144  		}
   145  		return &entry{summary.retract, nil}
   146  	}).(*entry)
   147  
   148  	if err := e.err; err != nil {
   149  		// Attribute the error to the version being checked, not the version from
   150  		// which the retractions were to be loaded.
   151  		var mErr *module.ModuleError
   152  		if errors.As(err, &mErr) {
   153  			err = mErr.Err
   154  		}
   155  		return &retractionLoadingError{m: m, err: err}
   156  	}
   157  
   158  	var rationale []string
   159  	isRetracted := false
   160  	for _, r := range e.retract {
   161  		if semver.Compare(r.Low, m.Version) <= 0 && semver.Compare(m.Version, r.High) <= 0 {
   162  			isRetracted = true
   163  			if r.Rationale != "" {
   164  				rationale = append(rationale, r.Rationale)
   165  			}
   166  		}
   167  	}
   168  	if isRetracted {
   169  		return module.VersionError(m, &ModuleRetractedError{Rationale: rationale})
   170  	}
   171  	return nil
   172  }
   173  
   174  var retractCache par.Cache
   175  
   176  type ModuleRetractedError struct {
   177  	Rationale []string
   178  }
   179  
   180  func (e *ModuleRetractedError) Error() string {
   181  	msg := "retracted by module author"
   182  	if len(e.Rationale) > 0 {
   183  		// This is meant to be a short error printed on a terminal, so just
   184  		// print the first rationale.
   185  		msg += ": " + ShortRetractionRationale(e.Rationale[0])
   186  	}
   187  	return msg
   188  }
   189  
   190  func (e *ModuleRetractedError) Is(err error) bool {
   191  	return err == ErrDisallowed
   192  }
   193  
   194  type retractionLoadingError struct {
   195  	m   module.Version
   196  	err error
   197  }
   198  
   199  func (e *retractionLoadingError) Error() string {
   200  	return fmt.Sprintf("loading module retractions for %v: %v", e.m, e.err)
   201  }
   202  
   203  func (e *retractionLoadingError) Unwrap() error {
   204  	return e.err
   205  }
   206  
   207  // ShortRetractionRationale returns a retraction rationale string that is safe
   208  // to print in a terminal. It returns hard-coded strings if the rationale
   209  // is empty, too long, or contains non-printable characters.
   210  func ShortRetractionRationale(rationale string) string {
   211  	const maxRationaleBytes = 500
   212  	if i := strings.Index(rationale, "\n"); i >= 0 {
   213  		rationale = rationale[:i]
   214  	}
   215  	rationale = strings.TrimSpace(rationale)
   216  	if rationale == "" {
   217  		return "retracted by module author"
   218  	}
   219  	if len(rationale) > maxRationaleBytes {
   220  		return "(rationale omitted: too long)"
   221  	}
   222  	for _, r := range rationale {
   223  		if !unicode.IsGraphic(r) && !unicode.IsSpace(r) {
   224  			return "(rationale omitted: contains non-printable characters)"
   225  		}
   226  	}
   227  	// NOTE: the go.mod parser rejects invalid UTF-8, so we don't check that here.
   228  	return rationale
   229  }
   230  
   231  // Replacement returns the replacement for mod, if any, from go.mod.
   232  // If there is no replacement for mod, Replacement returns
   233  // a module.Version with Path == "".
   234  func Replacement(mod module.Version) module.Version {
   235  	if index != nil {
   236  		if r, ok := index.replace[mod]; ok {
   237  			return r
   238  		}
   239  		if r, ok := index.replace[module.Version{Path: mod.Path}]; ok {
   240  			return r
   241  		}
   242  	}
   243  	return module.Version{}
   244  }
   245  
   246  // indexModFile rebuilds the index of modFile.
   247  // If modFile has been changed since it was first read,
   248  // modFile.Cleanup must be called before indexModFile.
   249  func indexModFile(data []byte, modFile *modfile.File, needsFix bool) *modFileIndex {
   250  	i := new(modFileIndex)
   251  	i.data = data
   252  	i.dataNeedsFix = needsFix
   253  
   254  	i.module = module.Version{}
   255  	if modFile.Module != nil {
   256  		i.module = modFile.Module.Mod
   257  	}
   258  
   259  	i.goVersionV = ""
   260  	if modFile.Go != nil {
   261  		// We're going to use the semver package to compare Go versions, so go ahead
   262  		// and add the "v" prefix it expects once instead of every time.
   263  		i.goVersionV = "v" + modFile.Go.Version
   264  	}
   265  
   266  	i.require = make(map[module.Version]requireMeta, len(modFile.Require))
   267  	for _, r := range modFile.Require {
   268  		i.require[r.Mod] = requireMeta{indirect: r.Indirect}
   269  	}
   270  
   271  	i.replace = make(map[module.Version]module.Version, len(modFile.Replace))
   272  	for _, r := range modFile.Replace {
   273  		if prev, dup := i.replace[r.Old]; dup && prev != r.New {
   274  			base.Fatalf("go: conflicting replacements for %v:\n\t%v\n\t%v", r.Old, prev, r.New)
   275  		}
   276  		i.replace[r.Old] = r.New
   277  	}
   278  
   279  	i.highestReplaced = make(map[string]string)
   280  	for _, r := range modFile.Replace {
   281  		v, ok := i.highestReplaced[r.Old.Path]
   282  		if !ok || semver.Compare(r.Old.Version, v) > 0 {
   283  			i.highestReplaced[r.Old.Path] = r.Old.Version
   284  		}
   285  	}
   286  
   287  	i.exclude = make(map[module.Version]bool, len(modFile.Exclude))
   288  	for _, x := range modFile.Exclude {
   289  		i.exclude[x.Mod] = true
   290  	}
   291  
   292  	return i
   293  }
   294  
   295  // allPatternClosesOverTests reports whether the "all" pattern includes
   296  // dependencies of tests outside the main module (as in Go 1.11–1.15).
   297  // (Otherwise — as in Go 1.16+ — the "all" pattern includes only the packages
   298  // transitively *imported by* the packages and tests in the main module.)
   299  func (i *modFileIndex) allPatternClosesOverTests() bool {
   300  	if !go116EnableNarrowAll {
   301  		return true
   302  	}
   303  	if i != nil && semver.Compare(i.goVersionV, narrowAllVersionV) < 0 {
   304  		// The module explicitly predates the change in "all" for lazy loading, so
   305  		// continue to use the older interpretation. (If i == nil, we not in any
   306  		// module at all and should use the latest semantics.)
   307  		return true
   308  	}
   309  	return false
   310  }
   311  
   312  // modFileIsDirty reports whether the go.mod file differs meaningfully
   313  // from what was indexed.
   314  // If modFile has been changed (even cosmetically) since it was first read,
   315  // modFile.Cleanup must be called before modFileIsDirty.
   316  func (i *modFileIndex) modFileIsDirty(modFile *modfile.File) bool {
   317  	if i == nil {
   318  		return modFile != nil
   319  	}
   320  
   321  	if i.dataNeedsFix {
   322  		return true
   323  	}
   324  
   325  	if modFile.Module == nil {
   326  		if i.module != (module.Version{}) {
   327  			return true
   328  		}
   329  	} else if modFile.Module.Mod != i.module {
   330  		return true
   331  	}
   332  
   333  	if modFile.Go == nil {
   334  		if i.goVersionV != "" {
   335  			return true
   336  		}
   337  	} else if "v"+modFile.Go.Version != i.goVersionV {
   338  		if i.goVersionV == "" && cfg.BuildMod == "readonly" {
   339  			// go.mod files did not always require a 'go' version, so do not error out
   340  			// if one is missing — we may be inside an older module in the module
   341  			// cache, and should bias toward providing useful behavior.
   342  		} else {
   343  			return true
   344  		}
   345  	}
   346  
   347  	if len(modFile.Require) != len(i.require) ||
   348  		len(modFile.Replace) != len(i.replace) ||
   349  		len(modFile.Exclude) != len(i.exclude) {
   350  		return true
   351  	}
   352  
   353  	for _, r := range modFile.Require {
   354  		if meta, ok := i.require[r.Mod]; !ok {
   355  			return true
   356  		} else if r.Indirect != meta.indirect {
   357  			if cfg.BuildMod == "readonly" {
   358  				// The module's requirements are consistent; only the "// indirect"
   359  				// comments that are wrong. But those are only guaranteed to be accurate
   360  				// after a "go mod tidy" — it's a good idea to run those before
   361  				// committing a change, but it's certainly not mandatory.
   362  			} else {
   363  				return true
   364  			}
   365  		}
   366  	}
   367  
   368  	for _, r := range modFile.Replace {
   369  		if r.New != i.replace[r.Old] {
   370  			return true
   371  		}
   372  	}
   373  
   374  	for _, x := range modFile.Exclude {
   375  		if !i.exclude[x.Mod] {
   376  			return true
   377  		}
   378  	}
   379  
   380  	return false
   381  }
   382  
   383  // rawGoVersion records the Go version parsed from each module's go.mod file.
   384  //
   385  // If a module is replaced, the version of the replacement is keyed by the
   386  // replacement module.Version, not the version being replaced.
   387  var rawGoVersion sync.Map // map[module.Version]string
   388  
   389  // A modFileSummary is a summary of a go.mod file for which we do not need to
   390  // retain complete information — for example, the go.mod file of a dependency
   391  // module.
   392  type modFileSummary struct {
   393  	module     module.Version
   394  	goVersionV string // GoVersion with "v" prefix
   395  	require    []module.Version
   396  	retract    []retraction
   397  }
   398  
   399  // A retraction consists of a retracted version interval and rationale.
   400  // retraction is like modfile.Retract, but it doesn't point to the syntax tree.
   401  type retraction struct {
   402  	modfile.VersionInterval
   403  	Rationale string
   404  }
   405  
   406  // goModSummary returns a summary of the go.mod file for module m,
   407  // taking into account any replacements for m, exclusions of its dependencies,
   408  // and/or vendoring.
   409  //
   410  // m must be a version in the module graph, reachable from the Target module.
   411  // In readonly mode, the go.sum file must contain an entry for m's go.mod file
   412  // (or its replacement). goModSummary must not be called for the Target module
   413  // itself, as its requirements may change. Use rawGoModSummary for other
   414  // module versions.
   415  //
   416  // The caller must not modify the returned summary.
   417  func goModSummary(m module.Version) (*modFileSummary, error) {
   418  	if m == Target {
   419  		panic("internal error: goModSummary called on the Target module")
   420  	}
   421  
   422  	if cfg.BuildMod == "vendor" {
   423  		summary := &modFileSummary{
   424  			module: module.Version{Path: m.Path},
   425  		}
   426  		if vendorVersion[m.Path] != m.Version {
   427  			// This module is not vendored, so packages cannot be loaded from it and
   428  			// it cannot be relevant to the build.
   429  			return summary, nil
   430  		}
   431  
   432  		// For every module other than the target,
   433  		// return the full list of modules from modules.txt.
   434  		readVendorList()
   435  
   436  		// TODO(#36876): Load the "go" version from vendor/modules.txt and store it
   437  		// in rawGoVersion with the appropriate key.
   438  
   439  		// We don't know what versions the vendored module actually relies on,
   440  		// so assume that it requires everything.
   441  		summary.require = vendorList
   442  		return summary, nil
   443  	}
   444  
   445  	actual := Replacement(m)
   446  	if actual.Path == "" {
   447  		actual = m
   448  	}
   449  	if HasModRoot() && cfg.BuildMod == "readonly" && actual.Version != "" {
   450  		key := module.Version{Path: actual.Path, Version: actual.Version + "/go.mod"}
   451  		if !modfetch.HaveSum(key) {
   452  			suggestion := fmt.Sprintf("; to add it:\n\tgo mod download %s", m.Path)
   453  			return nil, module.VersionError(actual, &sumMissingError{suggestion: suggestion})
   454  		}
   455  	}
   456  	summary, err := rawGoModSummary(actual)
   457  	if err != nil {
   458  		return nil, err
   459  	}
   460  
   461  	if actual.Version == "" {
   462  		// The actual module is a filesystem-local replacement, for which we have
   463  		// unfortunately not enforced any sort of invariants about module lines or
   464  		// matching module paths. Anything goes.
   465  		//
   466  		// TODO(bcmills): Remove this special-case, update tests, and add a
   467  		// release note.
   468  	} else {
   469  		if summary.module.Path == "" {
   470  			return nil, module.VersionError(actual, errors.New("parsing go.mod: missing module line"))
   471  		}
   472  
   473  		// In theory we should only allow mpath to be unequal to m.Path here if the
   474  		// version that we fetched lacks an explicit go.mod file: if the go.mod file
   475  		// is explicit, then it should match exactly (to ensure that imports of other
   476  		// packages within the module are interpreted correctly). Unfortunately, we
   477  		// can't determine that information from the module proxy protocol: we'll have
   478  		// to leave that validation for when we load actual packages from within the
   479  		// module.
   480  		if mpath := summary.module.Path; mpath != m.Path && mpath != actual.Path {
   481  			return nil, module.VersionError(actual, fmt.Errorf(`parsing go.mod:
   482  	module declares its path as: %s
   483  	        but was required as: %s`, mpath, m.Path))
   484  		}
   485  	}
   486  
   487  	if index != nil && len(index.exclude) > 0 {
   488  		// Drop any requirements on excluded versions.
   489  		// Don't modify the cached summary though, since we might need the raw
   490  		// summary separately.
   491  		haveExcludedReqs := false
   492  		for _, r := range summary.require {
   493  			if index.exclude[r] {
   494  				haveExcludedReqs = true
   495  				break
   496  			}
   497  		}
   498  		if haveExcludedReqs {
   499  			s := new(modFileSummary)
   500  			*s = *summary
   501  			s.require = make([]module.Version, 0, len(summary.require))
   502  			for _, r := range summary.require {
   503  				if !index.exclude[r] {
   504  					s.require = append(s.require, r)
   505  				}
   506  			}
   507  			summary = s
   508  		}
   509  	}
   510  	return summary, nil
   511  }
   512  
   513  // rawGoModSummary returns a new summary of the go.mod file for module m,
   514  // ignoring all replacements that may apply to m and excludes that may apply to
   515  // its dependencies.
   516  //
   517  // rawGoModSummary cannot be used on the Target module.
   518  func rawGoModSummary(m module.Version) (*modFileSummary, error) {
   519  	if m == Target {
   520  		panic("internal error: rawGoModSummary called on the Target module")
   521  	}
   522  
   523  	type cached struct {
   524  		summary *modFileSummary
   525  		err     error
   526  	}
   527  	c := rawGoModSummaryCache.Do(m, func() interface{} {
   528  		summary := new(modFileSummary)
   529  		var f *modfile.File
   530  		if m.Version == "" {
   531  			// m is a replacement module with only a file path.
   532  			dir := m.Path
   533  			if !filepath.IsAbs(dir) {
   534  				dir = filepath.Join(ModRoot(), dir)
   535  			}
   536  			gomod := filepath.Join(dir, "go.mod")
   537  
   538  			data, err := lockedfile.Read(gomod)
   539  			if err != nil {
   540  				return cached{nil, module.VersionError(m, fmt.Errorf("reading %s: %v", base.ShortPath(gomod), err))}
   541  			}
   542  			f, err = modfile.ParseLax(gomod, data, nil)
   543  			if err != nil {
   544  				return cached{nil, module.VersionError(m, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err))}
   545  			}
   546  		} else {
   547  			if !semver.IsValid(m.Version) {
   548  				// Disallow the broader queries supported by fetch.Lookup.
   549  				base.Fatalf("go: internal error: %s@%s: unexpected invalid semantic version", m.Path, m.Version)
   550  			}
   551  
   552  			data, err := modfetch.GoMod(m.Path, m.Version)
   553  			if err != nil {
   554  				return cached{nil, err}
   555  			}
   556  			f, err = modfile.ParseLax("go.mod", data, nil)
   557  			if err != nil {
   558  				return cached{nil, module.VersionError(m, fmt.Errorf("parsing go.mod: %v", err))}
   559  			}
   560  		}
   561  
   562  		if f.Module != nil {
   563  			summary.module = f.Module.Mod
   564  		}
   565  		if f.Go != nil && f.Go.Version != "" {
   566  			rawGoVersion.LoadOrStore(m, f.Go.Version)
   567  			summary.goVersionV = "v" + f.Go.Version
   568  		}
   569  		if len(f.Require) > 0 {
   570  			summary.require = make([]module.Version, 0, len(f.Require))
   571  			for _, req := range f.Require {
   572  				summary.require = append(summary.require, req.Mod)
   573  			}
   574  		}
   575  		if len(f.Retract) > 0 {
   576  			summary.retract = make([]retraction, 0, len(f.Retract))
   577  			for _, ret := range f.Retract {
   578  				summary.retract = append(summary.retract, retraction{
   579  					VersionInterval: ret.VersionInterval,
   580  					Rationale:       ret.Rationale,
   581  				})
   582  			}
   583  		}
   584  
   585  		return cached{summary, nil}
   586  	}).(cached)
   587  
   588  	return c.summary, c.err
   589  }
   590  
   591  var rawGoModSummaryCache par.Cache // module.Version → rawGoModSummary result
   592
View as plain text