Source file src/cmd/go/internal/modindex/read.go

     1  // Copyright 2022 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modindex
     6  
     7  import (
     8  	"bytes"
     9  	"encoding/binary"
    10  	"errors"
    11  	"fmt"
    12  	"go/build"
    13  	"go/build/constraint"
    14  	"go/token"
    15  	"internal/godebug"
    16  	"internal/goroot"
    17  	"path"
    18  	"path/filepath"
    19  	"runtime"
    20  	"runtime/debug"
    21  	"sort"
    22  	"strings"
    23  	"sync"
    24  	"time"
    25  	"unsafe"
    26  
    27  	"cmd/go/internal/base"
    28  	"cmd/go/internal/cache"
    29  	"cmd/go/internal/cfg"
    30  	"cmd/go/internal/fsys"
    31  	"cmd/go/internal/imports"
    32  	"cmd/go/internal/par"
    33  	"cmd/go/internal/str"
    34  )
    35  
    36  // enabled is used to flag off the behavior of the module index on tip.
    37  // It will be removed before the release.
    38  // TODO(matloob): Remove enabled once we have more confidence on the
    39  // module index.
    40  var enabled = godebug.New("#goindex").Value() != "0"
    41  
    42  // Module represents and encoded module index file. It is used to
    43  // do the equivalent of build.Import of packages in the module and answer other
    44  // questions based on the index file's data.
    45  type Module struct {
    46  	modroot string
    47  	d       *decoder
    48  	n       int // number of packages
    49  }
    50  
    51  // moduleHash returns an ActionID corresponding to the state of the module
    52  // located at filesystem path modroot.
    53  func moduleHash(modroot string, ismodcache bool) (cache.ActionID, error) {
    54  	// We expect modules stored within the module cache to be checksummed and
    55  	// immutable, and we expect released modules within GOROOT to change only
    56  	// infrequently (when the Go version changes).
    57  	if !ismodcache {
    58  		// The contents of this module may change over time. We don't want to pay
    59  		// the cost to detect changes and re-index whenever they occur, so just
    60  		// don't index it at all.
    61  		//
    62  		// Note that this is true even for modules in GOROOT/src: non-release builds
    63  		// of the Go toolchain may have arbitrary development changes on top of the
    64  		// commit reported by runtime.Version, or could be completely artificial due
    65  		// to lacking a `git` binary (like "devel gomote.XXXXX", as synthesized by
    66  		// "gomote push" as of 2022-06-15). (Release builds shouldn't have
    67  		// modifications, but we don't want to use a behavior for releases that we
    68  		// haven't tested during development.)
    69  		return cache.ActionID{}, ErrNotIndexed
    70  	}
    71  
    72  	h := cache.NewHash("moduleIndex")
    73  	// TODO(bcmills): Since modules in the index are checksummed, we could
    74  	// probably improve the cache hit rate by keying off of the module
    75  	// path@version (perhaps including the checksum?) instead of the module root
    76  	// directory.
    77  	fmt.Fprintf(h, "module index %s %s %v\n", runtime.Version(), indexVersion, modroot)
    78  	return h.Sum(), nil
    79  }
    80  
    81  const modTimeCutoff = 2 * time.Second
    82  
    83  // dirHash returns an ActionID corresponding to the state of the package
    84  // located at filesystem path pkgdir.
    85  func dirHash(modroot, pkgdir string) (cache.ActionID, error) {
    86  	h := cache.NewHash("moduleIndex")
    87  	fmt.Fprintf(h, "modroot %s\n", modroot)
    88  	fmt.Fprintf(h, "package %s %s %v\n", runtime.Version(), indexVersion, pkgdir)
    89  	entries, err := fsys.ReadDir(pkgdir)
    90  	if err != nil {
    91  		// pkgdir might not be a directory. give up on hashing.
    92  		return cache.ActionID{}, ErrNotIndexed
    93  	}
    94  	cutoff := time.Now().Add(-modTimeCutoff)
    95  	for _, info := range entries {
    96  		if info.IsDir() {
    97  			continue
    98  		}
    99  
   100  		if !info.Mode().IsRegular() {
   101  			return cache.ActionID{}, ErrNotIndexed
   102  		}
   103  		// To avoid problems for very recent files where a new
   104  		// write might not change the mtime due to file system
   105  		// mtime precision, reject caching if a file was read that
   106  		// is less than modTimeCutoff old.
   107  		//
   108  		// This is the same strategy used for hashing test inputs.
   109  		// See hashOpen in cmd/go/internal/test/test.go for the
   110  		// corresponding code.
   111  		if info.ModTime().After(cutoff) {
   112  			return cache.ActionID{}, ErrNotIndexed
   113  		}
   114  
   115  		fmt.Fprintf(h, "file %v %v %v\n", info.Name(), info.ModTime(), info.Size())
   116  	}
   117  	return h.Sum(), nil
   118  }
   119  
   120  var ErrNotIndexed = errors.New("not in module index")
   121  
   122  var (
   123  	errDisabled           = fmt.Errorf("%w: module indexing disabled", ErrNotIndexed)
   124  	errNotFromModuleCache = fmt.Errorf("%w: not from module cache", ErrNotIndexed)
   125  )
   126  
   127  // GetPackage returns the IndexPackage for the package at the given path.
   128  // It will return ErrNotIndexed if the directory should be read without
   129  // using the index, for instance because the index is disabled, or the package
   130  // is not in a module.
   131  func GetPackage(modroot, pkgdir string) (*IndexPackage, error) {
   132  	mi, err := GetModule(modroot)
   133  	if err == nil {
   134  		return mi.Package(relPath(pkgdir, modroot)), nil
   135  	}
   136  	if !errors.Is(err, errNotFromModuleCache) {
   137  		return nil, err
   138  	}
   139  	if cfg.BuildContext.Compiler == "gccgo" && str.HasPathPrefix(modroot, cfg.GOROOTsrc) {
   140  		return nil, err // gccgo has no sources for GOROOT packages.
   141  	}
   142  	return openIndexPackage(modroot, pkgdir)
   143  }
   144  
   145  // GetModule returns the Module for the given modroot.
   146  // It will return ErrNotIndexed if the directory should be read without
   147  // using the index, for instance because the index is disabled, or the package
   148  // is not in a module.
   149  func GetModule(modroot string) (*Module, error) {
   150  	if !enabled || cache.DefaultDir() == "off" {
   151  		return nil, errDisabled
   152  	}
   153  	if modroot == "" {
   154  		panic("modindex.GetPackage called with empty modroot")
   155  	}
   156  	if cfg.BuildMod == "vendor" {
   157  		// Even if the main module is in the module cache,
   158  		// its vendored dependencies are not loaded from their
   159  		// usual cached locations.
   160  		return nil, errNotFromModuleCache
   161  	}
   162  	modroot = filepath.Clean(modroot)
   163  	if str.HasFilePathPrefix(modroot, cfg.GOROOTsrc) || !str.HasFilePathPrefix(modroot, cfg.GOMODCACHE) {
   164  		return nil, errNotFromModuleCache
   165  	}
   166  	return openIndexModule(modroot, true)
   167  }
   168  
   169  var mcache par.ErrCache[string, *Module]
   170  
   171  // openIndexModule returns the module index for modPath.
   172  // It will return ErrNotIndexed if the module can not be read
   173  // using the index because it contains symlinks.
   174  func openIndexModule(modroot string, ismodcache bool) (*Module, error) {
   175  	return mcache.Do(modroot, func() (*Module, error) {
   176  		fsys.Trace("openIndexModule", modroot)
   177  		id, err := moduleHash(modroot, ismodcache)
   178  		if err != nil {
   179  			return nil, err
   180  		}
   181  		data, _, err := cache.GetMmap(cache.Default(), id)
   182  		if err != nil {
   183  			// Couldn't read from modindex. Assume we couldn't read from
   184  			// the index because the module hasn't been indexed yet.
   185  			data, err = indexModule(modroot)
   186  			if err != nil {
   187  				return nil, err
   188  			}
   189  			if err = cache.PutBytes(cache.Default(), id, data); err != nil {
   190  				return nil, err
   191  			}
   192  		}
   193  		mi, err := fromBytes(modroot, data)
   194  		if err != nil {
   195  			return nil, err
   196  		}
   197  		return mi, nil
   198  	})
   199  }
   200  
   201  var pcache par.ErrCache[[2]string, *IndexPackage]
   202  
   203  func openIndexPackage(modroot, pkgdir string) (*IndexPackage, error) {
   204  	return pcache.Do([2]string{modroot, pkgdir}, func() (*IndexPackage, error) {
   205  		fsys.Trace("openIndexPackage", pkgdir)
   206  		id, err := dirHash(modroot, pkgdir)
   207  		if err != nil {
   208  			return nil, err
   209  		}
   210  		data, _, err := cache.GetMmap(cache.Default(), id)
   211  		if err != nil {
   212  			// Couldn't read from index. Assume we couldn't read from
   213  			// the index because the package hasn't been indexed yet.
   214  			data = indexPackage(modroot, pkgdir)
   215  			if err = cache.PutBytes(cache.Default(), id, data); err != nil {
   216  				return nil, err
   217  			}
   218  		}
   219  		pkg, err := packageFromBytes(modroot, data)
   220  		if err != nil {
   221  			return nil, err
   222  		}
   223  		return pkg, nil
   224  	})
   225  }
   226  
   227  var errCorrupt = errors.New("corrupt index")
   228  
   229  // protect marks the start of a large section of code that accesses the index.
   230  // It should be used as:
   231  //
   232  //	defer unprotect(protect, &err)
   233  //
   234  // It should not be used for trivial accesses which would be
   235  // dwarfed by the overhead of the defer.
   236  func protect() bool {
   237  	return debug.SetPanicOnFault(true)
   238  }
   239  
   240  var isTest = false
   241  
   242  // unprotect marks the end of a large section of code that accesses the index.
   243  // It should be used as:
   244  //
   245  //	defer unprotect(protect, &err)
   246  //
   247  // end looks for panics due to errCorrupt or bad mmap accesses.
   248  // When it finds them, it adds explanatory text, consumes the panic, and sets *errp instead.
   249  // If errp is nil, end adds the explanatory text but then calls base.Fatalf.
   250  func unprotect(old bool, errp *error) {
   251  	// SetPanicOnFault's errors _may_ satisfy this interface. Even though it's not guaranteed
   252  	// that all its errors satisfy this interface, we'll only check for these errors so that
   253  	// we don't suppress panics that could have been produced from other sources.
   254  	type addrer interface {
   255  		Addr() uintptr
   256  	}
   257  
   258  	debug.SetPanicOnFault(old)
   259  
   260  	if e := recover(); e != nil {
   261  		if _, ok := e.(addrer); ok || e == errCorrupt {
   262  			// This panic was almost certainly caused by SetPanicOnFault or our panic(errCorrupt).
   263  			err := fmt.Errorf("error reading module index: %v", e)
   264  			if errp != nil {
   265  				*errp = err
   266  				return
   267  			}
   268  			if isTest {
   269  				panic(err)
   270  			}
   271  			base.Fatalf("%v", err)
   272  		}
   273  		// The panic was likely not caused by SetPanicOnFault.
   274  		panic(e)
   275  	}
   276  }
   277  
   278  // fromBytes returns a *Module given the encoded representation.
   279  func fromBytes(moddir string, data []byte) (m *Module, err error) {
   280  	if !enabled {
   281  		panic("use of index")
   282  	}
   283  
   284  	defer unprotect(protect(), &err)
   285  
   286  	if !bytes.HasPrefix(data, []byte(indexVersion+"\n")) {
   287  		return nil, errCorrupt
   288  	}
   289  
   290  	const hdr = len(indexVersion + "\n")
   291  	d := &decoder{data: data}
   292  	str := d.intAt(hdr)
   293  	if str < hdr+8 || len(d.data) < str {
   294  		return nil, errCorrupt
   295  	}
   296  	d.data, d.str = data[:str], d.data[str:]
   297  	// Check that string table looks valid.
   298  	// First string is empty string (length 0),
   299  	// and we leave a marker byte 0xFF at the end
   300  	// just to make sure that the file is not truncated.
   301  	if len(d.str) == 0 || d.str[0] != 0 || d.str[len(d.str)-1] != 0xFF {
   302  		return nil, errCorrupt
   303  	}
   304  
   305  	n := d.intAt(hdr + 4)
   306  	if n < 0 || n > (len(d.data)-8)/8 {
   307  		return nil, errCorrupt
   308  	}
   309  
   310  	m = &Module{
   311  		moddir,
   312  		d,
   313  		n,
   314  	}
   315  	return m, nil
   316  }
   317  
   318  // packageFromBytes returns a *IndexPackage given the encoded representation.
   319  func packageFromBytes(modroot string, data []byte) (p *IndexPackage, err error) {
   320  	m, err := fromBytes(modroot, data)
   321  	if err != nil {
   322  		return nil, err
   323  	}
   324  	if m.n != 1 {
   325  		return nil, fmt.Errorf("corrupt single-package index")
   326  	}
   327  	return m.pkg(0), nil
   328  }
   329  
   330  // pkgDir returns the dir string of the i'th package in the index.
   331  func (m *Module) pkgDir(i int) string {
   332  	if i < 0 || i >= m.n {
   333  		panic(errCorrupt)
   334  	}
   335  	return m.d.stringAt(12 + 8 + 8*i)
   336  }
   337  
   338  // pkgOff returns the offset of the data for the i'th package in the index.
   339  func (m *Module) pkgOff(i int) int {
   340  	if i < 0 || i >= m.n {
   341  		panic(errCorrupt)
   342  	}
   343  	return m.d.intAt(12 + 8 + 8*i + 4)
   344  }
   345  
   346  // Walk calls f for each package in the index, passing the path to that package relative to the module root.
   347  func (m *Module) Walk(f func(path string)) {
   348  	defer unprotect(protect(), nil)
   349  	for i := 0; i < m.n; i++ {
   350  		f(m.pkgDir(i))
   351  	}
   352  }
   353  
   354  // relPath returns the path relative to the module's root.
   355  func relPath(path, modroot string) string {
   356  	return str.TrimFilePathPrefix(filepath.Clean(path), filepath.Clean(modroot))
   357  }
   358  
   359  var installgorootAll = godebug.New("installgoroot").Value() == "all"
   360  
   361  // Import is the equivalent of build.Import given the information in Module.
   362  func (rp *IndexPackage) Import(bctxt build.Context, mode build.ImportMode) (p *build.Package, err error) {
   363  	defer unprotect(protect(), &err)
   364  
   365  	ctxt := (*Context)(&bctxt)
   366  
   367  	p = &build.Package{}
   368  
   369  	p.ImportPath = "."
   370  	p.Dir = filepath.Join(rp.modroot, rp.dir)
   371  
   372  	var pkgerr error
   373  	switch ctxt.Compiler {
   374  	case "gccgo", "gc":
   375  	default:
   376  		// Save error for end of function.
   377  		pkgerr = fmt.Errorf("import %q: unknown compiler %q", p.Dir, ctxt.Compiler)
   378  	}
   379  
   380  	if p.Dir == "" {
   381  		return p, fmt.Errorf("import %q: import of unknown directory", p.Dir)
   382  	}
   383  
   384  	// goroot and gopath
   385  	inTestdata := func(sub string) bool {
   386  		return strings.Contains(sub, "/testdata/") || strings.HasSuffix(sub, "/testdata") || str.HasPathPrefix(sub, "testdata")
   387  	}
   388  	var pkga string
   389  	if !inTestdata(rp.dir) {
   390  		// In build.go, p.Root should only be set in the non-local-import case, or in
   391  		// GOROOT or GOPATH. Since module mode only calls Import with path set to "."
   392  		// and the module index doesn't apply outside modules, the GOROOT case is
   393  		// the only case where p.Root needs to be set.
   394  		if ctxt.GOROOT != "" && str.HasFilePathPrefix(p.Dir, cfg.GOROOTsrc) && p.Dir != cfg.GOROOTsrc {
   395  			p.Root = ctxt.GOROOT
   396  			p.Goroot = true
   397  			modprefix := str.TrimFilePathPrefix(rp.modroot, cfg.GOROOTsrc)
   398  			p.ImportPath = rp.dir
   399  			if modprefix != "" {
   400  				p.ImportPath = filepath.Join(modprefix, p.ImportPath)
   401  			}
   402  
   403  			// Set GOROOT-specific fields (sometimes for modules in a GOPATH directory).
   404  			// The fields set below (SrcRoot, PkgRoot, BinDir, PkgTargetRoot, and PkgObj)
   405  			// are only set in build.Import if p.Root != "".
   406  			var pkgtargetroot string
   407  			suffix := ""
   408  			if ctxt.InstallSuffix != "" {
   409  				suffix = "_" + ctxt.InstallSuffix
   410  			}
   411  			switch ctxt.Compiler {
   412  			case "gccgo":
   413  				pkgtargetroot = "pkg/gccgo_" + ctxt.GOOS + "_" + ctxt.GOARCH + suffix
   414  				dir, elem := path.Split(p.ImportPath)
   415  				pkga = pkgtargetroot + "/" + dir + "lib" + elem + ".a"
   416  			case "gc":
   417  				pkgtargetroot = "pkg/" + ctxt.GOOS + "_" + ctxt.GOARCH + suffix
   418  				pkga = pkgtargetroot + "/" + p.ImportPath + ".a"
   419  			}
   420  			p.SrcRoot = ctxt.joinPath(p.Root, "src")
   421  			p.PkgRoot = ctxt.joinPath(p.Root, "pkg")
   422  			p.BinDir = ctxt.joinPath(p.Root, "bin")
   423  			if pkga != "" {
   424  				// Always set PkgTargetRoot. It might be used when building in shared
   425  				// mode.
   426  				p.PkgTargetRoot = ctxt.joinPath(p.Root, pkgtargetroot)
   427  
   428  				// Set the install target if applicable.
   429  				if !p.Goroot || (installgorootAll && p.ImportPath != "unsafe" && p.ImportPath != "builtin") {
   430  					p.PkgObj = ctxt.joinPath(p.Root, pkga)
   431  				}
   432  			}
   433  		}
   434  	}
   435  
   436  	if rp.error != nil {
   437  		if errors.Is(rp.error, errCannotFindPackage) && ctxt.Compiler == "gccgo" && p.Goroot {
   438  			return p, nil
   439  		}
   440  		return p, rp.error
   441  	}
   442  
   443  	if mode&build.FindOnly != 0 {
   444  		return p, pkgerr
   445  	}
   446  
   447  	// We need to do a second round of bad file processing.
   448  	var badGoError error
   449  	badGoFiles := make(map[string]bool)
   450  	badGoFile := func(name string, err error) {
   451  		if badGoError == nil {
   452  			badGoError = err
   453  		}
   454  		if !badGoFiles[name] {
   455  			p.InvalidGoFiles = append(p.InvalidGoFiles, name)
   456  			badGoFiles[name] = true
   457  		}
   458  	}
   459  
   460  	var Sfiles []string // files with ".S"(capital S)/.sx(capital s equivalent for case insensitive filesystems)
   461  	var firstFile string
   462  	embedPos := make(map[string][]token.Position)
   463  	testEmbedPos := make(map[string][]token.Position)
   464  	xTestEmbedPos := make(map[string][]token.Position)
   465  	importPos := make(map[string][]token.Position)
   466  	testImportPos := make(map[string][]token.Position)
   467  	xTestImportPos := make(map[string][]token.Position)
   468  	allTags := make(map[string]bool)
   469  	for _, tf := range rp.sourceFiles {
   470  		name := tf.name()
   471  		// Check errors for go files and call badGoFiles to put them in
   472  		// InvalidGoFiles if they do have an error.
   473  		if strings.HasSuffix(name, ".go") {
   474  			if error := tf.error(); error != "" {
   475  				badGoFile(name, errors.New(tf.error()))
   476  				continue
   477  			} else if parseError := tf.parseError(); parseError != "" {
   478  				badGoFile(name, parseErrorFromString(tf.parseError()))
   479  				// Fall through: we still want to list files with parse errors.
   480  			}
   481  		}
   482  
   483  		var shouldBuild = true
   484  		if !ctxt.goodOSArchFile(name, allTags) && !ctxt.UseAllFiles {
   485  			shouldBuild = false
   486  		} else if goBuildConstraint := tf.goBuildConstraint(); goBuildConstraint != "" {
   487  			x, err := constraint.Parse(goBuildConstraint)
   488  			if err != nil {
   489  				return p, fmt.Errorf("%s: parsing //go:build line: %v", name, err)
   490  			}
   491  			shouldBuild = ctxt.eval(x, allTags)
   492  		} else if plusBuildConstraints := tf.plusBuildConstraints(); len(plusBuildConstraints) > 0 {
   493  			for _, text := range plusBuildConstraints {
   494  				if x, err := constraint.Parse(text); err == nil {
   495  					if !ctxt.eval(x, allTags) {
   496  						shouldBuild = false
   497  					}
   498  				}
   499  			}
   500  		}
   501  
   502  		ext := nameExt(name)
   503  		if !shouldBuild || tf.ignoreFile() {
   504  			if ext == ".go" {
   505  				p.IgnoredGoFiles = append(p.IgnoredGoFiles, name)
   506  			} else if fileListForExt(p, ext) != nil {
   507  				p.IgnoredOtherFiles = append(p.IgnoredOtherFiles, name)
   508  			}
   509  			continue
   510  		}
   511  
   512  		// Going to save the file. For non-Go files, can stop here.
   513  		switch ext {
   514  		case ".go":
   515  			// keep going
   516  		case ".S", ".sx":
   517  			// special case for cgo, handled at end
   518  			Sfiles = append(Sfiles, name)
   519  			continue
   520  		default:
   521  			if list := fileListForExt(p, ext); list != nil {
   522  				*list = append(*list, name)
   523  			}
   524  			continue
   525  		}
   526  
   527  		pkg := tf.pkgName()
   528  		if pkg == "documentation" {
   529  			p.IgnoredGoFiles = append(p.IgnoredGoFiles, name)
   530  			continue
   531  		}
   532  		isTest := strings.HasSuffix(name, "_test.go")
   533  		isXTest := false
   534  		if isTest && strings.HasSuffix(tf.pkgName(), "_test") && p.Name != tf.pkgName() {
   535  			isXTest = true
   536  			pkg = pkg[:len(pkg)-len("_test")]
   537  		}
   538  
   539  		if !isTest && tf.binaryOnly() {
   540  			p.BinaryOnly = true
   541  		}
   542  
   543  		if p.Name == "" {
   544  			p.Name = pkg
   545  			firstFile = name
   546  		} else if pkg != p.Name {
   547  			// TODO(#45999): The choice of p.Name is arbitrary based on file iteration
   548  			// order. Instead of resolving p.Name arbitrarily, we should clear out the
   549  			// existing Name and mark the existing files as also invalid.
   550  			badGoFile(name, &MultiplePackageError{
   551  				Dir:      p.Dir,
   552  				Packages: []string{p.Name, pkg},
   553  				Files:    []string{firstFile, name},
   554  			})
   555  		}
   556  		// Grab the first package comment as docs, provided it is not from a test file.
   557  		if p.Doc == "" && !isTest && !isXTest {
   558  			if synopsis := tf.synopsis(); synopsis != "" {
   559  				p.Doc = synopsis
   560  			}
   561  		}
   562  
   563  		// Record Imports and information about cgo.
   564  		isCgo := false
   565  		imports := tf.imports()
   566  		for _, imp := range imports {
   567  			if imp.path == "C" {
   568  				if isTest {
   569  					badGoFile(name, fmt.Errorf("use of cgo in test %s not supported", name))
   570  					continue
   571  				}
   572  				isCgo = true
   573  			}
   574  		}
   575  		if directives := tf.cgoDirectives(); directives != "" {
   576  			if err := ctxt.saveCgo(name, p, directives); err != nil {
   577  				badGoFile(name, err)
   578  			}
   579  		}
   580  
   581  		var fileList *[]string
   582  		var importMap, embedMap map[string][]token.Position
   583  		var directives *[]build.Directive
   584  		switch {
   585  		case isCgo:
   586  			allTags["cgo"] = true
   587  			if ctxt.CgoEnabled {
   588  				fileList = &p.CgoFiles
   589  				importMap = importPos
   590  				embedMap = embedPos
   591  				directives = &p.Directives
   592  			} else {
   593  				// Ignore Imports and Embeds from cgo files if cgo is disabled.
   594  				fileList = &p.IgnoredGoFiles
   595  			}
   596  		case isXTest:
   597  			fileList = &p.XTestGoFiles
   598  			importMap = xTestImportPos
   599  			embedMap = xTestEmbedPos
   600  			directives = &p.XTestDirectives
   601  		case isTest:
   602  			fileList = &p.TestGoFiles
   603  			importMap = testImportPos
   604  			embedMap = testEmbedPos
   605  			directives = &p.TestDirectives
   606  		default:
   607  			fileList = &p.GoFiles
   608  			importMap = importPos
   609  			embedMap = embedPos
   610  			directives = &p.Directives
   611  		}
   612  		*fileList = append(*fileList, name)
   613  		if importMap != nil {
   614  			for _, imp := range imports {
   615  				importMap[imp.path] = append(importMap[imp.path], imp.position)
   616  			}
   617  		}
   618  		if embedMap != nil {
   619  			for _, e := range tf.embeds() {
   620  				embedMap[e.pattern] = append(embedMap[e.pattern], e.position)
   621  			}
   622  		}
   623  		if directives != nil {
   624  			*directives = append(*directives, tf.directives()...)
   625  		}
   626  	}
   627  
   628  	p.EmbedPatterns, p.EmbedPatternPos = cleanDecls(embedPos)
   629  	p.TestEmbedPatterns, p.TestEmbedPatternPos = cleanDecls(testEmbedPos)
   630  	p.XTestEmbedPatterns, p.XTestEmbedPatternPos = cleanDecls(xTestEmbedPos)
   631  
   632  	p.Imports, p.ImportPos = cleanDecls(importPos)
   633  	p.TestImports, p.TestImportPos = cleanDecls(testImportPos)
   634  	p.XTestImports, p.XTestImportPos = cleanDecls(xTestImportPos)
   635  
   636  	for tag := range allTags {
   637  		p.AllTags = append(p.AllTags, tag)
   638  	}
   639  	sort.Strings(p.AllTags)
   640  
   641  	if len(p.CgoFiles) > 0 {
   642  		p.SFiles = append(p.SFiles, Sfiles...)
   643  		sort.Strings(p.SFiles)
   644  	} else {
   645  		p.IgnoredOtherFiles = append(p.IgnoredOtherFiles, Sfiles...)
   646  		sort.Strings(p.IgnoredOtherFiles)
   647  	}
   648  
   649  	if badGoError != nil {
   650  		return p, badGoError
   651  	}
   652  	if len(p.GoFiles)+len(p.CgoFiles)+len(p.TestGoFiles)+len(p.XTestGoFiles) == 0 {
   653  		return p, &build.NoGoError{Dir: p.Dir}
   654  	}
   655  	return p, pkgerr
   656  }
   657  
   658  // IsStandardPackage reports whether path is a standard package
   659  // for the goroot and compiler using the module index if possible,
   660  // and otherwise falling back to internal/goroot.IsStandardPackage
   661  func IsStandardPackage(goroot_, compiler, path string) bool {
   662  	if !enabled || compiler != "gc" {
   663  		return goroot.IsStandardPackage(goroot_, compiler, path)
   664  	}
   665  
   666  	reldir := filepath.FromSlash(path) // relative dir path in module index for package
   667  	modroot := filepath.Join(goroot_, "src")
   668  	if str.HasFilePathPrefix(reldir, "cmd") {
   669  		reldir = str.TrimFilePathPrefix(reldir, "cmd")
   670  		modroot = filepath.Join(modroot, "cmd")
   671  	}
   672  	if _, err := GetPackage(modroot, filepath.Join(modroot, reldir)); err == nil {
   673  		// Note that goroot.IsStandardPackage doesn't check that the directory
   674  		// actually contains any go files-- merely that it exists. GetPackage
   675  		// returning a nil error is enough for us to know the directory exists.
   676  		return true
   677  	} else if errors.Is(err, ErrNotIndexed) {
   678  		// Fall back because package isn't indexable. (Probably because
   679  		// a file was modified recently)
   680  		return goroot.IsStandardPackage(goroot_, compiler, path)
   681  	}
   682  	return false
   683  }
   684  
   685  // IsDirWithGoFiles is the equivalent of fsys.IsDirWithGoFiles using the information in the index.
   686  func (rp *IndexPackage) IsDirWithGoFiles() (_ bool, err error) {
   687  	defer func() {
   688  		if e := recover(); e != nil {
   689  			err = fmt.Errorf("error reading module index: %v", e)
   690  		}
   691  	}()
   692  	for _, sf := range rp.sourceFiles {
   693  		if strings.HasSuffix(sf.name(), ".go") {
   694  			return true, nil
   695  		}
   696  	}
   697  	return false, nil
   698  }
   699  
   700  // ScanDir implements imports.ScanDir using the information in the index.
   701  func (rp *IndexPackage) ScanDir(tags map[string]bool) (sortedImports []string, sortedTestImports []string, err error) {
   702  	// TODO(matloob) dir should eventually be relative to indexed directory
   703  	// TODO(matloob): skip reading raw package and jump straight to data we need?
   704  
   705  	defer func() {
   706  		if e := recover(); e != nil {
   707  			err = fmt.Errorf("error reading module index: %v", e)
   708  		}
   709  	}()
   710  
   711  	imports_ := make(map[string]bool)
   712  	testImports := make(map[string]bool)
   713  	numFiles := 0
   714  
   715  Files:
   716  	for _, sf := range rp.sourceFiles {
   717  		name := sf.name()
   718  		if strings.HasPrefix(name, "_") || strings.HasPrefix(name, ".") || !strings.HasSuffix(name, ".go") || !imports.MatchFile(name, tags) {
   719  			continue
   720  		}
   721  
   722  		// The following section exists for backwards compatibility reasons:
   723  		// scanDir ignores files with import "C" when collecting the list
   724  		// of imports unless the "cgo" tag is provided. The following comment
   725  		// is copied from the original.
   726  		//
   727  		// import "C" is implicit requirement of cgo tag.
   728  		// When listing files on the command line (explicitFiles=true)
   729  		// we do not apply build tag filtering but we still do apply
   730  		// cgo filtering, so no explicitFiles check here.
   731  		// Why? Because we always have, and it's not worth breaking
   732  		// that behavior now.
   733  		imps := sf.imports() // TODO(matloob): directly read import paths to avoid the extra strings?
   734  		for _, imp := range imps {
   735  			if imp.path == "C" && !tags["cgo"] && !tags["*"] {
   736  				continue Files
   737  			}
   738  		}
   739  
   740  		if !shouldBuild(sf, tags) {
   741  			continue
   742  		}
   743  		numFiles++
   744  		m := imports_
   745  		if strings.HasSuffix(name, "_test.go") {
   746  			m = testImports
   747  		}
   748  		for _, p := range imps {
   749  			m[p.path] = true
   750  		}
   751  	}
   752  	if numFiles == 0 {
   753  		return nil, nil, imports.ErrNoGo
   754  	}
   755  	return keys(imports_), keys(testImports), nil
   756  }
   757  
   758  func keys(m map[string]bool) []string {
   759  	list := make([]string, 0, len(m))
   760  	for k := range m {
   761  		list = append(list, k)
   762  	}
   763  	sort.Strings(list)
   764  	return list
   765  }
   766  
   767  // implements imports.ShouldBuild in terms of an index sourcefile.
   768  func shouldBuild(sf *sourceFile, tags map[string]bool) bool {
   769  	if goBuildConstraint := sf.goBuildConstraint(); goBuildConstraint != "" {
   770  		x, err := constraint.Parse(goBuildConstraint)
   771  		if err != nil {
   772  			return false
   773  		}
   774  		return imports.Eval(x, tags, true)
   775  	}
   776  
   777  	plusBuildConstraints := sf.plusBuildConstraints()
   778  	for _, text := range plusBuildConstraints {
   779  		if x, err := constraint.Parse(text); err == nil {
   780  			if !imports.Eval(x, tags, true) {
   781  				return false
   782  			}
   783  		}
   784  	}
   785  
   786  	return true
   787  }
   788  
   789  // IndexPackage holds the information needed to access information in the
   790  // index needed to load a package in a specific directory.
   791  type IndexPackage struct {
   792  	error error
   793  	dir   string // directory of the package relative to the modroot
   794  
   795  	modroot string
   796  
   797  	// Source files
   798  	sourceFiles []*sourceFile
   799  }
   800  
   801  var errCannotFindPackage = errors.New("cannot find package")
   802  
   803  // Package and returns finds the package with the given path (relative to the module root).
   804  // If the package does not exist, Package returns an IndexPackage that will return an
   805  // appropriate error from its methods.
   806  func (m *Module) Package(path string) *IndexPackage {
   807  	defer unprotect(protect(), nil)
   808  
   809  	i, ok := sort.Find(m.n, func(i int) int {
   810  		return strings.Compare(path, m.pkgDir(i))
   811  	})
   812  	if !ok {
   813  		return &IndexPackage{error: fmt.Errorf("%w %q in:\n\t%s", errCannotFindPackage, path, filepath.Join(m.modroot, path))}
   814  	}
   815  	return m.pkg(i)
   816  }
   817  
   818  // pkg returns the i'th IndexPackage in m.
   819  func (m *Module) pkg(i int) *IndexPackage {
   820  	r := m.d.readAt(m.pkgOff(i))
   821  	p := new(IndexPackage)
   822  	if errstr := r.string(); errstr != "" {
   823  		p.error = errors.New(errstr)
   824  	}
   825  	p.dir = r.string()
   826  	p.sourceFiles = make([]*sourceFile, r.int())
   827  	for i := range p.sourceFiles {
   828  		p.sourceFiles[i] = &sourceFile{
   829  			d:   m.d,
   830  			pos: r.int(),
   831  		}
   832  	}
   833  	p.modroot = m.modroot
   834  	return p
   835  }
   836  
   837  // sourceFile represents the information of a given source file in the module index.
   838  type sourceFile struct {
   839  	d               *decoder // encoding of this source file
   840  	pos             int      // start of sourceFile encoding in d
   841  	onceReadImports sync.Once
   842  	savedImports    []rawImport // saved imports so that they're only read once
   843  }
   844  
   845  // Offsets for fields in the sourceFile.
   846  const (
   847  	sourceFileError = 4 * iota
   848  	sourceFileParseError
   849  	sourceFileSynopsis
   850  	sourceFileName
   851  	sourceFilePkgName
   852  	sourceFileIgnoreFile
   853  	sourceFileBinaryOnly
   854  	sourceFileCgoDirectives
   855  	sourceFileGoBuildConstraint
   856  	sourceFileNumPlusBuildConstraints
   857  )
   858  
   859  func (sf *sourceFile) error() string {
   860  	return sf.d.stringAt(sf.pos + sourceFileError)
   861  }
   862  func (sf *sourceFile) parseError() string {
   863  	return sf.d.stringAt(sf.pos + sourceFileParseError)
   864  }
   865  func (sf *sourceFile) synopsis() string {
   866  	return sf.d.stringAt(sf.pos + sourceFileSynopsis)
   867  }
   868  func (sf *sourceFile) name() string {
   869  	return sf.d.stringAt(sf.pos + sourceFileName)
   870  }
   871  func (sf *sourceFile) pkgName() string {
   872  	return sf.d.stringAt(sf.pos + sourceFilePkgName)
   873  }
   874  func (sf *sourceFile) ignoreFile() bool {
   875  	return sf.d.boolAt(sf.pos + sourceFileIgnoreFile)
   876  }
   877  func (sf *sourceFile) binaryOnly() bool {
   878  	return sf.d.boolAt(sf.pos + sourceFileBinaryOnly)
   879  }
   880  func (sf *sourceFile) cgoDirectives() string {
   881  	return sf.d.stringAt(sf.pos + sourceFileCgoDirectives)
   882  }
   883  func (sf *sourceFile) goBuildConstraint() string {
   884  	return sf.d.stringAt(sf.pos + sourceFileGoBuildConstraint)
   885  }
   886  
   887  func (sf *sourceFile) plusBuildConstraints() []string {
   888  	pos := sf.pos + sourceFileNumPlusBuildConstraints
   889  	n := sf.d.intAt(pos)
   890  	pos += 4
   891  	ret := make([]string, n)
   892  	for i := 0; i < n; i++ {
   893  		ret[i] = sf.d.stringAt(pos)
   894  		pos += 4
   895  	}
   896  	return ret
   897  }
   898  
   899  func (sf *sourceFile) importsOffset() int {
   900  	pos := sf.pos + sourceFileNumPlusBuildConstraints
   901  	n := sf.d.intAt(pos)
   902  	// each build constraint is 1 uint32
   903  	return pos + 4 + n*4
   904  }
   905  
   906  func (sf *sourceFile) embedsOffset() int {
   907  	pos := sf.importsOffset()
   908  	n := sf.d.intAt(pos)
   909  	// each import is 5 uint32s (string + tokpos)
   910  	return pos + 4 + n*(4*5)
   911  }
   912  
   913  func (sf *sourceFile) directivesOffset() int {
   914  	pos := sf.embedsOffset()
   915  	n := sf.d.intAt(pos)
   916  	// each embed is 5 uint32s (string + tokpos)
   917  	return pos + 4 + n*(4*5)
   918  }
   919  
   920  func (sf *sourceFile) imports() []rawImport {
   921  	sf.onceReadImports.Do(func() {
   922  		importsOffset := sf.importsOffset()
   923  		r := sf.d.readAt(importsOffset)
   924  		numImports := r.int()
   925  		ret := make([]rawImport, numImports)
   926  		for i := 0; i < numImports; i++ {
   927  			ret[i] = rawImport{r.string(), r.tokpos()}
   928  		}
   929  		sf.savedImports = ret
   930  	})
   931  	return sf.savedImports
   932  }
   933  
   934  func (sf *sourceFile) embeds() []embed {
   935  	embedsOffset := sf.embedsOffset()
   936  	r := sf.d.readAt(embedsOffset)
   937  	numEmbeds := r.int()
   938  	ret := make([]embed, numEmbeds)
   939  	for i := range ret {
   940  		ret[i] = embed{r.string(), r.tokpos()}
   941  	}
   942  	return ret
   943  }
   944  
   945  func (sf *sourceFile) directives() []build.Directive {
   946  	directivesOffset := sf.directivesOffset()
   947  	r := sf.d.readAt(directivesOffset)
   948  	numDirectives := r.int()
   949  	ret := make([]build.Directive, numDirectives)
   950  	for i := range ret {
   951  		ret[i] = build.Directive{Text: r.string(), Pos: r.tokpos()}
   952  	}
   953  	return ret
   954  }
   955  
   956  func asString(b []byte) string {
   957  	return unsafe.String(unsafe.SliceData(b), len(b))
   958  }
   959  
   960  // A decoder helps decode the index format.
   961  type decoder struct {
   962  	data []byte // data after header
   963  	str  []byte // string table
   964  }
   965  
   966  // intAt returns the int at the given offset in d.data.
   967  func (d *decoder) intAt(off int) int {
   968  	if off < 0 || len(d.data)-off < 4 {
   969  		panic(errCorrupt)
   970  	}
   971  	i := binary.LittleEndian.Uint32(d.data[off : off+4])
   972  	if int32(i)>>31 != 0 {
   973  		panic(errCorrupt)
   974  	}
   975  	return int(i)
   976  }
   977  
   978  // boolAt returns the bool at the given offset in d.data.
   979  func (d *decoder) boolAt(off int) bool {
   980  	return d.intAt(off) != 0
   981  }
   982  
   983  // stringAt returns the string pointed at by the int at the given offset in d.data.
   984  func (d *decoder) stringAt(off int) string {
   985  	return d.stringTableAt(d.intAt(off))
   986  }
   987  
   988  // stringTableAt returns the string at the given offset in the string table d.str.
   989  func (d *decoder) stringTableAt(off int) string {
   990  	if off < 0 || off >= len(d.str) {
   991  		panic(errCorrupt)
   992  	}
   993  	s := d.str[off:]
   994  	v, n := binary.Uvarint(s)
   995  	if n <= 0 || v > uint64(len(s[n:])) {
   996  		panic(errCorrupt)
   997  	}
   998  	return asString(s[n : n+int(v)])
   999  }
  1000  
  1001  // A reader reads sequential fields from a section of the index format.
  1002  type reader struct {
  1003  	d   *decoder
  1004  	pos int
  1005  }
  1006  
  1007  // readAt returns a reader starting at the given position in d.
  1008  func (d *decoder) readAt(pos int) *reader {
  1009  	return &reader{d, pos}
  1010  }
  1011  
  1012  // int reads the next int.
  1013  func (r *reader) int() int {
  1014  	i := r.d.intAt(r.pos)
  1015  	r.pos += 4
  1016  	return i
  1017  }
  1018  
  1019  // string reads the next string.
  1020  func (r *reader) string() string {
  1021  	return r.d.stringTableAt(r.int())
  1022  }
  1023  
  1024  // bool reads the next bool.
  1025  func (r *reader) bool() bool {
  1026  	return r.int() != 0
  1027  }
  1028  
  1029  // tokpos reads the next token.Position.
  1030  func (r *reader) tokpos() token.Position {
  1031  	return token.Position{
  1032  		Filename: r.string(),
  1033  		Offset:   r.int(),
  1034  		Line:     r.int(),
  1035  		Column:   r.int(),
  1036  	}
  1037  }
  1038  

View as plain text