unified.go

     1  // Copyright 2021 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package noder
     6  
     7  import (
     8  	"fmt"
     9  	"internal/pkgbits"
    10  	"io"
    11  	"runtime"
    12  	"sort"
    13  	"strings"
    14  
    15  	"cmd/compile/internal/base"
    16  	"cmd/compile/internal/inline"
    17  	"cmd/compile/internal/ir"
    18  	"cmd/compile/internal/pgo"
    19  	"cmd/compile/internal/typecheck"
    20  	"cmd/compile/internal/types"
    21  	"cmd/compile/internal/types2"
    22  	"cmd/internal/src"
    23  )
    24  
    25  // localPkgReader holds the package reader used for reading the local
    26  // package. It exists so the unified IR linker can refer back to it
    27  // later.
    28  var localPkgReader *pkgReader
    29  
    30  // LookupMethodFunc returns the ir.Func for an arbitrary full symbol name if
    31  // that function exists in the set of available export data.
    32  //
    33  // This allows lookup of arbitrary functions and methods that aren't otherwise
    34  // referenced by the local package and thus haven't been read yet.
    35  //
    36  // TODO(prattmic): Does not handle instantiation of generic types. Currently
    37  // profiles don't contain the original type arguments, so we won't be able to
    38  // create the runtime dictionaries.
    39  //
    40  // TODO(prattmic): Hit rate of this function is usually fairly low, and errors
    41  // are only used when debug logging is enabled. Consider constructing cheaper
    42  // errors by default.
    43  func LookupFunc(fullName string) (*ir.Func, error) {
    44  	pkgPath, symName, err := ir.ParseLinkFuncName(fullName)
    45  	if err != nil {
    46  		return nil, fmt.Errorf("error parsing symbol name %q: %v", fullName, err)
    47  	}
    48  
    49  	pkg, ok := types.PkgMap()[pkgPath]
    50  	if !ok {
    51  		return nil, fmt.Errorf("pkg %s doesn't exist in %v", pkgPath, types.PkgMap())
    52  	}
    53  
    54  	// Symbol naming is ambiguous. We can't necessarily distinguish between
    55  	// a method and a closure. e.g., is foo.Bar.func1 a closure defined in
    56  	// function Bar, or a method on type Bar? Thus we must simply attempt
    57  	// to lookup both.
    58  
    59  	fn, err := lookupFunction(pkg, symName)
    60  	if err == nil {
    61  		return fn, nil
    62  	}
    63  
    64  	fn, mErr := lookupMethod(pkg, symName)
    65  	if mErr == nil {
    66  		return fn, nil
    67  	}
    68  
    69  	return nil, fmt.Errorf("%s is not a function (%v) or method (%v)", fullName, err, mErr)
    70  }
    71  
    72  func lookupFunction(pkg *types.Pkg, symName string) (*ir.Func, error) {
    73  	sym := pkg.Lookup(symName)
    74  
    75  	// TODO(prattmic): Enclosed functions (e.g., foo.Bar.func1) are not
    76  	// present in objReader, only as OCLOSURE nodes in the enclosing
    77  	// function.
    78  	pri, ok := objReader[sym]
    79  	if !ok {
    80  		return nil, fmt.Errorf("func sym %v missing objReader", sym)
    81  	}
    82  
    83  	node, err := pri.pr.objIdxMayFail(pri.idx, nil, nil, false)
    84  	if err != nil {
    85  		return nil, fmt.Errorf("func sym %v lookup error: %w", sym, err)
    86  	}
    87  	name := node.(*ir.Name)
    88  	if name.Op() != ir.ONAME || name.Class != ir.PFUNC {
    89  		return nil, fmt.Errorf("func sym %v refers to non-function name: %v", sym, name)
    90  	}
    91  	return name.Func, nil
    92  }
    93  
    94  func lookupMethod(pkg *types.Pkg, symName string) (*ir.Func, error) {
    95  	// N.B. readPackage creates a Sym for every object in the package to
    96  	// initialize objReader and importBodyReader, even if the object isn't
    97  	// read.
    98  	//
    99  	// However, objReader is only initialized for top-level objects, so we
   100  	// must first lookup the type and use that to find the method rather
   101  	// than looking for the method directly.
   102  	typ, meth, err := ir.LookupMethodSelector(pkg, symName)
   103  	if err != nil {
   104  		return nil, fmt.Errorf("error looking up method symbol %q: %v", symName, err)
   105  	}
   106  
   107  	pri, ok := objReader[typ]
   108  	if !ok {
   109  		return nil, fmt.Errorf("type sym %v missing objReader", typ)
   110  	}
   111  
   112  	node, err := pri.pr.objIdxMayFail(pri.idx, nil, nil, false)
   113  	if err != nil {
   114  		return nil, fmt.Errorf("func sym %v lookup error: %w", typ, err)
   115  	}
   116  	name := node.(*ir.Name)
   117  	if name.Op() != ir.OTYPE {
   118  		return nil, fmt.Errorf("type sym %v refers to non-type name: %v", typ, name)
   119  	}
   120  	if name.Alias() {
   121  		return nil, fmt.Errorf("type sym %v refers to alias", typ)
   122  	}
   123  	if name.Type().IsInterface() {
   124  		return nil, fmt.Errorf("type sym %v refers to interface type", typ)
   125  	}
   126  
   127  	for _, m := range name.Type().Methods() {
   128  		if m.Sym == meth {
   129  			fn := m.Nname.(*ir.Name).Func
   130  			return fn, nil
   131  		}
   132  	}
   133  
   134  	return nil, fmt.Errorf("method %s missing from method set of %v", symName, typ)
   135  }
   136  
   137  // unified constructs the local package's Internal Representation (IR)
   138  // from its syntax tree (AST).
   139  //
   140  // The pipeline contains 2 steps:
   141  //
   142  //  1. Generate the export data "stub".
   143  //
   144  //  2. Generate the IR from the export data above.
   145  //
   146  // The package data "stub" at step (1) contains everything from the local package,
   147  // but nothing that has been imported. When we're actually writing out export data
   148  // to the output files (see writeNewExport), we run the "linker", which:
   149  //
   150  //   - Updates compiler extensions data (e.g. inlining cost, escape analysis results).
   151  //
   152  //   - Handles re-exporting any transitive dependencies.
   153  //
   154  //   - Prunes out any unnecessary details (e.g. non-inlineable functions, because any
   155  //     downstream importers only care about inlinable functions).
   156  //
   157  // The source files are typechecked twice: once before writing the export data
   158  // using types2, and again after reading the export data using gc/typecheck.
   159  // The duplication of work will go away once we only use the types2 type checker,
   160  // removing the gc/typecheck step. For now, it is kept because:
   161  //
   162  //   - It reduces the engineering costs in maintaining a fork of typecheck
   163  //     (e.g. no need to backport fixes like CL 327651).
   164  //
   165  //   - It makes it easier to pass toolstash -cmp.
   166  //
   167  //   - Historically, we would always re-run the typechecker after importing a package,
   168  //     even though we know the imported data is valid. It's not ideal, but it's
   169  //     not causing any problems either.
   170  //
   171  //   - gc/typecheck is still in charge of some transformations, such as rewriting
   172  //     multi-valued function calls or transforming ir.OINDEX to ir.OINDEXMAP.
   173  //
   174  // Using the syntax tree with types2, which has a complete representation of generics,
   175  // the unified IR has the full typed AST needed for introspection during step (1).
   176  // In other words, we have all the necessary information to build the generic IR form
   177  // (see writer.captureVars for an example).
   178  func unified(m posMap, noders []*noder) {
   179  	inline.InlineCall = unifiedInlineCall
   180  	typecheck.HaveInlineBody = unifiedHaveInlineBody
   181  	pgo.LookupFunc = LookupFunc
   182  
   183  	data := writePkgStub(m, noders)
   184  
   185  	target := typecheck.Target
   186  
   187  	localPkgReader = newPkgReader(pkgbits.NewPkgDecoder(types.LocalPkg.Path, data))
   188  	readPackage(localPkgReader, types.LocalPkg, true)
   189  
   190  	r := localPkgReader.newReader(pkgbits.RelocMeta, pkgbits.PrivateRootIdx, pkgbits.SyncPrivate)
   191  	r.pkgInit(types.LocalPkg, target)
   192  
   193  	readBodies(target, false)
   194  
   195  	// Check that nothing snuck past typechecking.
   196  	for _, fn := range target.Funcs {
   197  		if fn.Typecheck() == 0 {
   198  			base.FatalfAt(fn.Pos(), "missed typecheck: %v", fn)
   199  		}
   200  
   201  		// For functions, check that at least their first statement (if
   202  		// any) was typechecked too.
   203  		if len(fn.Body) != 0 {
   204  			if stmt := fn.Body[0]; stmt.Typecheck() == 0 {
   205  				base.FatalfAt(stmt.Pos(), "missed typecheck: %v", stmt)
   206  			}
   207  		}
   208  	}
   209  
   210  	// For functions originally came from package runtime,
   211  	// mark as norace to prevent instrumenting, see issue #60439.
   212  	for _, fn := range target.Funcs {
   213  		if !base.Flag.CompilingRuntime && types.RuntimeSymName(fn.Sym()) != "" {
   214  			fn.Pragma |= ir.Norace
   215  		}
   216  	}
   217  
   218  	base.ExitIfErrors() // just in case
   219  }
   220  
   221  // readBodies iteratively expands all pending dictionaries and
   222  // function bodies.
   223  //
   224  // If duringInlining is true, then the inline.InlineDecls is called as
   225  // necessary on instantiations of imported generic functions, so their
   226  // inlining costs can be computed.
   227  func readBodies(target *ir.Package, duringInlining bool) {
   228  	var inlDecls []*ir.Func
   229  
   230  	// Don't use range--bodyIdx can add closures to todoBodies.
   231  	for {
   232  		// The order we expand dictionaries and bodies doesn't matter, so
   233  		// pop from the end to reduce todoBodies reallocations if it grows
   234  		// further.
   235  		//
   236  		// However, we do at least need to flush any pending dictionaries
   237  		// before reading bodies, because bodies might reference the
   238  		// dictionaries.
   239  
   240  		if len(todoDicts) > 0 {
   241  			fn := todoDicts[len(todoDicts)-1]
   242  			todoDicts = todoDicts[:len(todoDicts)-1]
   243  			fn()
   244  			continue
   245  		}
   246  
   247  		if len(todoBodies) > 0 {
   248  			fn := todoBodies[len(todoBodies)-1]
   249  			todoBodies = todoBodies[:len(todoBodies)-1]
   250  
   251  			pri, ok := bodyReader[fn]
   252  			assert(ok)
   253  			pri.funcBody(fn)
   254  
   255  			// Instantiated generic function: add to Decls for typechecking
   256  			// and compilation.
   257  			if fn.OClosure == nil && len(pri.dict.targs) != 0 {
   258  				// cmd/link does not support a type symbol referencing a method symbol
   259  				// across DSO boundary, so force re-compiling methods on a generic type
   260  				// even it was seen from imported package in linkshared mode, see #58966.
   261  				canSkipNonGenericMethod := !(base.Ctxt.Flag_linkshared && ir.IsMethod(fn))
   262  				if duringInlining && canSkipNonGenericMethod {
   263  					inlDecls = append(inlDecls, fn)
   264  				} else {
   265  					target.Funcs = append(target.Funcs, fn)
   266  				}
   267  			}
   268  
   269  			continue
   270  		}
   271  
   272  		break
   273  	}
   274  
   275  	todoDicts = nil
   276  	todoBodies = nil
   277  
   278  	if len(inlDecls) != 0 {
   279  		// If we instantiated any generic functions during inlining, we need
   280  		// to call CanInline on them so they'll be transitively inlined
   281  		// correctly (#56280).
   282  		//
   283  		// We know these functions were already compiled in an imported
   284  		// package though, so we don't need to actually apply InlineCalls or
   285  		// save the function bodies any further than this.
   286  		//
   287  		// We can also lower the -m flag to 0, to suppress duplicate "can
   288  		// inline" diagnostics reported against the imported package. Again,
   289  		// we already reported those diagnostics in the original package, so
   290  		// it's pointless repeating them here.
   291  
   292  		oldLowerM := base.Flag.LowerM
   293  		base.Flag.LowerM = 0
   294  		inline.CanInlineFuncs(inlDecls, nil)
   295  		base.Flag.LowerM = oldLowerM
   296  
   297  		for _, fn := range inlDecls {
   298  			fn.Body = nil // free memory
   299  		}
   300  	}
   301  }
   302  
   303  // writePkgStub type checks the given parsed source files,
   304  // writes an export data package stub representing them,
   305  // and returns the result.
   306  func writePkgStub(m posMap, noders []*noder) string {
   307  	pkg, info := checkFiles(m, noders)
   308  
   309  	pw := newPkgWriter(m, pkg, info)
   310  
   311  	pw.collectDecls(noders)
   312  
   313  	publicRootWriter := pw.newWriter(pkgbits.RelocMeta, pkgbits.SyncPublic)
   314  	privateRootWriter := pw.newWriter(pkgbits.RelocMeta, pkgbits.SyncPrivate)
   315  
   316  	assert(publicRootWriter.Idx == pkgbits.PublicRootIdx)
   317  	assert(privateRootWriter.Idx == pkgbits.PrivateRootIdx)
   318  
   319  	{
   320  		w := publicRootWriter
   321  		w.pkg(pkg)
   322  		w.Bool(false) // TODO(mdempsky): Remove; was "has init"
   323  
   324  		scope := pkg.Scope()
   325  		names := scope.Names()
   326  		w.Len(len(names))
   327  		for _, name := range names {
   328  			w.obj(scope.Lookup(name), nil)
   329  		}
   330  
   331  		w.Sync(pkgbits.SyncEOF)
   332  		w.Flush()
   333  	}
   334  
   335  	{
   336  		w := privateRootWriter
   337  		w.pkgInit(noders)
   338  		w.Flush()
   339  	}
   340  
   341  	var sb strings.Builder
   342  	pw.DumpTo(&sb)
   343  
   344  	// At this point, we're done with types2. Make sure the package is
   345  	// garbage collected.
   346  	freePackage(pkg)
   347  
   348  	return sb.String()
   349  }
   350  
   351  // freePackage ensures the given package is garbage collected.
   352  func freePackage(pkg *types2.Package) {
   353  	// The GC test below relies on a precise GC that runs finalizers as
   354  	// soon as objects are unreachable. Our implementation provides
   355  	// this, but other/older implementations may not (e.g., Go 1.4 does
   356  	// not because of #22350). To avoid imposing unnecessary
   357  	// restrictions on the GOROOT_BOOTSTRAP toolchain, we skip the test
   358  	// during bootstrapping.
   359  	if base.CompilerBootstrap || base.Debug.GCCheck == 0 {
   360  		*pkg = types2.Package{}
   361  		return
   362  	}
   363  
   364  	// Set a finalizer on pkg so we can detect if/when it's collected.
   365  	done := make(chan struct{})
   366  	runtime.SetFinalizer(pkg, func(*types2.Package) { close(done) })
   367  
   368  	// Important: objects involved in cycles are not finalized, so zero
   369  	// out pkg to break its cycles and allow the finalizer to run.
   370  	*pkg = types2.Package{}
   371  
   372  	// It typically takes just 1 or 2 cycles to release pkg, but it
   373  	// doesn't hurt to try a few more times.
   374  	for i := 0; i < 10; i++ {
   375  		select {
   376  		case <-done:
   377  			return
   378  		default:
   379  			runtime.GC()
   380  		}
   381  	}
   382  
   383  	base.Fatalf("package never finalized")
   384  }
   385  
   386  // readPackage reads package export data from pr to populate
   387  // importpkg.
   388  //
   389  // localStub indicates whether pr is reading the stub export data for
   390  // the local package, as opposed to relocated export data for an
   391  // import.
   392  func readPackage(pr *pkgReader, importpkg *types.Pkg, localStub bool) {
   393  	{
   394  		r := pr.newReader(pkgbits.RelocMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic)
   395  
   396  		pkg := r.pkg()
   397  		base.Assertf(pkg == importpkg, "have package %q (%p), want package %q (%p)", pkg.Path, pkg, importpkg.Path, importpkg)
   398  
   399  		r.Bool() // TODO(mdempsky): Remove; was "has init"
   400  
   401  		for i, n := 0, r.Len(); i < n; i++ {
   402  			r.Sync(pkgbits.SyncObject)
   403  			assert(!r.Bool())
   404  			idx := r.Reloc(pkgbits.RelocObj)
   405  			assert(r.Len() == 0)
   406  
   407  			path, name, code := r.p.PeekObj(idx)
   408  			if code != pkgbits.ObjStub {
   409  				objReader[types.NewPkg(path, "").Lookup(name)] = pkgReaderIndex{pr, idx, nil, nil, nil}
   410  			}
   411  		}
   412  
   413  		r.Sync(pkgbits.SyncEOF)
   414  	}
   415  
   416  	if !localStub {
   417  		r := pr.newReader(pkgbits.RelocMeta, pkgbits.PrivateRootIdx, pkgbits.SyncPrivate)
   418  
   419  		if r.Bool() {
   420  			sym := importpkg.Lookup(".inittask")
   421  			task := ir.NewNameAt(src.NoXPos, sym, nil)
   422  			task.Class = ir.PEXTERN
   423  			sym.Def = task
   424  		}
   425  
   426  		for i, n := 0, r.Len(); i < n; i++ {
   427  			path := r.String()
   428  			name := r.String()
   429  			idx := r.Reloc(pkgbits.RelocBody)
   430  
   431  			sym := types.NewPkg(path, "").Lookup(name)
   432  			if _, ok := importBodyReader[sym]; !ok {
   433  				importBodyReader[sym] = pkgReaderIndex{pr, idx, nil, nil, nil}
   434  			}
   435  		}
   436  
   437  		r.Sync(pkgbits.SyncEOF)
   438  	}
   439  }
   440  
   441  // writeUnifiedExport writes to `out` the finalized, self-contained
   442  // Unified IR export data file for the current compilation unit.
   443  func writeUnifiedExport(out io.Writer) {
   444  	l := linker{
   445  		pw: pkgbits.NewPkgEncoder(base.Debug.SyncFrames),
   446  
   447  		pkgs:   make(map[string]pkgbits.Index),
   448  		decls:  make(map[*types.Sym]pkgbits.Index),
   449  		bodies: make(map[*types.Sym]pkgbits.Index),
   450  	}
   451  
   452  	publicRootWriter := l.pw.NewEncoder(pkgbits.RelocMeta, pkgbits.SyncPublic)
   453  	privateRootWriter := l.pw.NewEncoder(pkgbits.RelocMeta, pkgbits.SyncPrivate)
   454  	assert(publicRootWriter.Idx == pkgbits.PublicRootIdx)
   455  	assert(privateRootWriter.Idx == pkgbits.PrivateRootIdx)
   456  
   457  	var selfPkgIdx pkgbits.Index
   458  
   459  	{
   460  		pr := localPkgReader
   461  		r := pr.NewDecoder(pkgbits.RelocMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic)
   462  
   463  		r.Sync(pkgbits.SyncPkg)
   464  		selfPkgIdx = l.relocIdx(pr, pkgbits.RelocPkg, r.Reloc(pkgbits.RelocPkg))
   465  
   466  		r.Bool() // TODO(mdempsky): Remove; was "has init"
   467  
   468  		for i, n := 0, r.Len(); i < n; i++ {
   469  			r.Sync(pkgbits.SyncObject)
   470  			assert(!r.Bool())
   471  			idx := r.Reloc(pkgbits.RelocObj)
   472  			assert(r.Len() == 0)
   473  
   474  			xpath, xname, xtag := pr.PeekObj(idx)
   475  			assert(xpath == pr.PkgPath())
   476  			assert(xtag != pkgbits.ObjStub)
   477  
   478  			if types.IsExported(xname) {
   479  				l.relocIdx(pr, pkgbits.RelocObj, idx)
   480  			}
   481  		}
   482  
   483  		r.Sync(pkgbits.SyncEOF)
   484  	}
   485  
   486  	{
   487  		var idxs []pkgbits.Index
   488  		for _, idx := range l.decls {
   489  			idxs = append(idxs, idx)
   490  		}
   491  		sort.Slice(idxs, func(i, j int) bool { return idxs[i] < idxs[j] })
   492  
   493  		w := publicRootWriter
   494  
   495  		w.Sync(pkgbits.SyncPkg)
   496  		w.Reloc(pkgbits.RelocPkg, selfPkgIdx)
   497  		w.Bool(false) // TODO(mdempsky): Remove; was "has init"
   498  
   499  		w.Len(len(idxs))
   500  		for _, idx := range idxs {
   501  			w.Sync(pkgbits.SyncObject)
   502  			w.Bool(false)
   503  			w.Reloc(pkgbits.RelocObj, idx)
   504  			w.Len(0)
   505  		}
   506  
   507  		w.Sync(pkgbits.SyncEOF)
   508  		w.Flush()
   509  	}
   510  
   511  	{
   512  		type symIdx struct {
   513  			sym *types.Sym
   514  			idx pkgbits.Index
   515  		}
   516  		var bodies []symIdx
   517  		for sym, idx := range l.bodies {
   518  			bodies = append(bodies, symIdx{sym, idx})
   519  		}
   520  		sort.Slice(bodies, func(i, j int) bool { return bodies[i].idx < bodies[j].idx })
   521  
   522  		w := privateRootWriter
   523  
   524  		w.Bool(typecheck.Lookup(".inittask").Def != nil)
   525  
   526  		w.Len(len(bodies))
   527  		for _, body := range bodies {
   528  			w.String(body.sym.Pkg.Path)
   529  			w.String(body.sym.Name)
   530  			w.Reloc(pkgbits.RelocBody, body.idx)
   531  		}
   532  
   533  		w.Sync(pkgbits.SyncEOF)
   534  		w.Flush()
   535  	}
   536  
   537  	base.Ctxt.Fingerprint = l.pw.DumpTo(out)
   538  }
   539
View as plain text