Black Lives Matter. Support the Equal Justice Initiative.

Source file src/cmd/go/internal/work/buildid.go

Documentation: cmd/go/internal/work

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package work
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"io/ioutil"
    11  	"os"
    12  	"os/exec"
    13  	"strings"
    14  
    15  	"cmd/go/internal/base"
    16  	"cmd/go/internal/cache"
    17  	"cmd/go/internal/cfg"
    18  	"cmd/go/internal/str"
    19  	"cmd/internal/buildid"
    20  )
    21  
    22  // Build IDs
    23  //
    24  // Go packages and binaries are stamped with build IDs that record both
    25  // the action ID, which is a hash of the inputs to the action that produced
    26  // the packages or binary, and the content ID, which is a hash of the action
    27  // output, namely the archive or binary itself. The hash is the same one
    28  // used by the build artifact cache (see cmd/go/internal/cache), but
    29  // truncated when stored in packages and binaries, as the full length is not
    30  // needed and is a bit unwieldy. The precise form is
    31  //
    32  //	actionID/[.../]contentID
    33  //
    34  // where the actionID and contentID are prepared by hashToString below.
    35  // and are found by looking for the first or last slash.
    36  // Usually the buildID is simply actionID/contentID, but see below for an
    37  // exception.
    38  //
    39  // The build ID serves two primary purposes.
    40  //
    41  // 1. The action ID half allows installed packages and binaries to serve as
    42  // one-element cache entries. If we intend to build math.a with a given
    43  // set of inputs summarized in the action ID, and the installed math.a already
    44  // has that action ID, we can reuse the installed math.a instead of rebuilding it.
    45  //
    46  // 2. The content ID half allows the easy preparation of action IDs for steps
    47  // that consume a particular package or binary. The content hash of every
    48  // input file for a given action must be included in the action ID hash.
    49  // Storing the content ID in the build ID lets us read it from the file with
    50  // minimal I/O, instead of reading and hashing the entire file.
    51  // This is especially effective since packages and binaries are typically
    52  // the largest inputs to an action.
    53  //
    54  // Separating action ID from content ID is important for reproducible builds.
    55  // The compiler is compiled with itself. If an output were represented by its
    56  // own action ID (instead of content ID) when computing the action ID of
    57  // the next step in the build process, then the compiler could never have its
    58  // own input action ID as its output action ID (short of a miraculous hash collision).
    59  // Instead we use the content IDs to compute the next action ID, and because
    60  // the content IDs converge, so too do the action IDs and therefore the
    61  // build IDs and the overall compiler binary. See cmd/dist's cmdbootstrap
    62  // for the actual convergence sequence.
    63  //
    64  // The “one-element cache” purpose is a bit more complex for installed
    65  // binaries. For a binary, like cmd/gofmt, there are two steps: compile
    66  // cmd/gofmt/*.go into main.a, and then link main.a into the gofmt binary.
    67  // We do not install gofmt's main.a, only the gofmt binary. Being able to
    68  // decide that the gofmt binary is up-to-date means computing the action ID
    69  // for the final link of the gofmt binary and comparing it against the
    70  // already-installed gofmt binary. But computing the action ID for the link
    71  // means knowing the content ID of main.a, which we did not keep.
    72  // To sidestep this problem, each binary actually stores an expanded build ID:
    73  //
    74  //	actionID(binary)/actionID(main.a)/contentID(main.a)/contentID(binary)
    75  //
    76  // (Note that this can be viewed equivalently as:
    77  //
    78  //	actionID(binary)/buildID(main.a)/contentID(binary)
    79  //
    80  // Storing the buildID(main.a) in the middle lets the computations that care
    81  // about the prefix or suffix halves ignore the middle and preserves the
    82  // original build ID as a contiguous string.)
    83  //
    84  // During the build, when it's time to build main.a, the gofmt binary has the
    85  // information needed to decide whether the eventual link would produce
    86  // the same binary: if the action ID for main.a's inputs matches and then
    87  // the action ID for the link step matches when assuming the given main.a
    88  // content ID, then the binary as a whole is up-to-date and need not be rebuilt.
    89  //
    90  // This is all a bit complex and may be simplified once we can rely on the
    91  // main cache, but at least at the start we will be using the content-based
    92  // staleness determination without a cache beyond the usual installed
    93  // package and binary locations.
    94  
    95  const buildIDSeparator = "/"
    96  
    97  // actionID returns the action ID half of a build ID.
    98  func actionID(buildID string) string {
    99  	i := strings.Index(buildID, buildIDSeparator)
   100  	if i < 0 {
   101  		return buildID
   102  	}
   103  	return buildID[:i]
   104  }
   105  
   106  // contentID returns the content ID half of a build ID.
   107  func contentID(buildID string) string {
   108  	return buildID[strings.LastIndex(buildID, buildIDSeparator)+1:]
   109  }
   110  
   111  // hashToString converts the hash h to a string to be recorded
   112  // in package archives and binaries as part of the build ID.
   113  // We use the first 96 bits of the hash and encode it in base64,
   114  // resulting in a 16-byte string. Because this is only used for
   115  // detecting the need to rebuild installed files (not for lookups
   116  // in the object file cache), 96 bits are sufficient to drive the
   117  // probability of a false "do not need to rebuild" decision to effectively zero.
   118  // We embed two different hashes in archives and four in binaries,
   119  // so cutting to 16 bytes is a significant savings when build IDs are displayed.
   120  // (16*4+3 = 67 bytes compared to 64*4+3 = 259 bytes for the
   121  // more straightforward option of printing the entire h in hex).
   122  func hashToString(h [cache.HashSize]byte) string {
   123  	const b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
   124  	const chunks = 5
   125  	var dst [chunks * 4]byte
   126  	for i := 0; i < chunks; i++ {
   127  		v := uint32(h[3*i])<<16 | uint32(h[3*i+1])<<8 | uint32(h[3*i+2])
   128  		dst[4*i+0] = b64[(v>>18)&0x3F]
   129  		dst[4*i+1] = b64[(v>>12)&0x3F]
   130  		dst[4*i+2] = b64[(v>>6)&0x3F]
   131  		dst[4*i+3] = b64[v&0x3F]
   132  	}
   133  	return string(dst[:])
   134  }
   135  
   136  // toolID returns the unique ID to use for the current copy of the
   137  // named tool (asm, compile, cover, link).
   138  //
   139  // It is important that if the tool changes (for example a compiler bug is fixed
   140  // and the compiler reinstalled), toolID returns a different string, so that old
   141  // package archives look stale and are rebuilt (with the fixed compiler).
   142  // This suggests using a content hash of the tool binary, as stored in the build ID.
   143  //
   144  // Unfortunately, we can't just open the tool binary, because the tool might be
   145  // invoked via a wrapper program specified by -toolexec and we don't know
   146  // what the wrapper program does. In particular, we want "-toolexec toolstash"
   147  // to continue working: it does no good if "-toolexec toolstash" is executing a
   148  // stashed copy of the compiler but the go command is acting as if it will run
   149  // the standard copy of the compiler. The solution is to ask the tool binary to tell
   150  // us its own build ID using the "-V=full" flag now supported by all tools.
   151  // Then we know we're getting the build ID of the compiler that will actually run
   152  // during the build. (How does the compiler binary know its own content hash?
   153  // We store it there using updateBuildID after the standard link step.)
   154  //
   155  // A final twist is that we'd prefer to have reproducible builds for release toolchains.
   156  // It should be possible to cross-compile for Windows from either Linux or Mac
   157  // or Windows itself and produce the same binaries, bit for bit. If the tool ID,
   158  // which influences the action ID half of the build ID, is based on the content ID,
   159  // then the Linux compiler binary and Mac compiler binary will have different tool IDs
   160  // and therefore produce executables with different action IDs.
   161  // To avoid this problem, for releases we use the release version string instead
   162  // of the compiler binary's content hash. This assumes that all compilers built
   163  // on all different systems are semantically equivalent, which is of course only true
   164  // modulo bugs. (Producing the exact same executables also requires that the different
   165  // build setups agree on details like $GOROOT and file name paths, but at least the
   166  // tool IDs do not make it impossible.)
   167  func (b *Builder) toolID(name string) string {
   168  	b.id.Lock()
   169  	id := b.toolIDCache[name]
   170  	b.id.Unlock()
   171  
   172  	if id != "" {
   173  		return id
   174  	}
   175  
   176  	path := base.Tool(name)
   177  	desc := "go tool " + name
   178  
   179  	// Special case: undocumented -vettool overrides usual vet,
   180  	// for testing vet or supplying an alternative analysis tool.
   181  	if name == "vet" && VetTool != "" {
   182  		path = VetTool
   183  		desc = VetTool
   184  	}
   185  
   186  	cmdline := str.StringList(cfg.BuildToolexec, path, "-V=full")
   187  	cmd := exec.Command(cmdline[0], cmdline[1:]...)
   188  	cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir)
   189  	var stdout, stderr bytes.Buffer
   190  	cmd.Stdout = &stdout
   191  	cmd.Stderr = &stderr
   192  	if err := cmd.Run(); err != nil {
   193  		base.Fatalf("%s: %v\n%s%s", desc, err, stdout.Bytes(), stderr.Bytes())
   194  	}
   195  
   196  	line := stdout.String()
   197  	f := strings.Fields(line)
   198  	if len(f) < 3 || f[0] != name && path != VetTool || f[1] != "version" || f[2] == "devel" && !strings.HasPrefix(f[len(f)-1], "buildID=") {
   199  		base.Fatalf("%s -V=full: unexpected output:\n\t%s", desc, line)
   200  	}
   201  	if f[2] == "devel" {
   202  		// On the development branch, use the content ID part of the build ID.
   203  		id = contentID(f[len(f)-1])
   204  	} else {
   205  		// For a release, the output is like: "compile version go1.9.1 X:framepointer".
   206  		// Use the whole line.
   207  		id = strings.TrimSpace(line)
   208  	}
   209  
   210  	b.id.Lock()
   211  	b.toolIDCache[name] = id
   212  	b.id.Unlock()
   213  
   214  	return id
   215  }
   216  
   217  // gccToolID returns the unique ID to use for a tool that is invoked
   218  // by the GCC driver. This is used particularly for gccgo, but this can also
   219  // be used for gcc, g++, gfortran, etc.; those tools all use the GCC
   220  // driver under different names. The approach used here should also
   221  // work for sufficiently new versions of clang. Unlike toolID, the
   222  // name argument is the program to run. The language argument is the
   223  // type of input file as passed to the GCC driver's -x option.
   224  //
   225  // For these tools we have no -V=full option to dump the build ID,
   226  // but we can run the tool with -v -### to reliably get the compiler proper
   227  // and hash that. That will work in the presence of -toolexec.
   228  //
   229  // In order to get reproducible builds for released compilers, we
   230  // detect a released compiler by the absence of "experimental" in the
   231  // --version output, and in that case we just use the version string.
   232  func (b *Builder) gccgoToolID(name, language string) (string, error) {
   233  	key := name + "." + language
   234  	b.id.Lock()
   235  	id := b.toolIDCache[key]
   236  	b.id.Unlock()
   237  
   238  	if id != "" {
   239  		return id, nil
   240  	}
   241  
   242  	// Invoke the driver with -### to see the subcommands and the
   243  	// version strings. Use -x to set the language. Pretend to
   244  	// compile an empty file on standard input.
   245  	cmdline := str.StringList(cfg.BuildToolexec, name, "-###", "-x", language, "-c", "-")
   246  	cmd := exec.Command(cmdline[0], cmdline[1:]...)
   247  	cmd.Env = base.AppendPWD(os.Environ(), cmd.Dir)
   248  	// Force untranslated output so that we see the string "version".
   249  	cmd.Env = append(cmd.Env, "LC_ALL=C")
   250  	out, err := cmd.CombinedOutput()
   251  	if err != nil {
   252  		return "", fmt.Errorf("%s: %v; output: %q", name, err, out)
   253  	}
   254  
   255  	version := ""
   256  	lines := strings.Split(string(out), "\n")
   257  	for _, line := range lines {
   258  		if fields := strings.Fields(line); len(fields) > 1 && fields[1] == "version" {
   259  			version = line
   260  			break
   261  		}
   262  	}
   263  	if version == "" {
   264  		return "", fmt.Errorf("%s: can not find version number in %q", name, out)
   265  	}
   266  
   267  	if !strings.Contains(version, "experimental") {
   268  		// This is a release. Use this line as the tool ID.
   269  		id = version
   270  	} else {
   271  		// This is a development version. The first line with
   272  		// a leading space is the compiler proper.
   273  		compiler := ""
   274  		for _, line := range lines {
   275  			if len(line) > 1 && line[0] == ' ' {
   276  				compiler = line
   277  				break
   278  			}
   279  		}
   280  		if compiler == "" {
   281  			return "", fmt.Errorf("%s: can not find compilation command in %q", name, out)
   282  		}
   283  
   284  		fields := strings.Fields(compiler)
   285  		if len(fields) == 0 {
   286  			return "", fmt.Errorf("%s: compilation command confusion %q", name, out)
   287  		}
   288  		exe := fields[0]
   289  		if !strings.ContainsAny(exe, `/\`) {
   290  			if lp, err := exec.LookPath(exe); err == nil {
   291  				exe = lp
   292  			}
   293  		}
   294  		id, err = buildid.ReadFile(exe)
   295  		if err != nil {
   296  			return "", err
   297  		}
   298  
   299  		// If we can't find a build ID, use a hash.
   300  		if id == "" {
   301  			id = b.fileHash(exe)
   302  		}
   303  	}
   304  
   305  	b.id.Lock()
   306  	b.toolIDCache[key] = id
   307  	b.id.Unlock()
   308  
   309  	return id, nil
   310  }
   311  
   312  // Check if assembler used by gccgo is GNU as.
   313  func assemblerIsGas() bool {
   314  	cmd := exec.Command(BuildToolchain.compiler(), "-print-prog-name=as")
   315  	assembler, err := cmd.Output()
   316  	if err == nil {
   317  		cmd := exec.Command(strings.TrimSpace(string(assembler)), "--version")
   318  		out, err := cmd.Output()
   319  		return err == nil && strings.Contains(string(out), "GNU")
   320  	} else {
   321  		return false
   322  	}
   323  }
   324  
   325  // gccgoBuildIDFile creates an assembler file that records the
   326  // action's build ID in an SHF_EXCLUDE section for ELF files or
   327  // in a CSECT in XCOFF files.
   328  func (b *Builder) gccgoBuildIDFile(a *Action) (string, error) {
   329  	sfile := a.Objdir + "_buildid.s"
   330  
   331  	var buf bytes.Buffer
   332  	if cfg.Goos == "aix" {
   333  		fmt.Fprintf(&buf, "\t.csect .go.buildid[XO]\n")
   334  	} else if (cfg.Goos != "solaris" && cfg.Goos != "illumos") || assemblerIsGas() {
   335  		fmt.Fprintf(&buf, "\t"+`.section .go.buildid,"e"`+"\n")
   336  	} else if cfg.Goarch == "sparc" || cfg.Goarch == "sparc64" {
   337  		fmt.Fprintf(&buf, "\t"+`.section ".go.buildid",#exclude`+"\n")
   338  	} else { // cfg.Goarch == "386" || cfg.Goarch == "amd64"
   339  		fmt.Fprintf(&buf, "\t"+`.section .go.buildid,#exclude`+"\n")
   340  	}
   341  	fmt.Fprintf(&buf, "\t.byte ")
   342  	for i := 0; i < len(a.buildID); i++ {
   343  		if i > 0 {
   344  			if i%8 == 0 {
   345  				fmt.Fprintf(&buf, "\n\t.byte ")
   346  			} else {
   347  				fmt.Fprintf(&buf, ",")
   348  			}
   349  		}
   350  		fmt.Fprintf(&buf, "%#02x", a.buildID[i])
   351  	}
   352  	fmt.Fprintf(&buf, "\n")
   353  	if cfg.Goos != "solaris" && cfg.Goos != "illumos" && cfg.Goos != "aix" {
   354  		secType := "@progbits"
   355  		if cfg.Goarch == "arm" {
   356  			secType = "%progbits"
   357  		}
   358  		fmt.Fprintf(&buf, "\t"+`.section .note.GNU-stack,"",%s`+"\n", secType)
   359  		fmt.Fprintf(&buf, "\t"+`.section .note.GNU-split-stack,"",%s`+"\n", secType)
   360  	}
   361  
   362  	if cfg.BuildN || cfg.BuildX {
   363  		for _, line := range bytes.Split(buf.Bytes(), []byte("\n")) {
   364  			b.Showcmd("", "echo '%s' >> %s", line, sfile)
   365  		}
   366  		if cfg.BuildN {
   367  			return sfile, nil
   368  		}
   369  	}
   370  
   371  	if err := ioutil.WriteFile(sfile, buf.Bytes(), 0666); err != nil {
   372  		return "", err
   373  	}
   374  
   375  	return sfile, nil
   376  }
   377  
   378  // buildID returns the build ID found in the given file.
   379  // If no build ID is found, buildID returns the content hash of the file.
   380  func (b *Builder) buildID(file string) string {
   381  	b.id.Lock()
   382  	id := b.buildIDCache[file]
   383  	b.id.Unlock()
   384  
   385  	if id != "" {
   386  		return id
   387  	}
   388  
   389  	id, err := buildid.ReadFile(file)
   390  	if err != nil {
   391  		id = b.fileHash(file)
   392  	}
   393  
   394  	b.id.Lock()
   395  	b.buildIDCache[file] = id
   396  	b.id.Unlock()
   397  
   398  	return id
   399  }
   400  
   401  // fileHash returns the content hash of the named file.
   402  func (b *Builder) fileHash(file string) string {
   403  	sum, err := cache.FileHash(file)
   404  	if err != nil {
   405  		return ""
   406  	}
   407  	return hashToString(sum)
   408  }
   409  
   410  // useCache tries to satisfy the action a, which has action ID actionHash,
   411  // by using a cached result from an earlier build. At the moment, the only
   412  // cached result is the installed package or binary at target.
   413  // If useCache decides that the cache can be used, it sets a.buildID
   414  // and a.built for use by parent actions and then returns true.
   415  // Otherwise it sets a.buildID to a temporary build ID for use in the build
   416  // and returns false. When useCache returns false the expectation is that
   417  // the caller will build the target and then call updateBuildID to finish the
   418  // build ID computation.
   419  // When useCache returns false, it may have initiated buffering of output
   420  // during a's work. The caller should defer b.flushOutput(a), to make sure
   421  // that flushOutput is eventually called regardless of whether the action
   422  // succeeds. The flushOutput call must happen after updateBuildID.
   423  func (b *Builder) useCache(a *Action, actionHash cache.ActionID, target string) bool {
   424  	// The second half of the build ID here is a placeholder for the content hash.
   425  	// It's important that the overall buildID be unlikely verging on impossible
   426  	// to appear in the output by chance, but that should be taken care of by
   427  	// the actionID half; if it also appeared in the input that would be like an
   428  	// engineered 96-bit partial SHA256 collision.
   429  	a.actionID = actionHash
   430  	actionID := hashToString(actionHash)
   431  	if a.json != nil {
   432  		a.json.ActionID = actionID
   433  	}
   434  	contentID := actionID // temporary placeholder, likely unique
   435  	a.buildID = actionID + buildIDSeparator + contentID
   436  
   437  	// Executable binaries also record the main build ID in the middle.
   438  	// See "Build IDs" comment above.
   439  	if a.Mode == "link" {
   440  		mainpkg := a.Deps[0]
   441  		a.buildID = actionID + buildIDSeparator + mainpkg.buildID + buildIDSeparator + contentID
   442  	}
   443  
   444  	// Check to see if target exists and matches the expected action ID.
   445  	// If so, it's up to date and we can reuse it instead of rebuilding it.
   446  	var buildID string
   447  	if target != "" && !cfg.BuildA {
   448  		buildID, _ = buildid.ReadFile(target)
   449  		if strings.HasPrefix(buildID, actionID+buildIDSeparator) {
   450  			a.buildID = buildID
   451  			if a.json != nil {
   452  				a.json.BuildID = a.buildID
   453  			}
   454  			a.built = target
   455  			// Poison a.Target to catch uses later in the build.
   456  			a.Target = "DO NOT USE - " + a.Mode
   457  			return true
   458  		}
   459  	}
   460  
   461  	// Special case for building a main package: if the only thing we
   462  	// want the package for is to link a binary, and the binary is
   463  	// already up-to-date, then to avoid a rebuild, report the package
   464  	// as up-to-date as well. See "Build IDs" comment above.
   465  	// TODO(rsc): Rewrite this code to use a TryCache func on the link action.
   466  	if target != "" && !cfg.BuildA && !b.NeedExport && a.Mode == "build" && len(a.triggers) == 1 && a.triggers[0].Mode == "link" {
   467  		buildID, err := buildid.ReadFile(target)
   468  		if err == nil {
   469  			id := strings.Split(buildID, buildIDSeparator)
   470  			if len(id) == 4 && id[1] == actionID {
   471  				// Temporarily assume a.buildID is the package build ID
   472  				// stored in the installed binary, and see if that makes
   473  				// the upcoming link action ID a match. If so, report that
   474  				// we built the package, safe in the knowledge that the
   475  				// link step will not ask us for the actual package file.
   476  				// Note that (*Builder).LinkAction arranged that all of
   477  				// a.triggers[0]'s dependencies other than a are also
   478  				// dependencies of a, so that we can be sure that,
   479  				// other than a.buildID, b.linkActionID is only accessing
   480  				// build IDs of completed actions.
   481  				oldBuildID := a.buildID
   482  				a.buildID = id[1] + buildIDSeparator + id[2]
   483  				linkID := hashToString(b.linkActionID(a.triggers[0]))
   484  				if id[0] == linkID {
   485  					// Best effort attempt to display output from the compile and link steps.
   486  					// If it doesn't work, it doesn't work: reusing the cached binary is more
   487  					// important than reprinting diagnostic information.
   488  					if c := cache.Default(); c != nil {
   489  						showStdout(b, c, a.actionID, "stdout")      // compile output
   490  						showStdout(b, c, a.actionID, "link-stdout") // link output
   491  					}
   492  
   493  					// Poison a.Target to catch uses later in the build.
   494  					a.Target = "DO NOT USE - main build pseudo-cache Target"
   495  					a.built = "DO NOT USE - main build pseudo-cache built"
   496  					if a.json != nil {
   497  						a.json.BuildID = a.buildID
   498  					}
   499  					return true
   500  				}
   501  				// Otherwise restore old build ID for main build.
   502  				a.buildID = oldBuildID
   503  			}
   504  		}
   505  	}
   506  
   507  	// Special case for linking a test binary: if the only thing we
   508  	// want the binary for is to run the test, and the test result is cached,
   509  	// then to avoid the link step, report the link as up-to-date.
   510  	// We avoid the nested build ID problem in the previous special case
   511  	// by recording the test results in the cache under the action ID half.
   512  	if !cfg.BuildA && len(a.triggers) == 1 && a.triggers[0].TryCache != nil && a.triggers[0].TryCache(b, a.triggers[0]) {
   513  		// Best effort attempt to display output from the compile and link steps.
   514  		// If it doesn't work, it doesn't work: reusing the test result is more
   515  		// important than reprinting diagnostic information.
   516  		if c := cache.Default(); c != nil {
   517  			showStdout(b, c, a.Deps[0].actionID, "stdout")      // compile output
   518  			showStdout(b, c, a.Deps[0].actionID, "link-stdout") // link output
   519  		}
   520  
   521  		// Poison a.Target to catch uses later in the build.
   522  		a.Target = "DO NOT USE -  pseudo-cache Target"
   523  		a.built = "DO NOT USE - pseudo-cache built"
   524  		return true
   525  	}
   526  
   527  	if b.IsCmdList {
   528  		// Invoked during go list to compute and record staleness.
   529  		if p := a.Package; p != nil && !p.Stale {
   530  			p.Stale = true
   531  			if cfg.BuildA {
   532  				p.StaleReason = "build -a flag in use"
   533  			} else {
   534  				p.StaleReason = "build ID mismatch"
   535  				for _, p1 := range p.Internal.Imports {
   536  					if p1.Stale && p1.StaleReason != "" {
   537  						if strings.HasPrefix(p1.StaleReason, "stale dependency: ") {
   538  							p.StaleReason = p1.StaleReason
   539  							break
   540  						}
   541  						if strings.HasPrefix(p.StaleReason, "build ID mismatch") {
   542  							p.StaleReason = "stale dependency: " + p1.ImportPath
   543  						}
   544  					}
   545  				}
   546  			}
   547  		}
   548  
   549  		// Fall through to update a.buildID from the build artifact cache,
   550  		// which will affect the computation of buildIDs for targets
   551  		// higher up in the dependency graph.
   552  	}
   553  
   554  	// Check the build artifact cache.
   555  	// We treat hits in this cache as being "stale" for the purposes of go list
   556  	// (in effect, "stale" means whether p.Target is up-to-date),
   557  	// but we're still happy to use results from the build artifact cache.
   558  	if c := cache.Default(); c != nil {
   559  		if !cfg.BuildA {
   560  			if file, _, err := c.GetFile(actionHash); err == nil {
   561  				if buildID, err := buildid.ReadFile(file); err == nil {
   562  					if err := showStdout(b, c, a.actionID, "stdout"); err == nil {
   563  						a.built = file
   564  						a.Target = "DO NOT USE - using cache"
   565  						a.buildID = buildID
   566  						if a.json != nil {
   567  							a.json.BuildID = a.buildID
   568  						}
   569  						if p := a.Package; p != nil {
   570  							// Clearer than explaining that something else is stale.
   571  							p.StaleReason = "not installed but available in build cache"
   572  						}
   573  						return true
   574  					}
   575  				}
   576  			}
   577  		}
   578  
   579  		// Begin saving output for later writing to cache.
   580  		a.output = []byte{}
   581  	}
   582  
   583  	return false
   584  }
   585  
   586  func showStdout(b *Builder, c *cache.Cache, actionID cache.ActionID, key string) error {
   587  	stdout, stdoutEntry, err := c.GetBytes(cache.Subkey(actionID, key))
   588  	if err != nil {
   589  		return err
   590  	}
   591  
   592  	if len(stdout) > 0 {
   593  		if cfg.BuildX || cfg.BuildN {
   594  			b.Showcmd("", "%s  # internal", joinUnambiguously(str.StringList("cat", c.OutputFile(stdoutEntry.OutputID))))
   595  		}
   596  		if !cfg.BuildN {
   597  			b.Print(string(stdout))
   598  		}
   599  	}
   600  	return nil
   601  }
   602  
   603  // flushOutput flushes the output being queued in a.
   604  func (b *Builder) flushOutput(a *Action) {
   605  	b.Print(string(a.output))
   606  	a.output = nil
   607  }
   608  
   609  // updateBuildID updates the build ID in the target written by action a.
   610  // It requires that useCache was called for action a and returned false,
   611  // and that the build was then carried out and given the temporary
   612  // a.buildID to record as the build ID in the resulting package or binary.
   613  // updateBuildID computes the final content ID and updates the build IDs
   614  // in the binary.
   615  //
   616  // Keep in sync with src/cmd/buildid/buildid.go
   617  func (b *Builder) updateBuildID(a *Action, target string, rewrite bool) error {
   618  	if cfg.BuildX || cfg.BuildN {
   619  		if rewrite {
   620  			b.Showcmd("", "%s # internal", joinUnambiguously(str.StringList(base.Tool("buildid"), "-w", target)))
   621  		}
   622  		if cfg.BuildN {
   623  			return nil
   624  		}
   625  	}
   626  
   627  	// Cache output from compile/link, even if we don't do the rest.
   628  	if c := cache.Default(); c != nil {
   629  		switch a.Mode {
   630  		case "build":
   631  			c.PutBytes(cache.Subkey(a.actionID, "stdout"), a.output)
   632  		case "link":
   633  			// Even though we don't cache the binary, cache the linker text output.
   634  			// We might notice that an installed binary is up-to-date but still
   635  			// want to pretend to have run the linker.
   636  			// Store it under the main package's action ID
   637  			// to make it easier to find when that's all we have.
   638  			for _, a1 := range a.Deps {
   639  				if p1 := a1.Package; p1 != nil && p1.Name == "main" {
   640  					c.PutBytes(cache.Subkey(a1.actionID, "link-stdout"), a.output)
   641  					break
   642  				}
   643  			}
   644  		}
   645  	}
   646  
   647  	// Find occurrences of old ID and compute new content-based ID.
   648  	r, err := os.Open(target)
   649  	if err != nil {
   650  		return err
   651  	}
   652  	matches, hash, err := buildid.FindAndHash(r, a.buildID, 0)
   653  	r.Close()
   654  	if err != nil {
   655  		return err
   656  	}
   657  	newID := a.buildID[:strings.LastIndex(a.buildID, buildIDSeparator)] + buildIDSeparator + hashToString(hash)
   658  	if len(newID) != len(a.buildID) {
   659  		return fmt.Errorf("internal error: build ID length mismatch %q vs %q", a.buildID, newID)
   660  	}
   661  
   662  	// Replace with new content-based ID.
   663  	a.buildID = newID
   664  	if a.json != nil {
   665  		a.json.BuildID = a.buildID
   666  	}
   667  	if len(matches) == 0 {
   668  		// Assume the user specified -buildid= to override what we were going to choose.
   669  		return nil
   670  	}
   671  
   672  	if rewrite {
   673  		w, err := os.OpenFile(target, os.O_WRONLY, 0)
   674  		if err != nil {
   675  			return err
   676  		}
   677  		err = buildid.Rewrite(w, matches, newID)
   678  		if err != nil {
   679  			w.Close()
   680  			return err
   681  		}
   682  		if err := w.Close(); err != nil {
   683  			return err
   684  		}
   685  	}
   686  
   687  	// Cache package builds, but not binaries (link steps).
   688  	// The expectation is that binaries are not reused
   689  	// nearly as often as individual packages, and they're
   690  	// much larger, so the cache-footprint-to-utility ratio
   691  	// of binaries is much lower for binaries.
   692  	// Not caching the link step also makes sure that repeated "go run" at least
   693  	// always rerun the linker, so that they don't get too fast.
   694  	// (We don't want people thinking go is a scripting language.)
   695  	// Note also that if we start caching binaries, then we will
   696  	// copy the binaries out of the cache to run them, and then
   697  	// that will mean the go process is itself writing a binary
   698  	// and then executing it, so we will need to defend against
   699  	// ETXTBSY problems as discussed in exec.go and golang.org/issue/22220.
   700  	if c := cache.Default(); c != nil && a.Mode == "build" {
   701  		r, err := os.Open(target)
   702  		if err == nil {
   703  			if a.output == nil {
   704  				panic("internal error: a.output not set")
   705  			}
   706  			outputID, _, err := c.Put(a.actionID, r)
   707  			r.Close()
   708  			if err == nil && cfg.BuildX {
   709  				b.Showcmd("", "%s # internal", joinUnambiguously(str.StringList("cp", target, c.OutputFile(outputID))))
   710  			}
   711  			if b.NeedExport {
   712  				if err != nil {
   713  					return err
   714  				}
   715  				a.Package.Export = c.OutputFile(outputID)
   716  			}
   717  		}
   718  	}
   719  
   720  	return nil
   721  }
   722  

View as plain text