Black Lives Matter. Support the Equal Justice Initiative.

Source file src/cmd/dist/buildtool.go

Documentation: cmd/dist

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Build toolchain using Go 1.4.
     6  //
     7  // The general strategy is to copy the source files we need into
     8  // a new GOPATH workspace, adjust import paths appropriately,
     9  // invoke the Go 1.4 go command to build those sources,
    10  // and then copy the binaries back.
    11  
    12  package main
    13  
    14  import (
    15  	"fmt"
    16  	"os"
    17  	"path/filepath"
    18  	"runtime"
    19  	"strings"
    20  )
    21  
    22  // bootstrapDirs is a list of directories holding code that must be
    23  // compiled with a Go 1.4 toolchain to produce the bootstrapTargets.
    24  // All directories in this list are relative to and must be below $GOROOT/src.
    25  //
    26  // The list has have two kinds of entries: names beginning with cmd/ with
    27  // no other slashes, which are commands, and other paths, which are packages
    28  // supporting the commands. Packages in the standard library can be listed
    29  // if a newer copy needs to be substituted for the Go 1.4 copy when used
    30  // by the command packages.
    31  // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
    32  var bootstrapDirs = []string{
    33  	"cmd/asm",
    34  	"cmd/asm/internal/arch",
    35  	"cmd/asm/internal/asm",
    36  	"cmd/asm/internal/flags",
    37  	"cmd/asm/internal/lex",
    38  	"cmd/cgo",
    39  	"cmd/compile",
    40  	"cmd/compile/internal/amd64",
    41  	"cmd/compile/internal/arm",
    42  	"cmd/compile/internal/arm64",
    43  	"cmd/compile/internal/gc",
    44  	"cmd/compile/internal/logopt",
    45  	"cmd/compile/internal/mips",
    46  	"cmd/compile/internal/mips64",
    47  	"cmd/compile/internal/ppc64",
    48  	"cmd/compile/internal/riscv64",
    49  	"cmd/compile/internal/s390x",
    50  	"cmd/compile/internal/ssa",
    51  	"cmd/compile/internal/syntax",
    52  	"cmd/compile/internal/types",
    53  	"cmd/compile/internal/x86",
    54  	"cmd/compile/internal/wasm",
    55  	"cmd/internal/bio",
    56  	"cmd/internal/codesign",
    57  	"cmd/internal/gcprog",
    58  	"cmd/internal/dwarf",
    59  	"cmd/internal/edit",
    60  	"cmd/internal/goobj",
    61  	"cmd/internal/objabi",
    62  	"cmd/internal/obj",
    63  	"cmd/internal/obj/arm",
    64  	"cmd/internal/obj/arm64",
    65  	"cmd/internal/obj/mips",
    66  	"cmd/internal/obj/ppc64",
    67  	"cmd/internal/obj/riscv",
    68  	"cmd/internal/obj/s390x",
    69  	"cmd/internal/obj/x86",
    70  	"cmd/internal/obj/wasm",
    71  	"cmd/internal/pkgpath",
    72  	"cmd/internal/src",
    73  	"cmd/internal/sys",
    74  	"cmd/link",
    75  	"cmd/link/internal/amd64",
    76  	"cmd/link/internal/arm",
    77  	"cmd/link/internal/arm64",
    78  	"cmd/link/internal/benchmark",
    79  	"cmd/link/internal/ld",
    80  	"cmd/link/internal/loadelf",
    81  	"cmd/link/internal/loader",
    82  	"cmd/link/internal/loadmacho",
    83  	"cmd/link/internal/loadpe",
    84  	"cmd/link/internal/loadxcoff",
    85  	"cmd/link/internal/mips",
    86  	"cmd/link/internal/mips64",
    87  	"cmd/link/internal/ppc64",
    88  	"cmd/link/internal/riscv64",
    89  	"cmd/link/internal/s390x",
    90  	"cmd/link/internal/sym",
    91  	"cmd/link/internal/x86",
    92  	"compress/flate",
    93  	"compress/zlib",
    94  	"cmd/link/internal/wasm",
    95  	"container/heap",
    96  	"debug/dwarf",
    97  	"debug/elf",
    98  	"debug/macho",
    99  	"debug/pe",
   100  	"internal/goversion",
   101  	"internal/race",
   102  	"internal/unsafeheader",
   103  	"internal/xcoff",
   104  	"math/big",
   105  	"math/bits",
   106  	"sort",
   107  }
   108  
   109  // File prefixes that are ignored by go/build anyway, and cause
   110  // problems with editor generated temporary files (#18931).
   111  var ignorePrefixes = []string{
   112  	".",
   113  	"_",
   114  }
   115  
   116  // File suffixes that use build tags introduced since Go 1.4.
   117  // These must not be copied into the bootstrap build directory.
   118  // Also ignore test files.
   119  var ignoreSuffixes = []string{
   120  	"_arm64.s",
   121  	"_arm64.go",
   122  	"_riscv64.s",
   123  	"_riscv64.go",
   124  	"_wasm.s",
   125  	"_wasm.go",
   126  	"_test.s",
   127  }
   128  
   129  func bootstrapBuildTools() {
   130  	goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
   131  	if goroot_bootstrap == "" {
   132  		goroot_bootstrap = pathf("%s/go1.4", os.Getenv("HOME"))
   133  	}
   134  	xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
   135  
   136  	mkzbootstrap(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
   137  
   138  	// Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
   139  	// We use a subdirectory of $GOROOT/pkg because that's the
   140  	// space within $GOROOT where we store all generated objects.
   141  	// We could use a temporary directory outside $GOROOT instead,
   142  	// but it is easier to debug on failure if the files are in a known location.
   143  	workspace := pathf("%s/pkg/bootstrap", goroot)
   144  	xremoveall(workspace)
   145  	xatexit(func() { xremoveall(workspace) })
   146  	base := pathf("%s/src/bootstrap", workspace)
   147  	xmkdirall(base)
   148  
   149  	// Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
   150  	writefile("module bootstrap\n", pathf("%s/%s", base, "go.mod"), 0)
   151  	for _, dir := range bootstrapDirs {
   152  		src := pathf("%s/src/%s", goroot, dir)
   153  		dst := pathf("%s/%s", base, dir)
   154  		xmkdirall(dst)
   155  		if dir == "cmd/cgo" {
   156  			// Write to src because we need the file both for bootstrap
   157  			// and for later in the main build.
   158  			mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
   159  		}
   160  	Dir:
   161  		for _, name := range xreaddirfiles(src) {
   162  			for _, pre := range ignorePrefixes {
   163  				if strings.HasPrefix(name, pre) {
   164  					continue Dir
   165  				}
   166  			}
   167  			for _, suf := range ignoreSuffixes {
   168  				if strings.HasSuffix(name, suf) {
   169  					continue Dir
   170  				}
   171  			}
   172  			srcFile := pathf("%s/%s", src, name)
   173  			dstFile := pathf("%s/%s", dst, name)
   174  			text := bootstrapRewriteFile(srcFile)
   175  			writefile(text, dstFile, 0)
   176  		}
   177  	}
   178  
   179  	// Set up environment for invoking Go 1.4 go command.
   180  	// GOROOT points at Go 1.4 GOROOT,
   181  	// GOPATH points at our bootstrap workspace,
   182  	// GOBIN is empty, so that binaries are installed to GOPATH/bin,
   183  	// and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
   184  	// so that Go 1.4 builds whatever kind of binary it knows how to build.
   185  	// Restore GOROOT, GOPATH, and GOBIN when done.
   186  	// Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
   187  	// because setup will take care of those when bootstrapBuildTools returns.
   188  
   189  	defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
   190  	os.Setenv("GOROOT", goroot_bootstrap)
   191  
   192  	defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
   193  	os.Setenv("GOPATH", workspace)
   194  
   195  	defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
   196  	os.Setenv("GOBIN", "")
   197  
   198  	os.Setenv("GOOS", "")
   199  	os.Setenv("GOHOSTOS", "")
   200  	os.Setenv("GOARCH", "")
   201  	os.Setenv("GOHOSTARCH", "")
   202  
   203  	// Run Go 1.4 to build binaries. Use -gcflags=-l to disable inlining to
   204  	// workaround bugs in Go 1.4's compiler. See discussion thread:
   205  	// https://groups.google.com/d/msg/golang-dev/Ss7mCKsvk8w/Gsq7VYI0AwAJ
   206  	// Use the math_big_pure_go build tag to disable the assembly in math/big
   207  	// which may contain unsupported instructions.
   208  	// Note that if we are using Go 1.10 or later as bootstrap, the -gcflags=-l
   209  	// only applies to the final cmd/go binary, but that's OK: if this is Go 1.10
   210  	// or later we don't need to disable inlining to work around bugs in the Go 1.4 compiler.
   211  	cmd := []string{
   212  		pathf("%s/bin/go", goroot_bootstrap),
   213  		"install",
   214  		"-gcflags=-l",
   215  		"-tags=math_big_pure_go compiler_bootstrap",
   216  	}
   217  	if vflag > 0 {
   218  		cmd = append(cmd, "-v")
   219  	}
   220  	if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
   221  		cmd = append(cmd, "-toolexec="+tool)
   222  	}
   223  	cmd = append(cmd, "bootstrap/cmd/...")
   224  	run(base, ShowOutput|CheckExit, cmd...)
   225  
   226  	// Copy binaries into tool binary directory.
   227  	for _, name := range bootstrapDirs {
   228  		if !strings.HasPrefix(name, "cmd/") {
   229  			continue
   230  		}
   231  		name = name[len("cmd/"):]
   232  		if !strings.Contains(name, "/") {
   233  			copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
   234  		}
   235  	}
   236  
   237  	if vflag > 0 {
   238  		xprintf("\n")
   239  	}
   240  }
   241  
   242  var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
   243  
   244  // isUnneededSSARewriteFile reports whether srcFile is a
   245  // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
   246  // architecture that isn't for the current runtime.GOARCH.
   247  //
   248  // When unneeded is true archCaps is the rewrite base filename without
   249  // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
   250  func isUnneededSSARewriteFile(srcFile string) (archCaps string, unneeded bool) {
   251  	if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
   252  		return "", false
   253  	}
   254  	fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
   255  	if fileArch == "" {
   256  		return "", false
   257  	}
   258  	b := fileArch[0]
   259  	if b == '_' || ('a' <= b && b <= 'z') {
   260  		return "", false
   261  	}
   262  	archCaps = fileArch
   263  	fileArch = strings.ToLower(fileArch)
   264  	fileArch = strings.TrimSuffix(fileArch, "splitload")
   265  	if fileArch == os.Getenv("GOHOSTARCH") {
   266  		return "", false
   267  	}
   268  	if fileArch == strings.TrimSuffix(runtime.GOARCH, "le") {
   269  		return "", false
   270  	}
   271  	if fileArch == strings.TrimSuffix(os.Getenv("GOARCH"), "le") {
   272  		return "", false
   273  	}
   274  	return archCaps, true
   275  }
   276  
   277  func bootstrapRewriteFile(srcFile string) string {
   278  	// During bootstrap, generate dummy rewrite files for
   279  	// irrelevant architectures. We only need to build a bootstrap
   280  	// binary that works for the current runtime.GOARCH.
   281  	// This saves 6+ seconds of bootstrap.
   282  	if archCaps, ok := isUnneededSSARewriteFile(srcFile); ok {
   283  		return fmt.Sprintf(`// Code generated by go tool dist; DO NOT EDIT.
   284  
   285  package ssa
   286  
   287  func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
   288  func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
   289  `, archCaps, archCaps)
   290  	}
   291  
   292  	return bootstrapFixImports(srcFile)
   293  }
   294  
   295  func bootstrapFixImports(srcFile string) string {
   296  	lines := strings.SplitAfter(readfile(srcFile), "\n")
   297  	inBlock := false
   298  	for i, line := range lines {
   299  		if strings.HasPrefix(line, "import (") {
   300  			inBlock = true
   301  			continue
   302  		}
   303  		if inBlock && strings.HasPrefix(line, ")") {
   304  			inBlock = false
   305  			continue
   306  		}
   307  		if strings.HasPrefix(line, `import "`) || strings.HasPrefix(line, `import . "`) ||
   308  			inBlock && (strings.HasPrefix(line, "\t\"") || strings.HasPrefix(line, "\t. \"") || strings.HasPrefix(line, "\texec \"")) {
   309  			line = strings.Replace(line, `"cmd/`, `"bootstrap/cmd/`, -1)
   310  			// During bootstrap, must use plain os/exec.
   311  			line = strings.Replace(line, `exec "internal/execabs"`, `"os/exec"`, -1)
   312  			for _, dir := range bootstrapDirs {
   313  				if strings.HasPrefix(dir, "cmd/") {
   314  					continue
   315  				}
   316  				line = strings.Replace(line, `"`+dir+`"`, `"bootstrap/`+dir+`"`, -1)
   317  			}
   318  			lines[i] = line
   319  		}
   320  	}
   321  
   322  	lines[0] = "// Code generated by go tool dist; DO NOT EDIT.\n// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
   323  
   324  	return strings.Join(lines, "")
   325  }
   326  

View as plain text