buildtool.go

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Build toolchain using Go bootstrap version.
     6  //
     7  // The general strategy is to copy the source files we need into
     8  // a new GOPATH workspace, adjust import paths appropriately,
     9  // invoke the Go bootstrap toolchains go command to build those sources,
    10  // and then copy the binaries back.
    11  
    12  package main
    13  
    14  import (
    15  	"fmt"
    16  	"go/version"
    17  	"os"
    18  	"path/filepath"
    19  	"regexp"
    20  	"strings"
    21  )
    22  
    23  // bootstrapDirs is a list of directories holding code that must be
    24  // compiled with the Go bootstrap toolchain to produce the bootstrapTargets.
    25  // All directories in this list are relative to and must be below $GOROOT/src.
    26  //
    27  // The list has two kinds of entries: names beginning with cmd/ with
    28  // no other slashes, which are commands, and other paths, which are packages
    29  // supporting the commands. Packages in the standard library can be listed
    30  // if a newer copy needs to be substituted for the Go bootstrap copy when used
    31  // by the command packages. Paths ending with /... automatically
    32  // include all packages within subdirectories as well.
    33  // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
    34  var bootstrapDirs = []string{
    35  	"cmp",
    36  	"cmd/asm",
    37  	"cmd/asm/internal/...",
    38  	"cmd/cgo",
    39  	"cmd/compile",
    40  	"cmd/compile/internal/...",
    41  	"cmd/internal/archive",
    42  	"cmd/internal/bio",
    43  	"cmd/internal/codesign",
    44  	"cmd/internal/dwarf",
    45  	"cmd/internal/edit",
    46  	"cmd/internal/gcprog",
    47  	"cmd/internal/goobj",
    48  	"cmd/internal/hash",
    49  	"cmd/internal/macho",
    50  	"cmd/internal/obj/...",
    51  	"cmd/internal/objabi",
    52  	"cmd/internal/pgo",
    53  	"cmd/internal/pkgpath",
    54  	"cmd/internal/quoted",
    55  	"cmd/internal/src",
    56  	"cmd/internal/sys",
    57  	"cmd/internal/telemetry",
    58  	"cmd/internal/telemetry/counter",
    59  	"cmd/link",
    60  	"cmd/link/internal/...",
    61  	"compress/flate",
    62  	"compress/zlib",
    63  	"container/heap",
    64  	"debug/dwarf",
    65  	"debug/elf",
    66  	"debug/macho",
    67  	"debug/pe",
    68  	"go/build/constraint",
    69  	"go/constant",
    70  	"go/version",
    71  	"internal/abi",
    72  	"internal/coverage",
    73  	"cmd/internal/cov/covcmd",
    74  	"internal/bisect",
    75  	"internal/buildcfg",
    76  	"internal/exportdata",
    77  	"internal/goarch",
    78  	"internal/godebugs",
    79  	"internal/goexperiment",
    80  	"internal/goroot",
    81  	"internal/gover",
    82  	"internal/goversion",
    83  	// internal/lazyregexp is provided by Go 1.17, which permits it to
    84  	// be imported by other packages in this list, but is not provided
    85  	// by the Go 1.17 version of gccgo. It's on this list only to
    86  	// support gccgo, and can be removed if we require gccgo 14 or later.
    87  	"internal/lazyregexp",
    88  	"internal/pkgbits",
    89  	"internal/platform",
    90  	"internal/profile",
    91  	"internal/race",
    92  	"internal/saferio",
    93  	"internal/syscall/unix",
    94  	"internal/types/errors",
    95  	"internal/unsafeheader",
    96  	"internal/xcoff",
    97  	"internal/zstd",
    98  	"math/bits",
    99  	"sort",
   100  }
   101  
   102  // File prefixes that are ignored by go/build anyway, and cause
   103  // problems with editor generated temporary files (#18931).
   104  var ignorePrefixes = []string{
   105  	".",
   106  	"_",
   107  	"#",
   108  }
   109  
   110  // File suffixes that use build tags introduced since Go 1.17.
   111  // These must not be copied into the bootstrap build directory.
   112  // Also ignore test files.
   113  var ignoreSuffixes = []string{
   114  	"_test.s",
   115  	"_test.go",
   116  	// Skip PGO profile. No need to build toolchain1 compiler
   117  	// with PGO. And as it is not a text file the import path
   118  	// rewrite will break it.
   119  	".pgo",
   120  	// Skip editor backup files.
   121  	"~",
   122  }
   123  
   124  const minBootstrap = "go1.22.6"
   125  
   126  var tryDirs = []string{
   127  	"sdk/" + minBootstrap,
   128  	minBootstrap,
   129  }
   130  
   131  func bootstrapBuildTools() {
   132  	goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
   133  	if goroot_bootstrap == "" {
   134  		home := os.Getenv("HOME")
   135  		goroot_bootstrap = pathf("%s/go1.4", home)
   136  		for _, d := range tryDirs {
   137  			if p := pathf("%s/%s", home, d); isdir(p) {
   138  				goroot_bootstrap = p
   139  			}
   140  		}
   141  	}
   142  
   143  	// check bootstrap version.
   144  	ver := run(pathf("%s/bin", goroot_bootstrap), CheckExit, pathf("%s/bin/go", goroot_bootstrap), "env", "GOVERSION")
   145  	// go env GOVERSION output like "go1.22.6\n" or "devel go1.24-ffb3e574 Thu Aug 29 20:16:26 2024 +0000\n".
   146  	ver = ver[:len(ver)-1]
   147  	if version.Compare(ver, version.Lang(minBootstrap)) > 0 && version.Compare(ver, minBootstrap) < 0 {
   148  		fatalf("%s does not meet the minimum bootstrap requirement of %s or later", ver, minBootstrap)
   149  	}
   150  
   151  	xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
   152  
   153  	mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
   154  	mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
   155  
   156  	// Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
   157  	// We use a subdirectory of $GOROOT/pkg because that's the
   158  	// space within $GOROOT where we store all generated objects.
   159  	// We could use a temporary directory outside $GOROOT instead,
   160  	// but it is easier to debug on failure if the files are in a known location.
   161  	workspace := pathf("%s/pkg/bootstrap", goroot)
   162  	xremoveall(workspace)
   163  	xatexit(func() { xremoveall(workspace) })
   164  	base := pathf("%s/src/bootstrap", workspace)
   165  	xmkdirall(base)
   166  
   167  	// Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
   168  	minBootstrapVers := requiredBootstrapVersion(goModVersion()) // require the minimum required go version to build this go version in the go.mod file
   169  	writefile("module bootstrap\ngo "+minBootstrapVers+"\n", pathf("%s/%s", base, "go.mod"), 0)
   170  	for _, dir := range bootstrapDirs {
   171  		recurse := strings.HasSuffix(dir, "/...")
   172  		dir = strings.TrimSuffix(dir, "/...")
   173  		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
   174  			if err != nil {
   175  				fatalf("walking bootstrap dirs failed: %v: %v", path, err)
   176  			}
   177  
   178  			name := filepath.Base(path)
   179  			src := pathf("%s/src/%s", goroot, path)
   180  			dst := pathf("%s/%s", base, path)
   181  
   182  			if info.IsDir() {
   183  				if !recurse && path != dir || name == "testdata" {
   184  					return filepath.SkipDir
   185  				}
   186  
   187  				xmkdirall(dst)
   188  				if path == "cmd/cgo" {
   189  					// Write to src because we need the file both for bootstrap
   190  					// and for later in the main build.
   191  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
   192  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
   193  				}
   194  				return nil
   195  			}
   196  
   197  			for _, pre := range ignorePrefixes {
   198  				if strings.HasPrefix(name, pre) {
   199  					return nil
   200  				}
   201  			}
   202  			for _, suf := range ignoreSuffixes {
   203  				if strings.HasSuffix(name, suf) {
   204  					return nil
   205  				}
   206  			}
   207  
   208  			text := bootstrapRewriteFile(src)
   209  			writefile(text, dst, 0)
   210  			return nil
   211  		})
   212  	}
   213  
   214  	// Set up environment for invoking Go bootstrap toolchains go command.
   215  	// GOROOT points at Go bootstrap GOROOT,
   216  	// GOPATH points at our bootstrap workspace,
   217  	// GOBIN is empty, so that binaries are installed to GOPATH/bin,
   218  	// and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
   219  	// so that Go bootstrap toolchain builds whatever kind of binary it knows how to build.
   220  	// Restore GOROOT, GOPATH, and GOBIN when done.
   221  	// Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
   222  	// because setup will take care of those when bootstrapBuildTools returns.
   223  
   224  	defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
   225  	os.Setenv("GOROOT", goroot_bootstrap)
   226  
   227  	defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
   228  	os.Setenv("GOPATH", workspace)
   229  
   230  	defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
   231  	os.Setenv("GOBIN", "")
   232  
   233  	os.Setenv("GOOS", "")
   234  	os.Setenv("GOHOSTOS", "")
   235  	os.Setenv("GOARCH", "")
   236  	os.Setenv("GOHOSTARCH", "")
   237  
   238  	// Run Go bootstrap to build binaries.
   239  	// Use the math_big_pure_go build tag to disable the assembly in math/big
   240  	// which may contain unsupported instructions.
   241  	// Use the purego build tag to disable other assembly code.
   242  	cmd := []string{
   243  		pathf("%s/bin/go", goroot_bootstrap),
   244  		"install",
   245  		"-tags=math_big_pure_go compiler_bootstrap purego",
   246  	}
   247  	if vflag > 0 {
   248  		cmd = append(cmd, "-v")
   249  	}
   250  	if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
   251  		cmd = append(cmd, "-toolexec="+tool)
   252  	}
   253  	cmd = append(cmd, "bootstrap/cmd/...")
   254  	run(base, ShowOutput|CheckExit, cmd...)
   255  
   256  	// Copy binaries into tool binary directory.
   257  	for _, name := range bootstrapDirs {
   258  		if !strings.HasPrefix(name, "cmd/") {
   259  			continue
   260  		}
   261  		name = name[len("cmd/"):]
   262  		if !strings.Contains(name, "/") {
   263  			copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
   264  		}
   265  	}
   266  
   267  	if vflag > 0 {
   268  		xprintf("\n")
   269  	}
   270  }
   271  
   272  var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
   273  
   274  // isUnneededSSARewriteFile reports whether srcFile is a
   275  // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
   276  // architecture that isn't for the given GOARCH.
   277  //
   278  // When unneeded is true archCaps is the rewrite base filename without
   279  // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
   280  func isUnneededSSARewriteFile(srcFile, goArch string) (archCaps string, unneeded bool) {
   281  	if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
   282  		return "", false
   283  	}
   284  	fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
   285  	if fileArch == "" {
   286  		return "", false
   287  	}
   288  	b := fileArch[0]
   289  	if b == '_' || ('a' <= b && b <= 'z') {
   290  		return "", false
   291  	}
   292  	archCaps = fileArch
   293  	fileArch = strings.ToLower(fileArch)
   294  	fileArch = strings.TrimSuffix(fileArch, "splitload")
   295  	fileArch = strings.TrimSuffix(fileArch, "latelower")
   296  	if fileArch == goArch {
   297  		return "", false
   298  	}
   299  	if fileArch == strings.TrimSuffix(goArch, "le") {
   300  		return "", false
   301  	}
   302  	return archCaps, true
   303  }
   304  
   305  func bootstrapRewriteFile(srcFile string) string {
   306  	// During bootstrap, generate dummy rewrite files for
   307  	// irrelevant architectures. We only need to build a bootstrap
   308  	// binary that works for the current gohostarch.
   309  	// This saves 6+ seconds of bootstrap.
   310  	if archCaps, ok := isUnneededSSARewriteFile(srcFile, gohostarch); ok {
   311  		return fmt.Sprintf(`%spackage ssa
   312  
   313  func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
   314  func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
   315  `, generatedHeader, archCaps, archCaps)
   316  	}
   317  
   318  	return bootstrapFixImports(srcFile)
   319  }
   320  
   321  var (
   322  	importRE      = regexp.MustCompile(`\Aimport\s+(\.|[A-Za-z0-9_]+)?\s*"([^"]+)"\s*(//.*)?\n\z`)
   323  	importBlockRE = regexp.MustCompile(`\A\s*(?:(\.|[A-Za-z0-9_]+)?\s*"([^"]+)")?\s*(//.*)?\n\z`)
   324  )
   325  
   326  func bootstrapFixImports(srcFile string) string {
   327  	text := readfile(srcFile)
   328  	lines := strings.SplitAfter(text, "\n")
   329  	inBlock := false
   330  	inComment := false
   331  	for i, line := range lines {
   332  		if strings.HasSuffix(line, "*/\n") {
   333  			inComment = false
   334  		}
   335  		if strings.HasSuffix(line, "/*\n") {
   336  			inComment = true
   337  		}
   338  		if inComment {
   339  			continue
   340  		}
   341  		if strings.HasPrefix(line, "import (") {
   342  			inBlock = true
   343  			continue
   344  		}
   345  		if inBlock && strings.HasPrefix(line, ")") {
   346  			inBlock = false
   347  			continue
   348  		}
   349  
   350  		var m []string
   351  		if !inBlock {
   352  			if !strings.HasPrefix(line, "import ") {
   353  				continue
   354  			}
   355  			m = importRE.FindStringSubmatch(line)
   356  			if m == nil {
   357  				fatalf("%s:%d: invalid import declaration: %q", srcFile, i+1, line)
   358  			}
   359  		} else {
   360  			m = importBlockRE.FindStringSubmatch(line)
   361  			if m == nil {
   362  				fatalf("%s:%d: invalid import block line", srcFile, i+1)
   363  			}
   364  			if m[2] == "" {
   365  				continue
   366  			}
   367  		}
   368  
   369  		path := m[2]
   370  		if strings.HasPrefix(path, "cmd/") {
   371  			path = "bootstrap/" + path
   372  		} else {
   373  			for _, dir := range bootstrapDirs {
   374  				if path == dir {
   375  					path = "bootstrap/" + dir
   376  					break
   377  				}
   378  			}
   379  		}
   380  
   381  		// Rewrite use of internal/reflectlite to be plain reflect.
   382  		if path == "internal/reflectlite" {
   383  			lines[i] = strings.ReplaceAll(line, `"reflect"`, `reflectlite "reflect"`)
   384  			continue
   385  		}
   386  
   387  		// Otherwise, reject direct imports of internal packages,
   388  		// since that implies knowledge of internal details that might
   389  		// change from one bootstrap toolchain to the next.
   390  		// There are many internal packages that are listed in
   391  		// bootstrapDirs and made into bootstrap copies based on the
   392  		// current repo's source code. Those are fine; this is catching
   393  		// references to internal packages in the older bootstrap toolchain.
   394  		if strings.HasPrefix(path, "internal/") {
   395  			fatalf("%s:%d: bootstrap-copied source file cannot import %s", srcFile, i+1, path)
   396  		}
   397  		if path != m[2] {
   398  			lines[i] = strings.ReplaceAll(line, `"`+m[2]+`"`, `"`+path+`"`)
   399  		}
   400  	}
   401  
   402  	lines[0] = generatedHeader + "// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
   403  
   404  	return strings.Join(lines, "")
   405  }
   406
View as plain text