buildtool.go

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Build toolchain using Go bootstrap version.
     6  //
     7  // The general strategy is to copy the source files we need into
     8  // a new GOPATH workspace, adjust import paths appropriately,
     9  // invoke the Go bootstrap toolchains go command to build those sources,
    10  // and then copy the binaries back.
    11  
    12  package main
    13  
    14  import (
    15  	"fmt"
    16  	"go/version"
    17  	"os"
    18  	"path/filepath"
    19  	"regexp"
    20  	"strings"
    21  )
    22  
    23  // bootstrapDirs is a list of directories holding code that must be
    24  // compiled with the Go bootstrap toolchain to produce the bootstrapTargets.
    25  // All directories in this list are relative to and must be below $GOROOT/src.
    26  //
    27  // The list has two kinds of entries: names beginning with cmd/ with
    28  // no other slashes, which are commands, and other paths, which are packages
    29  // supporting the commands. Packages in the standard library can be listed
    30  // if a newer copy needs to be substituted for the Go bootstrap copy when used
    31  // by the command packages. Paths ending with /... automatically
    32  // include all packages within subdirectories as well.
    33  // These will be imported during bootstrap as bootstrap/name, like bootstrap/math/big.
    34  var bootstrapDirs = []string{
    35  	"cmp",
    36  	"cmd/asm",
    37  	"cmd/asm/internal/...",
    38  	"cmd/cgo",
    39  	"cmd/compile",
    40  	"cmd/compile/internal/...",
    41  	"cmd/internal/archive",
    42  	"cmd/internal/bio",
    43  	"cmd/internal/codesign",
    44  	"cmd/internal/dwarf",
    45  	"cmd/internal/edit",
    46  	"cmd/internal/gcprog",
    47  	"cmd/internal/goobj",
    48  	"cmd/internal/hash",
    49  	"cmd/internal/macho",
    50  	"cmd/internal/obj/...",
    51  	"cmd/internal/objabi",
    52  	"cmd/internal/par",
    53  	"cmd/internal/pgo",
    54  	"cmd/internal/pkgpath",
    55  	"cmd/internal/quoted",
    56  	"cmd/internal/src",
    57  	"cmd/internal/sys",
    58  	"cmd/internal/telemetry",
    59  	"cmd/internal/telemetry/counter",
    60  	"cmd/link",
    61  	"cmd/link/internal/...",
    62  	"compress/flate",
    63  	"compress/zlib",
    64  	"container/heap",
    65  	"debug/dwarf",
    66  	"debug/elf",
    67  	"debug/macho",
    68  	"debug/pe",
    69  	"go/build/constraint",
    70  	"go/constant",
    71  	"go/version",
    72  	"internal/abi",
    73  	"internal/coverage",
    74  	"cmd/internal/cov/covcmd",
    75  	"internal/bisect",
    76  	"internal/buildcfg",
    77  	"internal/exportdata",
    78  	"internal/goarch",
    79  	"internal/godebugs",
    80  	"internal/goexperiment",
    81  	"internal/goroot",
    82  	"internal/gover",
    83  	"internal/goversion",
    84  	// internal/lazyregexp is provided by Go 1.17, which permits it to
    85  	// be imported by other packages in this list, but is not provided
    86  	// by the Go 1.17 version of gccgo. It's on this list only to
    87  	// support gccgo, and can be removed if we require gccgo 14 or later.
    88  	"internal/lazyregexp",
    89  	"internal/pkgbits",
    90  	"internal/platform",
    91  	"internal/profile",
    92  	"internal/race",
    93  	"internal/runtime/gc",
    94  	"internal/saferio",
    95  	"internal/strconv",
    96  	"internal/syscall/unix",
    97  	"internal/types/errors",
    98  	"internal/unsafeheader",
    99  	"internal/xcoff",
   100  	"internal/zstd",
   101  	"math/bits",
   102  	"sort",
   103  }
   104  
   105  // File prefixes that are ignored by go/build anyway, and cause
   106  // problems with editor generated temporary files (#18931).
   107  var ignorePrefixes = []string{
   108  	".",
   109  	"_",
   110  	"#",
   111  }
   112  
   113  // File suffixes that use build tags introduced since Go 1.17.
   114  // These must not be copied into the bootstrap build directory.
   115  // Also ignore test files.
   116  var ignoreSuffixes = []string{
   117  	"_test.s",
   118  	"_test.go",
   119  	// Skip PGO profile. No need to build toolchain1 compiler
   120  	// with PGO. And as it is not a text file the import path
   121  	// rewrite will break it.
   122  	".pgo",
   123  	// Skip editor backup files.
   124  	"~",
   125  }
   126  
   127  const minBootstrap = "go1.24.6"
   128  
   129  var tryDirs = []string{
   130  	"sdk/" + minBootstrap,
   131  	minBootstrap,
   132  }
   133  
   134  func bootstrapBuildTools() {
   135  	goroot_bootstrap := os.Getenv("GOROOT_BOOTSTRAP")
   136  	if goroot_bootstrap == "" {
   137  		home := os.Getenv("HOME")
   138  		goroot_bootstrap = pathf("%s/go1.4", home)
   139  		for _, d := range tryDirs {
   140  			if p := pathf("%s/%s", home, d); isdir(p) {
   141  				goroot_bootstrap = p
   142  			}
   143  		}
   144  	}
   145  
   146  	// check bootstrap version.
   147  	ver := run(pathf("%s/bin", goroot_bootstrap), CheckExit, pathf("%s/bin/go", goroot_bootstrap), "env", "GOVERSION")
   148  	// go env GOVERSION output like "go1.22.6\n" or "devel go1.24-ffb3e574 Thu Aug 29 20:16:26 2024 +0000\n".
   149  	ver = ver[:len(ver)-1]
   150  	if version.Compare(ver, version.Lang(minBootstrap)) > 0 && version.Compare(ver, minBootstrap) < 0 {
   151  		fatalf("%s does not meet the minimum bootstrap requirement of %s or later", ver, minBootstrap)
   152  	}
   153  
   154  	xprintf("Building Go toolchain1 using %s.\n", goroot_bootstrap)
   155  
   156  	mkbuildcfg(pathf("%s/src/internal/buildcfg/zbootstrap.go", goroot))
   157  	mkobjabi(pathf("%s/src/cmd/internal/objabi/zbootstrap.go", goroot))
   158  
   159  	// Use $GOROOT/pkg/bootstrap as the bootstrap workspace root.
   160  	// We use a subdirectory of $GOROOT/pkg because that's the
   161  	// space within $GOROOT where we store all generated objects.
   162  	// We could use a temporary directory outside $GOROOT instead,
   163  	// but it is easier to debug on failure if the files are in a known location.
   164  	workspace := pathf("%s/pkg/bootstrap", goroot)
   165  	xremoveall(workspace)
   166  	xatexit(func() { xremoveall(workspace) })
   167  	base := pathf("%s/src/bootstrap", workspace)
   168  	xmkdirall(base)
   169  
   170  	// Copy source code into $GOROOT/pkg/bootstrap and rewrite import paths.
   171  	minBootstrapVers := requiredBootstrapVersion(goModVersion()) // require the minimum required go version to build this go version in the go.mod file
   172  	writefile("module bootstrap\ngo "+minBootstrapVers+"\n", pathf("%s/%s", base, "go.mod"), 0)
   173  	for _, dir := range bootstrapDirs {
   174  		recurse := strings.HasSuffix(dir, "/...")
   175  		dir = strings.TrimSuffix(dir, "/...")
   176  		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
   177  			if err != nil {
   178  				fatalf("walking bootstrap dirs failed: %v: %v", path, err)
   179  			}
   180  
   181  			name := filepath.Base(path)
   182  			src := pathf("%s/src/%s", goroot, path)
   183  			dst := pathf("%s/%s", base, path)
   184  
   185  			if info.IsDir() {
   186  				if !recurse && path != dir || name == "testdata" {
   187  					return filepath.SkipDir
   188  				}
   189  
   190  				xmkdirall(dst)
   191  				if path == "cmd/cgo" {
   192  					// Write to src because we need the file both for bootstrap
   193  					// and for later in the main build.
   194  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", src))
   195  					mkzdefaultcc("", pathf("%s/zdefaultcc.go", dst))
   196  				}
   197  				return nil
   198  			}
   199  
   200  			for _, pre := range ignorePrefixes {
   201  				if strings.HasPrefix(name, pre) {
   202  					return nil
   203  				}
   204  			}
   205  			for _, suf := range ignoreSuffixes {
   206  				if strings.HasSuffix(name, suf) {
   207  					return nil
   208  				}
   209  			}
   210  
   211  			text := bootstrapRewriteFile(src)
   212  			writefile(text, dst, 0)
   213  			return nil
   214  		})
   215  	}
   216  
   217  	// Set up environment for invoking Go bootstrap toolchains go command.
   218  	// GOROOT points at Go bootstrap GOROOT,
   219  	// GOPATH points at our bootstrap workspace,
   220  	// GOBIN is empty, so that binaries are installed to GOPATH/bin,
   221  	// and GOOS, GOHOSTOS, GOARCH, and GOHOSTOS are empty,
   222  	// so that Go bootstrap toolchain builds whatever kind of binary it knows how to build.
   223  	// Restore GOROOT, GOPATH, and GOBIN when done.
   224  	// Don't bother with GOOS, GOHOSTOS, GOARCH, and GOHOSTARCH,
   225  	// because setup will take care of those when bootstrapBuildTools returns.
   226  
   227  	defer os.Setenv("GOROOT", os.Getenv("GOROOT"))
   228  	os.Setenv("GOROOT", goroot_bootstrap)
   229  
   230  	defer os.Setenv("GOPATH", os.Getenv("GOPATH"))
   231  	os.Setenv("GOPATH", workspace)
   232  
   233  	defer os.Setenv("GOBIN", os.Getenv("GOBIN"))
   234  	os.Setenv("GOBIN", "")
   235  
   236  	os.Setenv("GOOS", "")
   237  	os.Setenv("GOHOSTOS", "")
   238  	os.Setenv("GOARCH", "")
   239  	os.Setenv("GOHOSTARCH", "")
   240  
   241  	// Run Go bootstrap to build binaries.
   242  	// Use the math_big_pure_go build tag to disable the assembly in math/big
   243  	// which may contain unsupported instructions.
   244  	// Use the purego build tag to disable other assembly code.
   245  	cmd := []string{
   246  		pathf("%s/bin/go", goroot_bootstrap),
   247  		"install",
   248  		"-tags=math_big_pure_go compiler_bootstrap purego",
   249  	}
   250  	if vflag > 0 {
   251  		cmd = append(cmd, "-v")
   252  	}
   253  	if tool := os.Getenv("GOBOOTSTRAP_TOOLEXEC"); tool != "" {
   254  		cmd = append(cmd, "-toolexec="+tool)
   255  	}
   256  	cmd = append(cmd, "bootstrap/cmd/...")
   257  	run(base, ShowOutput|CheckExit, cmd...)
   258  
   259  	// Copy binaries into tool binary directory.
   260  	for _, name := range bootstrapDirs {
   261  		if !strings.HasPrefix(name, "cmd/") {
   262  			continue
   263  		}
   264  		name = name[len("cmd/"):]
   265  		if !strings.Contains(name, "/") {
   266  			copyfile(pathf("%s/%s%s", tooldir, name, exe), pathf("%s/bin/%s%s", workspace, name, exe), writeExec)
   267  		}
   268  	}
   269  
   270  	if vflag > 0 {
   271  		xprintf("\n")
   272  	}
   273  }
   274  
   275  var ssaRewriteFileSubstring = filepath.FromSlash("src/cmd/compile/internal/ssa/rewrite")
   276  
   277  // isUnneededSSARewriteFile reports whether srcFile is a
   278  // src/cmd/compile/internal/ssa/rewriteARCHNAME.go file for an
   279  // architecture that isn't for the given GOARCH.
   280  //
   281  // When unneeded is true archCaps is the rewrite base filename without
   282  // the "rewrite" prefix or ".go" suffix: AMD64, 386, ARM, ARM64, etc.
   283  func isUnneededSSARewriteFile(srcFile, goArch string) (archCaps string, unneeded bool) {
   284  	if !strings.Contains(srcFile, ssaRewriteFileSubstring) {
   285  		return "", false
   286  	}
   287  	fileArch := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(srcFile), "rewrite"), ".go")
   288  	if fileArch == "" {
   289  		return "", false
   290  	}
   291  	b := fileArch[0]
   292  	if b == '_' || ('a' <= b && b <= 'z') {
   293  		return "", false
   294  	}
   295  	archCaps = fileArch
   296  	fileArch = strings.ToLower(fileArch)
   297  	fileArch = strings.TrimSuffix(fileArch, "splitload")
   298  	fileArch = strings.TrimSuffix(fileArch, "latelower")
   299  	if fileArch == goArch {
   300  		return "", false
   301  	}
   302  	if fileArch == strings.TrimSuffix(goArch, "le") {
   303  		return "", false
   304  	}
   305  	return archCaps, true
   306  }
   307  
   308  func bootstrapRewriteFile(srcFile string) string {
   309  	// During bootstrap, generate dummy rewrite files for
   310  	// irrelevant architectures. We only need to build a bootstrap
   311  	// binary that works for the current gohostarch.
   312  	// This saves 6+ seconds of bootstrap.
   313  	if archCaps, ok := isUnneededSSARewriteFile(srcFile, gohostarch); ok {
   314  		return fmt.Sprintf(`%spackage ssa
   315  
   316  func rewriteValue%s(v *Value) bool { panic("unused during bootstrap") }
   317  func rewriteBlock%s(b *Block) bool { panic("unused during bootstrap") }
   318  `, generatedHeader, archCaps, archCaps)
   319  	}
   320  
   321  	return bootstrapFixImports(srcFile)
   322  }
   323  
   324  var (
   325  	importRE      = regexp.MustCompile(`\Aimport\s+(\.|[A-Za-z0-9_]+)?\s*"([^"]+)"\s*(//.*)?\n\z`)
   326  	importBlockRE = regexp.MustCompile(`\A\s*(?:(\.|[A-Za-z0-9_]+)?\s*"([^"]+)")?\s*(//.*)?\n\z`)
   327  )
   328  
   329  func bootstrapFixImports(srcFile string) string {
   330  	text := readfile(srcFile)
   331  	lines := strings.SplitAfter(text, "\n")
   332  	inBlock := false
   333  	inComment := false
   334  	for i, line := range lines {
   335  		if strings.HasSuffix(line, "*/\n") {
   336  			inComment = false
   337  		}
   338  		if strings.HasSuffix(line, "/*\n") {
   339  			inComment = true
   340  		}
   341  		if inComment {
   342  			continue
   343  		}
   344  		if strings.HasPrefix(line, "import (") {
   345  			inBlock = true
   346  			continue
   347  		}
   348  		if inBlock && strings.HasPrefix(line, ")") {
   349  			inBlock = false
   350  			continue
   351  		}
   352  
   353  		var m []string
   354  		if !inBlock {
   355  			if !strings.HasPrefix(line, "import ") {
   356  				continue
   357  			}
   358  			m = importRE.FindStringSubmatch(line)
   359  			if m == nil {
   360  				fatalf("%s:%d: invalid import declaration: %q", srcFile, i+1, line)
   361  			}
   362  		} else {
   363  			m = importBlockRE.FindStringSubmatch(line)
   364  			if m == nil {
   365  				fatalf("%s:%d: invalid import block line", srcFile, i+1)
   366  			}
   367  			if m[2] == "" {
   368  				continue
   369  			}
   370  		}
   371  
   372  		path := m[2]
   373  		if strings.HasPrefix(path, "cmd/") {
   374  			path = "bootstrap/" + path
   375  		} else {
   376  			for _, dir := range bootstrapDirs {
   377  				if path == dir {
   378  					path = "bootstrap/" + dir
   379  					break
   380  				}
   381  			}
   382  		}
   383  
   384  		// Rewrite use of internal/reflectlite to be plain reflect.
   385  		if path == "internal/reflectlite" {
   386  			lines[i] = strings.ReplaceAll(line, `"reflect"`, `reflectlite "reflect"`)
   387  			continue
   388  		}
   389  
   390  		// Otherwise, reject direct imports of internal packages,
   391  		// since that implies knowledge of internal details that might
   392  		// change from one bootstrap toolchain to the next.
   393  		// There are many internal packages that are listed in
   394  		// bootstrapDirs and made into bootstrap copies based on the
   395  		// current repo's source code. Those are fine; this is catching
   396  		// references to internal packages in the older bootstrap toolchain.
   397  		if strings.HasPrefix(path, "internal/") {
   398  			fatalf("%s:%d: bootstrap-copied source file cannot import %s", srcFile, i+1, path)
   399  		}
   400  		if path != m[2] {
   401  			lines[i] = strings.ReplaceAll(line, `"`+m[2]+`"`, `"`+path+`"`)
   402  		}
   403  	}
   404  
   405  	lines[0] = generatedHeader + "// This is a bootstrap copy of " + srcFile + "\n\n//line " + srcFile + ":1\n" + lines[0]
   406  
   407  	return strings.Join(lines, "")
   408  }
   409
View as plain text