cmd/dist: make toolchain build reproducible

- Build cmd with CGO_ENABLED=0. Doing so removes the C compiler
  toolchain from the reproducibility perimeter and also results in
  cmd/go and cmd/pprof binaries that are statically linked,
  so that they will run on a wider variety of systems.
  In particular the Linux versions will run on Alpine and NixOS
  without needing a simulation of libc.so.6.

  The potential downside of disabling cgo is that cmd/go and cmd/pprof
  use the pure Go network resolver instead of the host resolver on
  Unix systems. This means they will not be able to use non-DNS
  resolver mechanisms that may be specified in /etc/resolv.conf,
  such as mDNS. Neither program seems likely to need non-DNS names
  like those, however.

  macOS and Windows systems still use the host resolver, which they
  access without cgo.

- Build cmd with -trimpath when building a release.
  Doing so removes $GOPATH from the file name prefixes stored in the
  binary, so that the build directory does not leak into the final artifacts.

- When CC and CXX are empty, do not pick values to hard-code into
  the source tree and binaries. Instead, emit code that makes the
  right decision at runtime. In addition to reproducibility, this
  makes cross-compiled toolchains work better. A macOS toolchain
  cross-compiled on Linux will now correctly look for clang,
  instead of looking for gcc because it was built on Linux.

- Convert \ to / in file names stored in .a files.
  These are converted to / in the final binaries, but the hashes of
  the .a files affect the final build ID of the binaries. Without this
  change, builds of a Windows toolchain on Windows and non-Windows
  machines produce identical binaries except for the input hash part
  of the build ID.

- Due to the conversion of \ to / in .a files, convert back when
  reading inline bodies on Windows to preserve output file names
  in error messages.

Combined, these four changes (along with Go 1.20's removal of
installed pkg/**.a files and conversion of macOS net away from cgo)
make the output of make.bash fully reproducible, even when
cross-compiling: a released macOS toolchain built on Linux or Windows
will contain exactly the same bits as a released macOS toolchain
built on macOS.

The word "released" in the previous sentence is important.
For the build IDs in the binaries to work out the same on
both systems, a VERSION file must exist to provide a consistent
compiler build ID (instead of using a content hash of the binary).

For #24904.
Fixes #57007.

Change-Id: I665e1ef4ff207d6ff469452347dca5bfc81050e6
Reviewed-on: https://go-review.googlesource.com/c/go/+/454836
Reviewed-by: Bryan Mills <bcmills@google.com>
Run-TryBot: Russ Cox <rsc@golang.org>
Auto-Submit: Russ Cox <rsc@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
This commit is contained in:
Russ Cox 2022-12-02 10:39:30 -05:00 committed by Gopher Robot
parent dbe327a640
commit 8a27154bcd
9 changed files with 165 additions and 67 deletions

View file

@ -9,6 +9,7 @@ import (
"go/constant"
"internal/buildcfg"
"internal/pkgbits"
"path/filepath"
"strings"
"cmd/compile/internal/base"
@ -268,13 +269,14 @@ func (pr *pkgReader) posBaseIdx(idx pkgbits.Index) *src.PosBase {
// "$GOROOT" to buildcfg.GOROOT is a close-enough approximation to
// satisfy this.
//
// TODO(mdempsky): De-duplicate this logic with similar logic in
// cmd/link/internal/ld's expandGoroot. However, this will probably
// require being more consistent about when we use native vs UNIX
// file paths.
// The export data format only ever uses slash paths
// (for cross-operating-system reproducible builds),
// but error messages need to use native paths (backslash on Windows)
// as if they had been specified on the command line.
// (The go command always passes native paths to the compiler.)
const dollarGOROOT = "$GOROOT"
if buildcfg.GOROOT != "" && strings.HasPrefix(filename, dollarGOROOT) {
filename = buildcfg.GOROOT + filename[len(dollarGOROOT):]
filename = filepath.FromSlash(buildcfg.GOROOT + filename[len(dollarGOROOT):])
}
if r.Bool() {

View file

@ -222,7 +222,7 @@ func testInlineStack(t *testing.T, file, function string, wantStacks [][]int) {
sortInlineStacks(gotStacks)
sortInlineStacks(wantStacks)
if !reflect.DeepEqual(wantStacks, gotStacks) {
t.Errorf("wanted inlines %+v but got %+v", wantStacks, gotStacks)
t.Errorf("wanted inlines %+v but got %+v\n%s", wantStacks, gotStacks, dumpBytes)
}
}

119
src/cmd/dist/build.go vendored
View file

@ -53,7 +53,6 @@ var (
defaultldso string
rebuildall bool
defaultclang bool
noOpt bool
vflag int // verbosity
@ -210,12 +209,8 @@ func xinit() {
gogcflags = os.Getenv("BOOT_GO_GCFLAGS")
goldflags = os.Getenv("BOOT_GO_LDFLAGS")
cc, cxx := "gcc", "g++"
if defaultclang {
cc, cxx = "clang", "clang++"
}
defaultcc = compilerEnv("CC", cc)
defaultcxx = compilerEnv("CXX", cxx)
defaultcc = compilerEnv("CC", "")
defaultcxx = compilerEnv("CXX", "")
b = os.Getenv("PKG_CONFIG")
if b == "" {
@ -308,12 +303,34 @@ func compilerEnv(envName, def string) map[string]string {
return m
}
// clangos lists the operating systems where we prefer clang to gcc.
var clangos = []string{
"darwin", // macOS 10.9 and later require clang
"freebsd", // FreeBSD 10 and later do not ship gcc
"openbsd", // OpenBSD ships with GCC 4.2, which is now quite old.
}
// compilerEnvLookup returns the compiler settings for goos/goarch in map m.
func compilerEnvLookup(m map[string]string, goos, goarch string) string {
// kind is "CC" or "CXX".
func compilerEnvLookup(kind string, m map[string]string, goos, goarch string) string {
if cc := m[goos+"/"+goarch]; cc != "" {
return cc
}
return m[""]
if cc := m[""]; cc != "" {
return cc
}
for _, os := range clangos {
if goos == os {
if kind == "CXX" {
return "clang++"
}
return "clang"
}
}
if kind == "CXX" {
return "g++"
}
return "gcc"
}
// rmworkdir deletes the work directory.
@ -524,15 +541,25 @@ func setup() {
xremove(pathf("%s/bin/%s", goroot, old))
}
// For release, make sure excluded things are excluded.
// Special release-specific setup.
goversion := findgoversion()
if strings.HasPrefix(goversion, "release.") || (strings.HasPrefix(goversion, "go") && !strings.Contains(goversion, "beta")) {
isRelease := strings.HasPrefix(goversion, "release.") || (strings.HasPrefix(goversion, "go") && !strings.Contains(goversion, "beta"))
if isRelease {
// Make sure release-excluded things are excluded.
for _, dir := range unreleased {
if p := pathf("%s/%s", goroot, dir); isdir(p) {
fatalf("%s should not exist in release build", p)
}
}
}
if isRelease || os.Getenv("GO_BUILDER_NAME") != "" {
// Add -trimpath for reproducible builds of releases.
// Include builders so that -trimpath is well-tested ahead of releases.
// Do not include local development, so that people working in the
// main branch for day-to-day work on the Go toolchain itself can
// still have full paths for stack traces for compiler crashes and the like.
// toolenv = append(toolenv, "GOFLAGS=-trimpath")
}
}
/*
@ -675,7 +702,7 @@ func runInstall(pkg string, ch chan struct{}) {
if goldflags != "" {
link = append(link, goldflags)
}
link = append(link, "-extld="+compilerEnvLookup(defaultcc, goos, goarch))
link = append(link, "-extld="+compilerEnvLookup("CC", defaultcc, goos, goarch))
link = append(link, "-L="+pathf("%s/pkg/obj/go-bootstrap/%s_%s", goroot, goos, goarch))
link = append(link, "-o", pathf("%s/%s%s", tooldir, elem, exe))
targ = len(link) - 1
@ -1263,6 +1290,15 @@ func timelog(op, name string) {
fmt.Fprintf(timeLogFile, "%s %+.1fs %s %s\n", t.Format(time.UnixDate), t.Sub(timeLogStart).Seconds(), op, name)
}
// toolenv is the environment to use when building cmd.
// We disable cgo to get static binaries for cmd/go and cmd/pprof,
// so that they work on systems without the same dynamic libraries
// as the original build system.
// In release branches, we add -trimpath for reproducible builds.
// In the main branch we leave it off, so that compiler crashes and
// the like have full path names for easier navigation to source files.
var toolenv = []string{"CGO_ENABLED=0"}
var toolchain = []string{"cmd/asm", "cmd/cgo", "cmd/compile", "cmd/link"}
// The bootstrap command runs a build from scratch,
@ -1388,10 +1424,10 @@ func cmdbootstrap() {
xprintf("\n")
}
xprintf("Building Go toolchain2 using go_bootstrap and Go toolchain1.\n")
os.Setenv("CC", compilerEnvLookup(defaultcc, goos, goarch))
os.Setenv("CC", compilerEnvLookup("CC", defaultcc, goos, goarch))
// Now that cmd/go is in charge of the build process, enable GOEXPERIMENT.
os.Setenv("GOEXPERIMENT", goexperiment)
goInstall(goBootstrap, toolchain...)
goInstall(toolenv, goBootstrap, toolchain...)
if debug {
run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full")
copyfile(pathf("%s/compile2", tooldir), pathf("%s/compile", tooldir), writeExec)
@ -1418,7 +1454,7 @@ func cmdbootstrap() {
xprintf("\n")
}
xprintf("Building Go toolchain3 using go_bootstrap and Go toolchain2.\n")
goInstall(goBootstrap, append([]string{"-a"}, toolchain...)...)
goInstall(toolenv, goBootstrap, append([]string{"-a"}, toolchain...)...)
if debug {
run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full")
copyfile(pathf("%s/compile3", tooldir), pathf("%s/compile", tooldir), writeExec)
@ -1440,9 +1476,12 @@ func cmdbootstrap() {
xprintf("\n")
}
xprintf("Building packages and commands for host, %s/%s.\n", goos, goarch)
goInstall(goBootstrap, "std", "cmd")
checkNotStale(goBootstrap, "std", "cmd")
checkNotStale(cmdGo, "std", "cmd")
goInstall(nil, goBootstrap, "std")
goInstall(toolenv, goBootstrap, "cmd")
checkNotStale(nil, goBootstrap, "std")
checkNotStale(toolenv, goBootstrap, "cmd")
checkNotStale(nil, cmdGo, "std")
checkNotStale(toolenv, cmdGo, "cmd")
timelog("build", "target toolchain")
if vflag > 0 {
@ -1452,17 +1491,19 @@ func cmdbootstrap() {
goarch = oldgoarch
os.Setenv("GOOS", goos)
os.Setenv("GOARCH", goarch)
os.Setenv("CC", compilerEnvLookup(defaultcc, goos, goarch))
os.Setenv("CC", compilerEnvLookup("CC", defaultcc, goos, goarch))
xprintf("Building packages and commands for target, %s/%s.\n", goos, goarch)
}
targets := []string{"std", "cmd"}
goInstall(goBootstrap, targets...)
checkNotStale(goBootstrap, append(toolchain, "runtime/internal/sys")...)
checkNotStale(goBootstrap, targets...)
checkNotStale(cmdGo, targets...)
goInstall(nil, goBootstrap, "std")
goInstall(toolenv, goBootstrap, "cmd")
checkNotStale(toolenv, goBootstrap, append(toolchain, "runtime/internal/sys")...)
checkNotStale(nil, goBootstrap, "std")
checkNotStale(toolenv, goBootstrap, "cmd")
checkNotStale(nil, cmdGo, "std")
checkNotStale(toolenv, cmdGo, "cmd")
if debug {
run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full")
checkNotStale(goBootstrap, append(toolchain, "runtime/internal/sys")...)
checkNotStale(toolenv, goBootstrap, append(toolchain, "runtime/internal/sys")...)
copyfile(pathf("%s/compile4", tooldir), pathf("%s/compile", tooldir), writeExec)
}
@ -1492,8 +1533,8 @@ func cmdbootstrap() {
oldcc := os.Getenv("CC")
os.Setenv("GOOS", gohostos)
os.Setenv("GOARCH", gohostarch)
os.Setenv("CC", compilerEnvLookup(defaultcc, gohostos, gohostarch))
goCmd(cmdGo, "build", "-o", pathf("%s/go_%s_%s_exec%s", gorootBin, goos, goarch, exe), wrapperPath)
os.Setenv("CC", compilerEnvLookup("CC", defaultcc, gohostos, gohostarch))
goCmd(nil, cmdGo, "build", "-o", pathf("%s/go_%s_%s_exec%s", gorootBin, goos, goarch, exe), wrapperPath)
// Restore environment.
// TODO(elias.naur): support environment variables in goCmd?
os.Setenv("GOOS", goos)
@ -1521,8 +1562,8 @@ func wrapperPathFor(goos, goarch string) string {
return ""
}
func goInstall(goBinary string, args ...string) {
goCmd(goBinary, "install", args...)
func goInstall(env []string, goBinary string, args ...string) {
goCmd(env, goBinary, "install", args...)
}
func appendCompilerFlags(args []string) []string {
@ -1535,7 +1576,7 @@ func appendCompilerFlags(args []string) []string {
return args
}
func goCmd(goBinary string, cmd string, args ...string) {
func goCmd(env []string, goBinary string, cmd string, args ...string) {
goCmd := []string{goBinary, cmd}
if noOpt {
goCmd = append(goCmd, "-tags=noopt")
@ -1550,10 +1591,10 @@ func goCmd(goBinary string, cmd string, args ...string) {
goCmd = append(goCmd, "-p=1")
}
run(workdir, ShowOutput|CheckExit, append(goCmd, args...)...)
runEnv(workdir, ShowOutput|CheckExit, env, append(goCmd, args...)...)
}
func checkNotStale(goBinary string, targets ...string) {
func checkNotStale(env []string, goBinary string, targets ...string) {
goCmd := []string{goBinary, "list"}
if noOpt {
goCmd = append(goCmd, "-tags=noopt")
@ -1561,7 +1602,7 @@ func checkNotStale(goBinary string, targets ...string) {
goCmd = appendCompilerFlags(goCmd)
goCmd = append(goCmd, "-f={{if .Stale}}\tSTALE {{.ImportPath}}: {{.StaleReason}}{{end}}")
out := run(workdir, CheckExit, append(goCmd, targets...)...)
out := runEnv(workdir, CheckExit, env, append(goCmd, targets...)...)
if strings.Contains(out, "\tSTALE ") {
os.Setenv("GODEBUG", "gocachehash=1")
for _, target := range []string{"runtime/internal/sys", "cmd/dist", "cmd/link"} {
@ -1664,7 +1705,17 @@ func checkCC() {
if !needCC() {
return
}
cc, err := quotedSplit(defaultcc[""])
cc1 := defaultcc[""]
if cc1 == "" {
cc1 = "gcc"
for _, os := range clangos {
if gohostos == os {
cc1 = "clang"
break
}
}
}
cc, err := quotedSplit(cc1)
if err != nil {
fatalf("split CC: %v", err)
}

View file

@ -66,7 +66,26 @@ func defaultCCFunc(name string, defaultcc map[string]string) string {
fmt.Fprintf(&buf, "\tcase %q:\n\t\treturn %q\n", k, defaultcc[k])
}
fmt.Fprintf(&buf, "\t}\n")
fmt.Fprintf(&buf, "\treturn %q\n", defaultcc[""])
if cc := defaultcc[""]; cc != "" {
fmt.Fprintf(&buf, "\treturn %q\n", cc)
} else {
clang, gcc := "clang", "gcc"
if strings.HasSuffix(name, "CXX") {
clang, gcc = "clang++", "g++"
}
fmt.Fprintf(&buf, "\tswitch goos {\n")
fmt.Fprintf(&buf, "\tcase ")
for i, os := range clangos {
if i > 0 {
fmt.Fprintf(&buf, ", ")
}
fmt.Fprintf(&buf, "%q", os)
}
fmt.Fprintf(&buf, ":\n")
fmt.Fprintf(&buf, "\t\treturn %q\n", clang)
fmt.Fprintf(&buf, "\t}\n")
fmt.Fprintf(&buf, "\treturn %q\n", gcc)
}
fmt.Fprintf(&buf, "}\n")
return buf.String()

View file

@ -59,15 +59,6 @@ func main() {
case "aix":
// uname -m doesn't work under AIX
gohostarch = "ppc64"
case "darwin":
// macOS 10.9 and later require clang
defaultclang = true
case "freebsd":
// Since FreeBSD 10 gcc is no longer part of the base system.
defaultclang = true
case "openbsd":
// OpenBSD ships with GCC 4.2, which is now quite old.
defaultclang = true
case "plan9":
gohostarch = os.Getenv("objtype")
if gohostarch == "" {

27
src/cmd/dist/test.go vendored
View file

@ -149,7 +149,7 @@ func (t *tester) run() {
if t.rebuild {
t.out("Building packages and commands.")
// Force rebuild the whole toolchain.
goInstall("go", append([]string{"-a"}, toolchain...)...)
goInstall(toolenv, "go", append([]string{"-a"}, toolchain...)...)
}
if !t.listMode {
@ -166,9 +166,10 @@ func (t *tester) run() {
// to break if we don't automatically refresh things here.
// Rebuilding is a shortened bootstrap.
// See cmdbootstrap for a description of the overall process.
goInstall("go", toolchain...)
goInstall("go", toolchain...)
goInstall("go", "std", "cmd")
goInstall(toolenv, "go", toolchain...)
goInstall(toolenv, "go", toolchain...)
goInstall(toolenv, "go", "cmd")
goInstall(nil, "go", "std")
} else {
// The Go builder infrastructure should always begin running tests from a
// clean, non-stale state, so there is no need to rebuild the world.
@ -178,15 +179,15 @@ func (t *tester) run() {
// The cache used by dist when building is different from that used when
// running dist test, so rebuild (but don't install) std and cmd to make
// sure packages without install targets are cached so they are not stale.
goCmd("go", "build", "std", "cmd") // make sure dependencies of targets are cached
if builder == "aix-ppc64" {
goCmd(toolenv, "go", "build", "cmd") // make sure dependencies of targets are cached
goCmd(nil, "go", "build", "std")
checkNotStale(nil, "go", "std")
if builder != "aix-ppc64" {
// The aix-ppc64 builder for some reason does not have deterministic cgo
// builds, so "cmd" is stale. Fortunately, most of the tests don't care.
// TODO(#56896): remove this special case once the builder supports
// determistic cgo builds.
checkNotStale("go", "std")
} else {
checkNotStale("go", "std", "cmd")
checkNotStale(toolenv, "go", "cmd")
}
}
}
@ -1315,7 +1316,7 @@ func (t *tester) registerCgoTests() {
// Check for static linking support
var staticCheck rtPreFunc
cmd := t.dirCmd("misc/cgo/test",
compilerEnvLookup(defaultcc, goos, goarch), "-xc", "-o", "/dev/null", "-static", "-")
compilerEnvLookup("CC", defaultcc, goos, goarch), "-xc", "-o", "/dev/null", "-static", "-")
cmd.Stdin = strings.NewReader("int main() {}")
cmd.Stdout, cmd.Stderr = nil, nil // Discard output
if err := cmd.Run(); err != nil {
@ -1365,7 +1366,7 @@ func (t *tester) registerCgoTests() {
// running in parallel with earlier tests, or if it has some other reason
// for needing the earlier tests to be done.
func (t *tester) runPending(nextTest *distTest) {
checkNotStale("go", "std")
checkNotStale(nil, "go", "std")
worklist := t.worklist
t.worklist = nil
for _, w := range worklist {
@ -1423,7 +1424,7 @@ func (t *tester) runPending(nextTest *distTest) {
log.Printf("Failed: %v", w.err)
t.failed = true
}
checkNotStale("go", "std")
checkNotStale(nil, "go", "std")
}
if t.failed && !t.keepGoing {
fatalf("FAILED")
@ -1449,7 +1450,7 @@ func (t *tester) hasBash() bool {
}
func (t *tester) hasCxx() bool {
cxx, _ := exec.LookPath(compilerEnvLookup(defaultcxx, goos, goarch))
cxx, _ := exec.LookPath(compilerEnvLookup("CXX", defaultcxx, goos, goarch))
return cxx != ""
}

12
src/cmd/dist/util.go vendored
View file

@ -58,14 +58,19 @@ const (
var outputLock sync.Mutex
// run runs the command line cmd in dir.
// run is like runEnv with no additional environment.
func run(dir string, mode int, cmd ...string) string {
return runEnv(dir, mode, nil, cmd...)
}
// runEnv runs the command line cmd in dir with additional environment env.
// If mode has ShowOutput set and Background unset, run passes cmd's output to
// stdout/stderr directly. Otherwise, run returns cmd's output as a string.
// If mode has CheckExit set and the command fails, run calls fatalf.
// If mode has Background set, this command is being run as a
// Background job. Only bgrun should use the Background mode,
// not other callers.
func run(dir string, mode int, cmd ...string) string {
func runEnv(dir string, mode int, env []string, cmd ...string) string {
if vflag > 1 {
errprintf("run: %s\n", strings.Join(cmd, " "))
}
@ -75,6 +80,9 @@ func run(dir string, mode int, cmd ...string) string {
bin = gorootBinGo
}
xcmd := exec.Command(bin, cmd[1:]...)
if env != nil {
xcmd.Env = append(os.Environ(), env...)
}
setDir(xcmd, dir)
var data []byte
var err error

View file

@ -0,0 +1,18 @@
# .a files should use slash-separated paths even on windows
# This is important for reproducing native builds with cross-compiled builds.
go build -o x.a text/template
! grep 'GOROOT\\' x.a
! grep 'text\\template' x.a
! grep 'c:\\' x.a
# executables should use slash-separated paths even on windows
# This is important for reproducing native builds with cross-compiled builds.
go build -o hello.exe hello.go
! grep 'GOROOT\\' hello.exe
! grep '\\runtime' hello.exe
! grep 'runtime\\' hello.exe
! grep 'gofile..[A-Za-z]:\\' hello.exe
-- hello.go --
package main
func main() { println("hello") }

View file

@ -8,6 +8,7 @@ import (
"internal/buildcfg"
"os"
"path/filepath"
"runtime"
"strings"
)
@ -43,6 +44,13 @@ func AbsFile(dir, file, rewrites string) string {
abs = "$GOROOT" + abs[len(buildcfg.GOROOT):]
}
// Rewrite paths to match the slash convention of the target.
// This helps ensure that cross-compiled distributions remain
// bit-for-bit identical to natively compiled distributions.
if runtime.GOOS == "windows" {
abs = strings.ReplaceAll(abs, `\`, "/")
}
if abs == "" {
abs = "??"
}