go/src/cmd/internal/archive/archive_test.go

377 lines
7.4 KiB
Go
Raw Normal View History

// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package archive
import (
"bytes"
"debug/elf"
"debug/macho"
"debug/pe"
"fmt"
"internal/testenv"
"internal/xcoff"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"runtime"
"testing"
"unicode/utf8"
)
var (
buildDir string
go1obj string
go2obj string
goarchive string
cgoarchive string
)
func TestMain(m *testing.M) {
if !testenv.HasGoBuild() {
return
}
if err := buildGoobj(); err != nil {
fmt.Println(err)
os.RemoveAll(buildDir)
os.Exit(1)
}
exit := m.Run()
os.RemoveAll(buildDir)
os.Exit(exit)
}
func copyDir(dst, src string) error {
err := os.MkdirAll(dst, 0777)
if err != nil {
return err
}
fis, err := ioutil.ReadDir(src)
if err != nil {
return err
}
for _, fi := range fis {
err = copyFile(filepath.Join(dst, fi.Name()), filepath.Join(src, fi.Name()))
if err != nil {
return err
}
}
return nil
}
func copyFile(dst, src string) (err error) {
var s, d *os.File
s, err = os.Open(src)
if err != nil {
return err
}
defer s.Close()
d, err = os.Create(dst)
if err != nil {
return err
}
defer func() {
e := d.Close()
if err == nil {
err = e
}
}()
_, err = io.Copy(d, s)
if err != nil {
return err
}
return nil
}
func buildGoobj() error {
var err error
buildDir, err = ioutil.TempDir("", "TestGoobj")
if err != nil {
return err
}
go1obj = filepath.Join(buildDir, "go1.o")
go2obj = filepath.Join(buildDir, "go2.o")
goarchive = filepath.Join(buildDir, "go.a")
gotool, err := testenv.GoTool()
if err != nil {
return err
}
go1src := filepath.Join("testdata", "go1.go")
go2src := filepath.Join("testdata", "go2.go")
cmd/compile: require -p flag The -p flag specifies the import path of the package being compiled. This CL makes it required when invoking the compiler and adjusts tests that invoke the compiler directly to conform to this new requirement. The go command already passes the flag, so it is unmodified in this CL. It is expected that any other Go build systems also already pass -p, or else they will need to arrange to do so before updating to Go 1.19. Of particular note, Bazel already does for rules with an importpath= attribute, which includes all Gazelle-generated rules. There is more cleanup possible now in cmd/compile, cmd/link, and other consumers of Go object files, but that is left to future CLs. Additional historical background follows but can be ignored. Long ago, before the go command, or modules, or any kind of versioning, symbols in Go archive files were named using just the package name, so that for example func F in math/rand and func F in crypto/rand would both be the object file symbol 'rand.F'. This led to collisions even in small source trees, which made certain packages unusable in the presence of other packages and generally was a problem for Go's goal of scaling to very large source trees. Fixing this problem required changing from package names to import paths in symbol names, which was mostly straightforward. One wrinkle, though, is that the compiler did not know the import path of the package being compiled; it only knew the package name. At the time, there was no go command, just Makefiles that people had invoking 6g (now “go tool compile”) and then copying the resulting object file to an importable location. That is, everyone had a custom build setup for Go, because there was no standard one. So it was not particularly attractive to change how the compiler was invoked, since that would break approximately every Go user at the time. Instead, we arranged for the compiler to emit, and other tools reading object files to recognize, a special import path (the empty string, it turned out) denoting “the import path of this object file”. This worked well enough at the time and maintained complete command-line compatibility with existing Go usage. The changes implementing this transition can be found by searching the Git history for “package global name space”, which is what they eliminated. In particular, CL 190076 (a6736fa4), CL 186263 (758f2bc5), CL 193080 (1cecac81), CL 194053 (19126320), and CL 194071 (531e6b77) did the bulk of this transformation in January 2010. Later, in September 2011, we added the -p flag to the compiler for diagnostic purposes. The problem was that it was easy to create import cycles, especially in tests, and these could not be diagnosed until link time. You'd really want the compiler to diagnose these, for example if the compilation of package sort noticed it was importing a package that itself imported "sort". But the compilation of package sort didn't know its own import path, and so it could not tell whether it had found itself as a transitive dependency. Adding the -p flag solved this problem, and its use was optional, since the linker would still diagnose the import cycle in builds that had not updated to start passing -p. This was CL 4972057 (1e480cd1). There was still no go command at this point, but when we introduced the go command we made it pass -p, which it has for many years at this point. Over time, parts of the compiler began to depend on the presence of the -p flag for various reasonable purposes. For example: In CL 6497074 (041fc8bf; Oct 2012), the race detector used -p to detect packages that should not have race annotations, such as runtime/race and sync/atomic. In CL 13367052 (7276c02b; Sep 2013), a bug fix used -p to detect the compilation of package reflect. In CL 30539 (8aadcc55; Oct 2016), the compiler started using -p to identify package math, to be able to intrinsify calls to Sqrt inside that package. In CL 61019 (9daee931; Sep 2017), CL 71430 (2c1d2e06; Oct 2017), and later related CLs, the compiler started using the -p value when creating various DWARF debugging information. In CL 174657 (cc5eaf93; May 2019), the compiler started writing symbols without the magic empty string whenever -p was used, to reduce the amount of work required in the linker. In CL 179861 (dde7c770; Jun 2019), the compiler made the second argument to //go:linkname optional when -p is used, because in that case the compiler can derive an appropriate default. There are more examples. Today it is impossible to compile the Go standard library without using -p, and DWARF debug information is incomplete without using -p. All known Go build systems pass -p. In particular, the go command does, which is what nearly all Go developers invoke to build Go code. And Bazel does, for go_library rules that set the importpath attribute, which is all rules generated by Gazelle. Gccgo has an equivalent of -p and has required its use in order to disambiguate packages with the same name but different import paths since 2010. On top of all this, various parts of code generation for generics are made more complicated by needing to cope with the case where -p is not specified, even though it's essentially always specified. In summary, the current state is: - Use of the -p flag with cmd/compile is required for building the standard library, and for complete DWARF information, and to enable certain linker speedups. - The go command and Bazel, which we expect account for just about 100% of Go builds, both invoke cmd/compile with -p. - The code in cmd/compile to support builds without -p is complex and has become more complex with generics, but it is almost always dead code and therefore not worth maintaining. - Gccgo already requires its equivalent of -p in any build where two packages have the same name. All this supports the change in this CL, which makes -p required and adjusts tests that invoke cmd/compile to add -p appropriately. Future CLs will be able to remove all the code dealing with the possibility of -p not having been specified. Change-Id: I6b95b9d4cffe59c7bac82eb273ef6c4a67bb0e43 Reviewed-on: https://go-review.googlesource.com/c/go/+/391014 Trust: Russ Cox <rsc@golang.org> Run-TryBot: Russ Cox <rsc@golang.org> TryBot-Result: Gopher Robot <gobot@golang.org> Reviewed-by: Matthew Dempsky <mdempsky@google.com>
2022-03-08 18:16:35 -05:00
out, err := exec.Command(gotool, "tool", "compile", "-p=p", "-o", go1obj, go1src).CombinedOutput()
if err != nil {
return fmt.Errorf("go tool compile -o %s %s: %v\n%s", go1obj, go1src, err, out)
}
cmd/compile: require -p flag The -p flag specifies the import path of the package being compiled. This CL makes it required when invoking the compiler and adjusts tests that invoke the compiler directly to conform to this new requirement. The go command already passes the flag, so it is unmodified in this CL. It is expected that any other Go build systems also already pass -p, or else they will need to arrange to do so before updating to Go 1.19. Of particular note, Bazel already does for rules with an importpath= attribute, which includes all Gazelle-generated rules. There is more cleanup possible now in cmd/compile, cmd/link, and other consumers of Go object files, but that is left to future CLs. Additional historical background follows but can be ignored. Long ago, before the go command, or modules, or any kind of versioning, symbols in Go archive files were named using just the package name, so that for example func F in math/rand and func F in crypto/rand would both be the object file symbol 'rand.F'. This led to collisions even in small source trees, which made certain packages unusable in the presence of other packages and generally was a problem for Go's goal of scaling to very large source trees. Fixing this problem required changing from package names to import paths in symbol names, which was mostly straightforward. One wrinkle, though, is that the compiler did not know the import path of the package being compiled; it only knew the package name. At the time, there was no go command, just Makefiles that people had invoking 6g (now “go tool compile”) and then copying the resulting object file to an importable location. That is, everyone had a custom build setup for Go, because there was no standard one. So it was not particularly attractive to change how the compiler was invoked, since that would break approximately every Go user at the time. Instead, we arranged for the compiler to emit, and other tools reading object files to recognize, a special import path (the empty string, it turned out) denoting “the import path of this object file”. This worked well enough at the time and maintained complete command-line compatibility with existing Go usage. The changes implementing this transition can be found by searching the Git history for “package global name space”, which is what they eliminated. In particular, CL 190076 (a6736fa4), CL 186263 (758f2bc5), CL 193080 (1cecac81), CL 194053 (19126320), and CL 194071 (531e6b77) did the bulk of this transformation in January 2010. Later, in September 2011, we added the -p flag to the compiler for diagnostic purposes. The problem was that it was easy to create import cycles, especially in tests, and these could not be diagnosed until link time. You'd really want the compiler to diagnose these, for example if the compilation of package sort noticed it was importing a package that itself imported "sort". But the compilation of package sort didn't know its own import path, and so it could not tell whether it had found itself as a transitive dependency. Adding the -p flag solved this problem, and its use was optional, since the linker would still diagnose the import cycle in builds that had not updated to start passing -p. This was CL 4972057 (1e480cd1). There was still no go command at this point, but when we introduced the go command we made it pass -p, which it has for many years at this point. Over time, parts of the compiler began to depend on the presence of the -p flag for various reasonable purposes. For example: In CL 6497074 (041fc8bf; Oct 2012), the race detector used -p to detect packages that should not have race annotations, such as runtime/race and sync/atomic. In CL 13367052 (7276c02b; Sep 2013), a bug fix used -p to detect the compilation of package reflect. In CL 30539 (8aadcc55; Oct 2016), the compiler started using -p to identify package math, to be able to intrinsify calls to Sqrt inside that package. In CL 61019 (9daee931; Sep 2017), CL 71430 (2c1d2e06; Oct 2017), and later related CLs, the compiler started using the -p value when creating various DWARF debugging information. In CL 174657 (cc5eaf93; May 2019), the compiler started writing symbols without the magic empty string whenever -p was used, to reduce the amount of work required in the linker. In CL 179861 (dde7c770; Jun 2019), the compiler made the second argument to //go:linkname optional when -p is used, because in that case the compiler can derive an appropriate default. There are more examples. Today it is impossible to compile the Go standard library without using -p, and DWARF debug information is incomplete without using -p. All known Go build systems pass -p. In particular, the go command does, which is what nearly all Go developers invoke to build Go code. And Bazel does, for go_library rules that set the importpath attribute, which is all rules generated by Gazelle. Gccgo has an equivalent of -p and has required its use in order to disambiguate packages with the same name but different import paths since 2010. On top of all this, various parts of code generation for generics are made more complicated by needing to cope with the case where -p is not specified, even though it's essentially always specified. In summary, the current state is: - Use of the -p flag with cmd/compile is required for building the standard library, and for complete DWARF information, and to enable certain linker speedups. - The go command and Bazel, which we expect account for just about 100% of Go builds, both invoke cmd/compile with -p. - The code in cmd/compile to support builds without -p is complex and has become more complex with generics, but it is almost always dead code and therefore not worth maintaining. - Gccgo already requires its equivalent of -p in any build where two packages have the same name. All this supports the change in this CL, which makes -p required and adjusts tests that invoke cmd/compile to add -p appropriately. Future CLs will be able to remove all the code dealing with the possibility of -p not having been specified. Change-Id: I6b95b9d4cffe59c7bac82eb273ef6c4a67bb0e43 Reviewed-on: https://go-review.googlesource.com/c/go/+/391014 Trust: Russ Cox <rsc@golang.org> Run-TryBot: Russ Cox <rsc@golang.org> TryBot-Result: Gopher Robot <gobot@golang.org> Reviewed-by: Matthew Dempsky <mdempsky@google.com>
2022-03-08 18:16:35 -05:00
out, err = exec.Command(gotool, "tool", "compile", "-p=p", "-o", go2obj, go2src).CombinedOutput()
if err != nil {
return fmt.Errorf("go tool compile -o %s %s: %v\n%s", go2obj, go2src, err, out)
}
out, err = exec.Command(gotool, "tool", "pack", "c", goarchive, go1obj, go2obj).CombinedOutput()
if err != nil {
return fmt.Errorf("go tool pack c %s %s %s: %v\n%s", goarchive, go1obj, go2obj, err, out)
}
if testenv.HasCGO() {
gopath := filepath.Join(buildDir, "gopath")
err = copyDir(filepath.Join(gopath, "src", "mycgo"), filepath.Join("testdata", "mycgo"))
if err == nil {
err = ioutil.WriteFile(filepath.Join(gopath, "src", "mycgo", "go.mod"), []byte("module mycgo\n"), 0666)
}
if err != nil {
return err
}
cmd := exec.Command(gotool, "install", "-gcflags=all="+os.Getenv("GO_GCFLAGS"), "mycgo")
cmd.Dir = filepath.Join(gopath, "src", "mycgo")
cmd.Env = append(os.Environ(), "GOPATH="+gopath)
out, err = cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("go install mycgo: %v\n%s", err, out)
}
pat := filepath.Join(gopath, "pkg", "*", "mycgo.a")
ms, err := filepath.Glob(pat)
if err != nil {
return err
}
if len(ms) == 0 {
return fmt.Errorf("cannot found paths for pattern %s", pat)
}
cgoarchive = ms[0]
}
return nil
}
func TestParseGoobj(t *testing.T) {
path := go1obj
f, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
defer f.Close()
a, err := Parse(f, false)
if err != nil {
t.Fatal(err)
}
if len(a.Entries) != 2 {
t.Errorf("expect 2 entry, found %d", len(a.Entries))
}
for _, e := range a.Entries {
if e.Type == EntryPkgDef {
continue
}
if e.Type != EntryGoObj {
t.Errorf("wrong type of object: want EntryGoObj, got %v", e.Type)
}
if !bytes.Contains(e.Obj.TextHeader, []byte(runtime.GOARCH)) {
t.Errorf("text header does not contain GOARCH %s: %q", runtime.GOARCH, e.Obj.TextHeader)
}
}
}
func TestParseArchive(t *testing.T) {
path := goarchive
f, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
defer f.Close()
a, err := Parse(f, false)
if err != nil {
t.Fatal(err)
}
if len(a.Entries) != 3 {
t.Errorf("expect 3 entry, found %d", len(a.Entries))
}
var found1 bool
var found2 bool
for _, e := range a.Entries {
if e.Type == EntryPkgDef {
continue
}
if e.Type != EntryGoObj {
t.Errorf("wrong type of object: want EntryGoObj, got %v", e.Type)
}
if !bytes.Contains(e.Obj.TextHeader, []byte(runtime.GOARCH)) {
t.Errorf("text header does not contain GOARCH %s: %q", runtime.GOARCH, e.Obj.TextHeader)
}
if e.Name == "go1.o" {
found1 = true
}
if e.Name == "go2.o" {
found2 = true
}
}
if !found1 {
t.Errorf(`object "go1.o" not found`)
}
if !found2 {
t.Errorf(`object "go2.o" not found`)
}
}
func TestParseCGOArchive(t *testing.T) {
testenv.MustHaveCGO(t)
path := cgoarchive
f, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
defer f.Close()
a, err := Parse(f, false)
if err != nil {
t.Fatal(err)
}
c1 := "c1"
c2 := "c2"
switch runtime.GOOS {
case "darwin", "ios":
c1 = "_" + c1
c2 = "_" + c2
case "windows":
if runtime.GOARCH == "386" {
c1 = "_" + c1
c2 = "_" + c2
}
case "aix":
c1 = "." + c1
c2 = "." + c2
}
var foundgo, found1, found2 bool
for _, e := range a.Entries {
switch e.Type {
default:
t.Errorf("unknown object type")
case EntryPkgDef:
continue
case EntryGoObj:
foundgo = true
if !bytes.Contains(e.Obj.TextHeader, []byte(runtime.GOARCH)) {
t.Errorf("text header does not contain GOARCH %s: %q", runtime.GOARCH, e.Obj.TextHeader)
}
continue
case EntryNativeObj:
}
obj := io.NewSectionReader(f, e.Offset, e.Size)
switch runtime.GOOS {
case "darwin", "ios":
mf, err := macho.NewFile(obj)
if err != nil {
t.Fatal(err)
}
cmd/go: stop linking cgo objects together with ld -r https://golang.org/cl/5822049 introduced the idea of linking together all the cgo objects with -r, while also linking against -lgcc. This was to fix http://golang.org/issue/3261: cgo code that requires libgcc would break when using internal linking. This approach introduced https://golang.org/issue/9510: multiple different cgo packages could include the same libgcc object, leading to a multiple definition error during the final link. That problem was fixed by https://golang.org/cl/16741, as modified by https://golang.org/cl/16993, which did the link against libgcc only during the final link. After https://golang.org/cl/16741, and, on Windows, the later https://golang.org/cl/26670, ld -r no longer does anything useful. So, remove it. Doing this revealed that running ld -r on Darwin simplifies some relocs by making them specific to a symbol rather than a section. Correct the handling of unsigned relocations in internal linking mode by offsetting by the symbol value. This only really comes up when using the internal linker with C code that initializes a variable to the address of a local constant, such as a C string (as in const char *s = "str";). This change does not affect the normal case of external linking, where the Add field is ignored. The test case is misc/cgo/test/issue6612.go in internal linking mode. The cmd/internal/goobj test can now see an external object with no symbol table; fix it to not crash in that case. Change-Id: I15e5b7b5a8f48136bc14bf4e1c4c473d5eb58062 Reviewed-on: https://go-review.googlesource.com/64793 Run-TryBot: Ian Lance Taylor <iant@golang.org> TryBot-Result: Gobot Gobot <gobot@golang.org> Reviewed-by: Matthew Dempsky <mdempsky@google.com>
2017-09-19 13:53:20 -07:00
if mf.Symtab == nil {
continue
}
for _, s := range mf.Symtab.Syms {
switch s.Name {
case c1:
found1 = true
case c2:
found2 = true
}
}
case "windows":
pf, err := pe.NewFile(obj)
if err != nil {
t.Fatal(err)
}
for _, s := range pf.Symbols {
switch s.Name {
case c1:
found1 = true
case c2:
found2 = true
}
}
case "aix":
xf, err := xcoff.NewFile(obj)
if err != nil {
t.Fatal(err)
}
for _, s := range xf.Symbols {
switch s.Name {
case c1:
found1 = true
case c2:
found2 = true
}
}
default: // ELF
ef, err := elf.NewFile(obj)
if err != nil {
t.Fatal(err)
}
syms, err := ef.Symbols()
if err != nil {
t.Fatal(err)
}
for _, s := range syms {
switch s.Name {
case c1:
found1 = true
case c2:
found2 = true
}
}
}
}
if !foundgo {
t.Errorf(`go object not found`)
}
if !found1 {
t.Errorf(`symbol %q not found`, c1)
}
if !found2 {
t.Errorf(`symbol %q not found`, c2)
}
}
func TestExactly16Bytes(t *testing.T) {
var tests = []string{
"",
"a",
"日本語",
"1234567890123456",
"12345678901234567890",
"1234567890123本語4567890",
"12345678901234日本語567890",
"123456789012345日本語67890",
"1234567890123456日本語7890",
"1234567890123456日本語7日本語890",
}
for _, str := range tests {
got := exactly16Bytes(str)
if len(got) != 16 {
t.Errorf("exactly16Bytes(%q) is %q, length %d", str, got, len(got))
}
// Make sure it is full runes.
for _, c := range got {
if c == utf8.RuneError {
t.Errorf("exactly16Bytes(%q) is %q, has partial rune", str, got)
}
}
}
}