mirror of
https://github.com/golang/go.git
synced 2025-12-08 06:10:04 +00:00
This CL introduces a new indexed data format for package export data. This improves on the previous (sequential) binary format by allowing the compiler to selectively (and lazily) load only the data that's actually needed for compilation. In large Go projects, the package export data can become very large due to transitive type declaration dependencies and inline function/method bodies. By lazily loading these declarations and bodies as needed, we avoid wasting time and memory processing unnecessary and/or redundant data. In the benchmarks below, "old" is -iexport=false and "new" is -iexport=true. The suffixes indicate the compiler concurrency (-c) and inlining (-l) settings used for the build (using -gcflags=all=-foo). Benchmarks were run on an HP Z620. Juju is "go build -a github.com/juju/juju/cmd/...": name old real-time/op new real-time/op delta Juju/c=1/l=0 44.0s ± 1% 38.7s ± 9% -11.97% (p=0.001 n=7+7) Juju/c=1/l=4 53.7s ± 3% 45.3s ± 4% -15.53% (p=0.001 n=7+7) Juju/c=4/l=0 39.7s ± 8% 32.0s ± 4% -19.38% (p=0.001 n=7+7) Juju/c=4/l=4 46.3s ± 4% 38.0s ± 4% -18.06% (p=0.001 n=7+7) name old user-time/op new user-time/op delta Juju/c=1/l=0 371s ± 1% 300s ± 0% -19.07% (p=0.001 n=7+6) Juju/c=1/l=4 482s ± 0% 374s ± 1% -22.37% (p=0.001 n=7+7) Juju/c=4/l=0 410s ± 1% 340s ± 1% -17.19% (p=0.001 n=7+7) Juju/c=4/l=4 532s ± 1% 424s ± 1% -20.26% (p=0.001 n=7+7) name old sys-time/op new sys-time/op delta Juju/c=1/l=0 33.4s ± 1% 28.4s ± 2% -15.02% (p=0.001 n=7+7) Juju/c=1/l=4 40.7s ± 2% 32.8s ± 3% -19.51% (p=0.001 n=7+7) Juju/c=4/l=0 39.8s ± 2% 34.4s ± 2% -13.74% (p=0.001 n=7+7) Juju/c=4/l=4 48.4s ± 2% 40.4s ± 2% -16.50% (p=0.001 n=7+7) Kubelet is "go build -a k8s.io/kubernetes/cmd/kubelet": name old real-time/op new real-time/op delta Kubelet/c=1/l=0 42.0s ± 1% 34.8s ± 1% -17.27% (p=0.008 n=5+5) Kubelet/c=1/l=4 55.4s ± 3% 45.4s ± 3% -18.06% (p=0.002 n=6+6) Kubelet/c=4/l=0 37.4s ± 3% 29.9s ± 1% -20.25% (p=0.004 n=6+5) Kubelet/c=4/l=4 48.1s ± 2% 39.0s ± 5% -18.93% (p=0.002 n=6+6) name old user-time/op new user-time/op delta Kubelet/c=1/l=0 291s ± 1% 233s ± 1% -19.96% (p=0.002 n=6+6) Kubelet/c=1/l=4 385s ± 1% 298s ± 1% -22.51% (p=0.002 n=6+6) Kubelet/c=4/l=0 325s ± 0% 268s ± 1% -17.48% (p=0.004 n=5+6) Kubelet/c=4/l=4 429s ± 1% 343s ± 1% -20.08% (p=0.002 n=6+6) name old sys-time/op new sys-time/op delta Kubelet/c=1/l=0 25.1s ± 2% 20.9s ± 4% -16.69% (p=0.002 n=6+6) Kubelet/c=1/l=4 31.2s ± 3% 24.4s ± 0% -21.67% (p=0.010 n=6+4) Kubelet/c=4/l=0 30.2s ± 2% 25.6s ± 1% -15.34% (p=0.002 n=6+6) Kubelet/c=4/l=4 37.3s ± 1% 30.9s ± 2% -17.11% (p=0.002 n=6+6) Change-Id: Ie43eb3bbe1392cbb61c86792a17a57b33b9561f0 Reviewed-on: https://go-review.googlesource.com/106796 Run-TryBot: Matthew Dempsky <mdempsky@google.com> TryBot-Result: Gobot Gobot <gobot@golang.org> Reviewed-by: Robert Griesemer <gri@golang.org>
239 lines
5.6 KiB
Go
239 lines
5.6 KiB
Go
// Copyright 2009 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package gc
|
|
|
|
import (
|
|
"cmd/compile/internal/types"
|
|
"cmd/internal/bio"
|
|
"cmd/internal/src"
|
|
"fmt"
|
|
)
|
|
|
|
var (
|
|
flagiexport bool // if set, use indexed export data format
|
|
|
|
Debug_export int // if set, print debugging information about export data
|
|
)
|
|
|
|
func exportf(bout *bio.Writer, format string, args ...interface{}) {
|
|
fmt.Fprintf(bout, format, args...)
|
|
if Debug_export != 0 {
|
|
fmt.Printf(format, args...)
|
|
}
|
|
}
|
|
|
|
var asmlist []*Node
|
|
|
|
// exportsym marks n for export (or reexport).
|
|
func exportsym(n *Node) {
|
|
if n.Sym.OnExportList() {
|
|
return
|
|
}
|
|
n.Sym.SetOnExportList(true)
|
|
|
|
if Debug['E'] != 0 {
|
|
fmt.Printf("export symbol %v\n", n.Sym)
|
|
}
|
|
|
|
exportlist = append(exportlist, n)
|
|
}
|
|
|
|
func initname(s string) bool {
|
|
return s == "init"
|
|
}
|
|
|
|
func autoexport(n *Node, ctxt Class) {
|
|
if n.Sym.Pkg != localpkg {
|
|
return
|
|
}
|
|
if (ctxt != PEXTERN && ctxt != PFUNC) || dclcontext != PEXTERN {
|
|
return
|
|
}
|
|
if n.Type != nil && n.Type.IsKind(TFUNC) && n.IsMethod() {
|
|
return
|
|
}
|
|
|
|
if types.IsExported(n.Sym.Name) || initname(n.Sym.Name) {
|
|
exportsym(n)
|
|
}
|
|
if asmhdr != "" && !n.Sym.Asm() {
|
|
n.Sym.SetAsm(true)
|
|
asmlist = append(asmlist, n)
|
|
}
|
|
}
|
|
|
|
// methodbyname sorts types by symbol name.
|
|
type methodbyname []*types.Field
|
|
|
|
func (x methodbyname) Len() int { return len(x) }
|
|
func (x methodbyname) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
|
|
func (x methodbyname) Less(i, j int) bool { return x[i].Sym.Name < x[j].Sym.Name }
|
|
|
|
func dumpexport(bout *bio.Writer) {
|
|
// The linker also looks for the $$ marker - use char after $$ to distinguish format.
|
|
exportf(bout, "\n$$B\n") // indicate binary export format
|
|
off := bout.Offset()
|
|
if flagiexport {
|
|
iexport(bout.Writer)
|
|
} else {
|
|
export(bout.Writer, Debug_export != 0)
|
|
}
|
|
size := bout.Offset() - off
|
|
exportf(bout, "\n$$\n")
|
|
|
|
if Debug_export != 0 {
|
|
fmt.Printf("export data size = %d bytes\n", size)
|
|
}
|
|
}
|
|
|
|
func importsym(ipkg *types.Pkg, pos src.XPos, s *types.Sym, op Op) *Node {
|
|
n := asNode(s.Def)
|
|
if n == nil {
|
|
// iimport should have created a stub ONONAME
|
|
// declaration for all imported symbols. The exception
|
|
// is declarations for Runtimepkg, which are populated
|
|
// by loadsys instead.
|
|
if flagiexport && s.Pkg != Runtimepkg {
|
|
Fatalf("missing ONONAME for %v\n", s)
|
|
}
|
|
|
|
n = dclname(s)
|
|
s.Def = asTypesNode(n)
|
|
s.Importdef = ipkg
|
|
}
|
|
if n.Op != ONONAME && n.Op != op {
|
|
redeclare(lineno, s, fmt.Sprintf("during import %q", ipkg.Path))
|
|
}
|
|
return n
|
|
}
|
|
|
|
// pkgtype returns the named type declared by symbol s.
|
|
// If no such type has been declared yet, a forward declaration is returned.
|
|
// ipkg is the package being imported
|
|
func importtype(ipkg *types.Pkg, pos src.XPos, s *types.Sym) *types.Type {
|
|
n := importsym(ipkg, pos, s, OTYPE)
|
|
if n.Op != OTYPE {
|
|
t := types.New(TFORW)
|
|
t.Sym = s
|
|
t.Nod = asTypesNode(n)
|
|
|
|
n.Op = OTYPE
|
|
n.Pos = pos
|
|
n.Type = t
|
|
n.SetClass(PEXTERN)
|
|
}
|
|
|
|
t := n.Type
|
|
if t == nil {
|
|
Fatalf("importtype %v", s)
|
|
}
|
|
return t
|
|
}
|
|
|
|
// importobj declares symbol s as an imported object representable by op.
|
|
// ipkg is the package being imported
|
|
func importobj(ipkg *types.Pkg, pos src.XPos, s *types.Sym, op Op, ctxt Class, t *types.Type) *Node {
|
|
n := importsym(ipkg, pos, s, op)
|
|
if n.Op != ONONAME {
|
|
if n.Op == op && (n.Class() != ctxt || !eqtype(n.Type, t)) {
|
|
redeclare(lineno, s, fmt.Sprintf("during import %q", ipkg.Path))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
n.Op = op
|
|
n.Pos = pos
|
|
n.SetClass(ctxt)
|
|
n.Type = t
|
|
return n
|
|
}
|
|
|
|
// importconst declares symbol s as an imported constant with type t and value val.
|
|
// ipkg is the package being imported
|
|
func importconst(ipkg *types.Pkg, pos src.XPos, s *types.Sym, t *types.Type, val Val) {
|
|
n := importobj(ipkg, pos, s, OLITERAL, PEXTERN, t)
|
|
if n == nil { // TODO: Check that value matches.
|
|
return
|
|
}
|
|
|
|
n.SetVal(val)
|
|
|
|
if Debug['E'] != 0 {
|
|
fmt.Printf("import const %v %L = %v\n", s, t, val)
|
|
}
|
|
}
|
|
|
|
// importfunc declares symbol s as an imported function with type t.
|
|
// ipkg is the package being imported
|
|
func importfunc(ipkg *types.Pkg, pos src.XPos, s *types.Sym, t *types.Type) {
|
|
n := importobj(ipkg, pos, s, ONAME, PFUNC, t)
|
|
if n == nil {
|
|
return
|
|
}
|
|
|
|
n.Func = new(Func)
|
|
t.SetNname(asTypesNode(n))
|
|
|
|
if Debug['E'] != 0 {
|
|
fmt.Printf("import func %v%S\n", s, t)
|
|
}
|
|
}
|
|
|
|
// importvar declares symbol s as an imported variable with type t.
|
|
// ipkg is the package being imported
|
|
func importvar(ipkg *types.Pkg, pos src.XPos, s *types.Sym, t *types.Type) {
|
|
n := importobj(ipkg, pos, s, ONAME, PEXTERN, t)
|
|
if n == nil {
|
|
return
|
|
}
|
|
|
|
if Debug['E'] != 0 {
|
|
fmt.Printf("import var %v %L\n", s, t)
|
|
}
|
|
}
|
|
|
|
// importalias declares symbol s as an imported type alias with type t.
|
|
// ipkg is the package being imported
|
|
func importalias(ipkg *types.Pkg, pos src.XPos, s *types.Sym, t *types.Type) {
|
|
n := importobj(ipkg, pos, s, OTYPE, PEXTERN, t)
|
|
if n == nil {
|
|
return
|
|
}
|
|
|
|
if Debug['E'] != 0 {
|
|
fmt.Printf("import type %v = %L\n", s, t)
|
|
}
|
|
}
|
|
|
|
func dumpasmhdr() {
|
|
b, err := bio.Create(asmhdr)
|
|
if err != nil {
|
|
Fatalf("%v", err)
|
|
}
|
|
fmt.Fprintf(b, "// generated by compile -asmhdr from package %s\n\n", localpkg.Name)
|
|
for _, n := range asmlist {
|
|
if n.Sym.IsBlank() {
|
|
continue
|
|
}
|
|
switch n.Op {
|
|
case OLITERAL:
|
|
fmt.Fprintf(b, "#define const_%s %#v\n", n.Sym.Name, n.Val())
|
|
|
|
case OTYPE:
|
|
t := n.Type
|
|
if !t.IsStruct() || t.StructType().Map != nil || t.IsFuncArgStruct() {
|
|
break
|
|
}
|
|
fmt.Fprintf(b, "#define %s__size %d\n", n.Sym.Name, int(t.Width))
|
|
for _, f := range t.Fields().Slice() {
|
|
if !f.Sym.IsBlank() {
|
|
fmt.Fprintf(b, "#define %s_%s %d\n", n.Sym.Name, f.Sym.Name, int(f.Offset))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
b.Close()
|
|
}
|