go/src/cmd/compile/internal/gc/export.go
Matthew Dempsky ca2f85fd3f cmd/compile: add indexed export format
This CL introduces a new indexed data format for package export
data. This improves on the previous (sequential) binary format by
allowing the compiler to selectively (and lazily) load only the data
that's actually needed for compilation.

In large Go projects, the package export data can become very large
due to transitive type declaration dependencies and inline
function/method bodies. By lazily loading these declarations and
bodies as needed, we avoid wasting time and memory processing
unnecessary and/or redundant data.

In the benchmarks below, "old" is -iexport=false and "new" is
-iexport=true. The suffixes indicate the compiler concurrency (-c) and
inlining (-l) settings used for the build (using -gcflags=all=-foo).
Benchmarks were run on an HP Z620.

Juju is "go build -a github.com/juju/juju/cmd/...":

name          old real-time/op  new real-time/op  delta
Juju/c=1/l=0        44.0s ± 1%        38.7s ± 9%  -11.97%  (p=0.001 n=7+7)
Juju/c=1/l=4        53.7s ± 3%        45.3s ± 4%  -15.53%  (p=0.001 n=7+7)
Juju/c=4/l=0        39.7s ± 8%        32.0s ± 4%  -19.38%  (p=0.001 n=7+7)
Juju/c=4/l=4        46.3s ± 4%        38.0s ± 4%  -18.06%  (p=0.001 n=7+7)

name          old user-time/op  new user-time/op  delta
Juju/c=1/l=0         371s ± 1%         300s ± 0%  -19.07%  (p=0.001 n=7+6)
Juju/c=1/l=4         482s ± 0%         374s ± 1%  -22.37%  (p=0.001 n=7+7)
Juju/c=4/l=0         410s ± 1%         340s ± 1%  -17.19%  (p=0.001 n=7+7)
Juju/c=4/l=4         532s ± 1%         424s ± 1%  -20.26%  (p=0.001 n=7+7)

name          old sys-time/op   new sys-time/op   delta
Juju/c=1/l=0        33.4s ± 1%        28.4s ± 2%  -15.02%  (p=0.001 n=7+7)
Juju/c=1/l=4        40.7s ± 2%        32.8s ± 3%  -19.51%  (p=0.001 n=7+7)
Juju/c=4/l=0        39.8s ± 2%        34.4s ± 2%  -13.74%  (p=0.001 n=7+7)
Juju/c=4/l=4        48.4s ± 2%        40.4s ± 2%  -16.50%  (p=0.001 n=7+7)

Kubelet is "go build -a k8s.io/kubernetes/cmd/kubelet":

name             old real-time/op  new real-time/op  delta
Kubelet/c=1/l=0        42.0s ± 1%        34.8s ± 1%  -17.27%  (p=0.008 n=5+5)
Kubelet/c=1/l=4        55.4s ± 3%        45.4s ± 3%  -18.06%  (p=0.002 n=6+6)
Kubelet/c=4/l=0        37.4s ± 3%        29.9s ± 1%  -20.25%  (p=0.004 n=6+5)
Kubelet/c=4/l=4        48.1s ± 2%        39.0s ± 5%  -18.93%  (p=0.002 n=6+6)

name             old user-time/op  new user-time/op  delta
Kubelet/c=1/l=0         291s ± 1%         233s ± 1%  -19.96%  (p=0.002 n=6+6)
Kubelet/c=1/l=4         385s ± 1%         298s ± 1%  -22.51%  (p=0.002 n=6+6)
Kubelet/c=4/l=0         325s ± 0%         268s ± 1%  -17.48%  (p=0.004 n=5+6)
Kubelet/c=4/l=4         429s ± 1%         343s ± 1%  -20.08%  (p=0.002 n=6+6)

name             old sys-time/op   new sys-time/op   delta
Kubelet/c=1/l=0        25.1s ± 2%        20.9s ± 4%  -16.69%  (p=0.002 n=6+6)
Kubelet/c=1/l=4        31.2s ± 3%        24.4s ± 0%  -21.67%  (p=0.010 n=6+4)
Kubelet/c=4/l=0        30.2s ± 2%        25.6s ± 1%  -15.34%  (p=0.002 n=6+6)
Kubelet/c=4/l=4        37.3s ± 1%        30.9s ± 2%  -17.11%  (p=0.002 n=6+6)

Change-Id: Ie43eb3bbe1392cbb61c86792a17a57b33b9561f0
Reviewed-on: https://go-review.googlesource.com/106796
Run-TryBot: Matthew Dempsky <mdempsky@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>
2018-04-24 01:05:27 +00:00

239 lines
5.6 KiB
Go

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gc
import (
"cmd/compile/internal/types"
"cmd/internal/bio"
"cmd/internal/src"
"fmt"
)
var (
flagiexport bool // if set, use indexed export data format
Debug_export int // if set, print debugging information about export data
)
func exportf(bout *bio.Writer, format string, args ...interface{}) {
fmt.Fprintf(bout, format, args...)
if Debug_export != 0 {
fmt.Printf(format, args...)
}
}
var asmlist []*Node
// exportsym marks n for export (or reexport).
func exportsym(n *Node) {
if n.Sym.OnExportList() {
return
}
n.Sym.SetOnExportList(true)
if Debug['E'] != 0 {
fmt.Printf("export symbol %v\n", n.Sym)
}
exportlist = append(exportlist, n)
}
func initname(s string) bool {
return s == "init"
}
func autoexport(n *Node, ctxt Class) {
if n.Sym.Pkg != localpkg {
return
}
if (ctxt != PEXTERN && ctxt != PFUNC) || dclcontext != PEXTERN {
return
}
if n.Type != nil && n.Type.IsKind(TFUNC) && n.IsMethod() {
return
}
if types.IsExported(n.Sym.Name) || initname(n.Sym.Name) {
exportsym(n)
}
if asmhdr != "" && !n.Sym.Asm() {
n.Sym.SetAsm(true)
asmlist = append(asmlist, n)
}
}
// methodbyname sorts types by symbol name.
type methodbyname []*types.Field
func (x methodbyname) Len() int { return len(x) }
func (x methodbyname) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
func (x methodbyname) Less(i, j int) bool { return x[i].Sym.Name < x[j].Sym.Name }
func dumpexport(bout *bio.Writer) {
// The linker also looks for the $$ marker - use char after $$ to distinguish format.
exportf(bout, "\n$$B\n") // indicate binary export format
off := bout.Offset()
if flagiexport {
iexport(bout.Writer)
} else {
export(bout.Writer, Debug_export != 0)
}
size := bout.Offset() - off
exportf(bout, "\n$$\n")
if Debug_export != 0 {
fmt.Printf("export data size = %d bytes\n", size)
}
}
func importsym(ipkg *types.Pkg, pos src.XPos, s *types.Sym, op Op) *Node {
n := asNode(s.Def)
if n == nil {
// iimport should have created a stub ONONAME
// declaration for all imported symbols. The exception
// is declarations for Runtimepkg, which are populated
// by loadsys instead.
if flagiexport && s.Pkg != Runtimepkg {
Fatalf("missing ONONAME for %v\n", s)
}
n = dclname(s)
s.Def = asTypesNode(n)
s.Importdef = ipkg
}
if n.Op != ONONAME && n.Op != op {
redeclare(lineno, s, fmt.Sprintf("during import %q", ipkg.Path))
}
return n
}
// pkgtype returns the named type declared by symbol s.
// If no such type has been declared yet, a forward declaration is returned.
// ipkg is the package being imported
func importtype(ipkg *types.Pkg, pos src.XPos, s *types.Sym) *types.Type {
n := importsym(ipkg, pos, s, OTYPE)
if n.Op != OTYPE {
t := types.New(TFORW)
t.Sym = s
t.Nod = asTypesNode(n)
n.Op = OTYPE
n.Pos = pos
n.Type = t
n.SetClass(PEXTERN)
}
t := n.Type
if t == nil {
Fatalf("importtype %v", s)
}
return t
}
// importobj declares symbol s as an imported object representable by op.
// ipkg is the package being imported
func importobj(ipkg *types.Pkg, pos src.XPos, s *types.Sym, op Op, ctxt Class, t *types.Type) *Node {
n := importsym(ipkg, pos, s, op)
if n.Op != ONONAME {
if n.Op == op && (n.Class() != ctxt || !eqtype(n.Type, t)) {
redeclare(lineno, s, fmt.Sprintf("during import %q", ipkg.Path))
}
return nil
}
n.Op = op
n.Pos = pos
n.SetClass(ctxt)
n.Type = t
return n
}
// importconst declares symbol s as an imported constant with type t and value val.
// ipkg is the package being imported
func importconst(ipkg *types.Pkg, pos src.XPos, s *types.Sym, t *types.Type, val Val) {
n := importobj(ipkg, pos, s, OLITERAL, PEXTERN, t)
if n == nil { // TODO: Check that value matches.
return
}
n.SetVal(val)
if Debug['E'] != 0 {
fmt.Printf("import const %v %L = %v\n", s, t, val)
}
}
// importfunc declares symbol s as an imported function with type t.
// ipkg is the package being imported
func importfunc(ipkg *types.Pkg, pos src.XPos, s *types.Sym, t *types.Type) {
n := importobj(ipkg, pos, s, ONAME, PFUNC, t)
if n == nil {
return
}
n.Func = new(Func)
t.SetNname(asTypesNode(n))
if Debug['E'] != 0 {
fmt.Printf("import func %v%S\n", s, t)
}
}
// importvar declares symbol s as an imported variable with type t.
// ipkg is the package being imported
func importvar(ipkg *types.Pkg, pos src.XPos, s *types.Sym, t *types.Type) {
n := importobj(ipkg, pos, s, ONAME, PEXTERN, t)
if n == nil {
return
}
if Debug['E'] != 0 {
fmt.Printf("import var %v %L\n", s, t)
}
}
// importalias declares symbol s as an imported type alias with type t.
// ipkg is the package being imported
func importalias(ipkg *types.Pkg, pos src.XPos, s *types.Sym, t *types.Type) {
n := importobj(ipkg, pos, s, OTYPE, PEXTERN, t)
if n == nil {
return
}
if Debug['E'] != 0 {
fmt.Printf("import type %v = %L\n", s, t)
}
}
func dumpasmhdr() {
b, err := bio.Create(asmhdr)
if err != nil {
Fatalf("%v", err)
}
fmt.Fprintf(b, "// generated by compile -asmhdr from package %s\n\n", localpkg.Name)
for _, n := range asmlist {
if n.Sym.IsBlank() {
continue
}
switch n.Op {
case OLITERAL:
fmt.Fprintf(b, "#define const_%s %#v\n", n.Sym.Name, n.Val())
case OTYPE:
t := n.Type
if !t.IsStruct() || t.StructType().Map != nil || t.IsFuncArgStruct() {
break
}
fmt.Fprintf(b, "#define %s__size %d\n", n.Sym.Name, int(t.Width))
for _, f := range t.Fields().Slice() {
if !f.Sym.IsBlank() {
fmt.Fprintf(b, "#define %s_%s %d\n", n.Sym.Name, f.Sym.Name, int(f.Offset))
}
}
}
}
b.Close()
}