diff --git a/src/cmd/compile/internal/gc/main.go b/src/cmd/compile/internal/gc/main.go index 5633f1fc045..3055879a2f9 100644 --- a/src/cmd/compile/internal/gc/main.go +++ b/src/cmd/compile/internal/gc/main.go @@ -252,24 +252,15 @@ func Main(archInit func(*ssagen.ArchInfo)) { // Read profile file and build profile-graph and weighted-call-graph. base.Timer.Start("fe", "pgoprofile") + var profile *pgo.Profile if base.Flag.PgoProfile != "" { - pgo.BuildProfileGraph(base.Flag.PgoProfile) - pgo.BuildWeightedCallGraph() + profile = pgo.New(base.Flag.PgoProfile) } // Inlining base.Timer.Start("fe", "inlining") if base.Flag.LowerL != 0 { - if pgo.WeightedCG != nil { - inline.InlinePrologue() - } - inline.InlinePackage() - if pgo.WeightedCG != nil { - inline.InlineEpilogue() - // Delete the graphs as no other optimization uses this currently. - pgo.WeightedCG = nil - pgo.ProfileGraph = nil - } + inline.InlinePackage(profile) } noder.MakeWrappers(typecheck.Target) // must happen after inlining diff --git a/src/cmd/compile/internal/inline/inl.go b/src/cmd/compile/internal/inline/inl.go index 7e1a9adae87..4909650ae4b 100644 --- a/src/cmd/compile/internal/inline/inl.go +++ b/src/cmd/compile/internal/inline/inl.go @@ -75,8 +75,8 @@ var ( inlineHotMaxBudget int32 = 160 ) -// InlinePrologue records the hot callsites from ir-graph. -func InlinePrologue() { +// pgoInlinePrologue records the hot callsites from ir-graph. +func pgoInlinePrologue(p *pgo.Profile) { if s, err := strconv.ParseFloat(base.Debug.InlineHotFuncThreshold, 64); err == nil { inlineHotFuncThresholdPercent = s if base.Debug.PGOInline > 0 { @@ -98,14 +98,14 @@ func InlinePrologue() { ir.VisitFuncsBottomUp(typecheck.Target.Decls, func(list []*ir.Func, recursive bool) { for _, f := range list { name := ir.PkgFuncName(f) - if n, ok := pgo.WeightedCG.IRNodes[name]; ok { - nodeweight := pgo.WeightInPercentage(n.Flat, pgo.GlobalTotalNodeWeight) + if n, ok := p.WeightedCG.IRNodes[name]; ok { + nodeweight := pgo.WeightInPercentage(n.Flat, p.TotalNodeWeight) if nodeweight > inlineHotFuncThresholdPercent { candHotNodeMap[n] = struct{}{} } - for _, e := range pgo.WeightedCG.OutEdges[n] { + for _, e := range p.WeightedCG.OutEdges[n] { if e.Weight != 0 { - edgeweightpercent := pgo.WeightInPercentage(e.Weight, pgo.GlobalTotalEdgeWeight) + edgeweightpercent := pgo.WeightInPercentage(e.Weight, p.TotalEdgeWeight) if edgeweightpercent > inlineHotCallSiteThresholdPercent { csi := pgo.CallSiteInfo{Line: e.CallSite, Caller: n.AST, Callee: e.Dst.AST} if _, ok := candHotEdgeMap[csi]; !ok { @@ -119,29 +119,33 @@ func InlinePrologue() { }) if base.Debug.PGOInline > 0 { fmt.Printf("hot-cg before inline in dot format:") - pgo.PrintWeightedCallGraphDOT(inlineHotFuncThresholdPercent, inlineHotCallSiteThresholdPercent) + p.PrintWeightedCallGraphDOT(inlineHotFuncThresholdPercent, inlineHotCallSiteThresholdPercent) } } -// InlineEpilogue updates IRGraph after inlining. -func InlineEpilogue() { +// pgoInlineEpilogue updates IRGraph after inlining. +func pgoInlineEpilogue(p *pgo.Profile) { if base.Debug.PGOInline > 0 { ir.VisitFuncsBottomUp(typecheck.Target.Decls, func(list []*ir.Func, recursive bool) { for _, f := range list { name := ir.PkgFuncName(f) - if n, ok := pgo.WeightedCG.IRNodes[name]; ok { - pgo.RedirectEdges(n, inlinedCallSites) + if n, ok := p.WeightedCG.IRNodes[name]; ok { + p.RedirectEdges(n, inlinedCallSites) } } }) // Print the call-graph after inlining. This is a debugging feature. fmt.Printf("hot-cg after inline in dot:") - pgo.PrintWeightedCallGraphDOT(inlineHotFuncThresholdPercent, inlineHotCallSiteThresholdPercent) + p.PrintWeightedCallGraphDOT(inlineHotFuncThresholdPercent, inlineHotCallSiteThresholdPercent) } } // InlinePackage finds functions that can be inlined and clones them before walk expands them. -func InlinePackage() { +func InlinePackage(p *pgo.Profile) { + if p != nil { + pgoInlinePrologue(p) + } + ir.VisitFuncsBottomUp(typecheck.Target.Decls, func(list []*ir.Func, recursive bool) { numfns := numNonClosures(list) for _, n := range list { @@ -149,21 +153,25 @@ func InlinePackage() { // We allow inlining if there is no // recursion, or the recursion cycle is // across more than one function. - CanInline(n) + CanInline(n, p) } else { if base.Flag.LowerM > 1 { fmt.Printf("%v: cannot inline %v: recursive\n", ir.Line(n), n.Nname) } } - InlineCalls(n) + InlineCalls(n, p) } }) + + if p != nil { + pgoInlineEpilogue(p) + } } // CanInline determines whether fn is inlineable. // If so, CanInline saves copies of fn.Body and fn.Dcl in fn.Inl. // fn and fn.Body will already have been typechecked. -func CanInline(fn *ir.Func) { +func CanInline(fn *ir.Func, profile *pgo.Profile) { if fn.Nname == nil { base.Fatalf("CanInline no nname %+v", fn) } @@ -260,8 +268,8 @@ func CanInline(fn *ir.Func) { // Update the budget for profile-guided inlining. budget := int32(inlineMaxBudget) - if base.Flag.PgoProfile != "" && pgo.WeightedCG != nil { - if n, ok := pgo.WeightedCG.IRNodes[ir.PkgFuncName(fn)]; ok { + if profile != nil { + if n, ok := profile.WeightedCG.IRNodes[ir.PkgFuncName(fn)]; ok { if _, ok := candHotNodeMap[n]; ok { budget = int32(inlineHotMaxBudget) if base.Debug.PGOInline > 0 { @@ -285,6 +293,7 @@ func CanInline(fn *ir.Func) { budget: budget, maxBudget: budget, extraCallCost: cc, + profile: profile, } if visitor.tooHairy(fn) { reason = visitor.reason @@ -352,6 +361,7 @@ type hairyVisitor struct { extraCallCost int32 usedLocals ir.NameSet do func(ir.Node) bool + profile *pgo.Profile } func (v *hairyVisitor) tooHairy(fn *ir.Func) bool { @@ -439,8 +449,8 @@ func (v *hairyVisitor) doNode(n ir.Node) bool { } // Determine if the callee edge is a for hot callee or not. - if base.Flag.PgoProfile != "" && pgo.WeightedCG != nil && v.curFunc != nil { - if fn := inlCallee(n.X); fn != nil && typecheck.HaveInlineBody(fn) { + if v.profile != nil && v.curFunc != nil { + if fn := inlCallee(n.X, v.profile); fn != nil && typecheck.HaveInlineBody(fn) { line := int(base.Ctxt.InnermostPos(n.Pos()).RelLine()) csi := pgo.CallSiteInfo{Line: line, Caller: v.curFunc, Callee: fn} if _, o := candHotEdgeMap[csi]; o { @@ -457,7 +467,7 @@ func (v *hairyVisitor) doNode(n ir.Node) bool { break } - if fn := inlCallee(n.X); fn != nil && typecheck.HaveInlineBody(fn) { + if fn := inlCallee(n.X, v.profile); fn != nil && typecheck.HaveInlineBody(fn) { v.budget -= fn.Inl.Cost break } @@ -687,7 +697,7 @@ func inlcopy(n ir.Node) ir.Node { // InlineCalls/inlnode walks fn's statements and expressions and substitutes any // calls made to inlineable functions. This is the external entry point. -func InlineCalls(fn *ir.Func) { +func InlineCalls(fn *ir.Func, profile *pgo.Profile) { savefn := ir.CurFunc ir.CurFunc = fn maxCost := int32(inlineMaxBudget) @@ -697,7 +707,7 @@ func InlineCalls(fn *ir.Func) { var inlCalls []*ir.InlinedCallExpr var edit func(ir.Node) ir.Node edit = func(n ir.Node) ir.Node { - return inlnode(n, maxCost, &inlCalls, edit) + return inlnode(n, maxCost, &inlCalls, edit, profile) } ir.EditChildren(fn, edit) @@ -728,7 +738,7 @@ func InlineCalls(fn *ir.Func) { // The result of inlnode MUST be assigned back to n, e.g. // // n.Left = inlnode(n.Left) -func inlnode(n ir.Node, maxCost int32, inlCalls *[]*ir.InlinedCallExpr, edit func(ir.Node) ir.Node) ir.Node { +func inlnode(n ir.Node, maxCost int32, inlCalls *[]*ir.InlinedCallExpr, edit func(ir.Node) ir.Node, profile *pgo.Profile) ir.Node { if n == nil { return n } @@ -789,7 +799,7 @@ func inlnode(n ir.Node, maxCost int32, inlCalls *[]*ir.InlinedCallExpr, edit fun if ir.IsIntrinsicCall(call) { break } - if fn := inlCallee(call.X); fn != nil && typecheck.HaveInlineBody(fn) { + if fn := inlCallee(call.X, profile); fn != nil && typecheck.HaveInlineBody(fn) { n = mkinlcall(call, fn, maxCost, inlCalls, edit) } } @@ -801,7 +811,7 @@ func inlnode(n ir.Node, maxCost int32, inlCalls *[]*ir.InlinedCallExpr, edit fun // inlCallee takes a function-typed expression and returns the underlying function ONAME // that it refers to if statically known. Otherwise, it returns nil. -func inlCallee(fn ir.Node) *ir.Func { +func inlCallee(fn ir.Node, profile *pgo.Profile) *ir.Func { fn = ir.StaticValue(fn) switch fn.Op() { case ir.OMETHEXPR: @@ -822,7 +832,7 @@ func inlCallee(fn ir.Node) *ir.Func { case ir.OCLOSURE: fn := fn.(*ir.ClosureExpr) c := fn.Func - CanInline(c) + CanInline(c, profile) return c } return nil diff --git a/src/cmd/compile/internal/noder/reader.go b/src/cmd/compile/internal/noder/reader.go index b8df7c97734..fe90f52b4db 100644 --- a/src/cmd/compile/internal/noder/reader.go +++ b/src/cmd/compile/internal/noder/reader.go @@ -3867,7 +3867,8 @@ func finishWrapperFunc(fn *ir.Func, target *ir.Package) { // We generate wrappers after the global inlining pass, // so we're responsible for applying inlining ourselves here. - inline.InlineCalls(fn) + // TODO(prattmic): plumb PGO. + inline.InlineCalls(fn, nil) // The body of wrapper function after inlining may reveal new ir.OMETHVALUE node, // we don't know whether wrapper function has been generated for it or not, so diff --git a/src/cmd/compile/internal/pgo/graph.go b/src/cmd/compile/internal/pgo/graph.go index d7b9432f0fd..bc91dd48239 100644 --- a/src/cmd/compile/internal/pgo/graph.go +++ b/src/cmd/compile/internal/pgo/graph.go @@ -289,15 +289,6 @@ func SortTags(t []*Tag, flat bool) []*Tag { return ts.t } -// New summarizes performance data from a profile into a graph. -func New(prof *profile.Profile, o *Options) *Graph { - if o.CallTree { - return newTree(prof, o) - } - g, _ := newGraph(prof, o) - return g -} - // newGraph computes a graph from a profile. It returns the graph, and // a map from the profile location indices to the corresponding graph // nodes. diff --git a/src/cmd/compile/internal/pgo/irgraph.go b/src/cmd/compile/internal/pgo/irgraph.go index e6e183cdd22..8fb256739ef 100644 --- a/src/cmd/compile/internal/pgo/irgraph.go +++ b/src/cmd/compile/internal/pgo/irgraph.go @@ -52,7 +52,10 @@ import ( "os" ) -// IRGraph is the key datastrcture that is built from profile. It is essentially a call graph with nodes pointing to IRs of functions and edges carrying weights and callsite information. The graph is bidirectional that helps in removing nodes efficiently. +// IRGraph is the key datastrcture that is built from profile. It is +// essentially a call graph with nodes pointing to IRs of functions and edges +// carrying weights and callsite information. The graph is bidirectional that +// helps in removing nodes efficiently. type IRGraph struct { // Nodes of the graph IRNodes map[string]*IRNode @@ -73,7 +76,8 @@ type IRNode struct { // IREdgeMap maps an IRNode to its successors. type IREdgeMap map[*IRNode][]*IREdge -// IREdge represents a call edge in the IRGraph with source, destination, weight, callsite, and line number information. +// IREdge represents a call edge in the IRGraph with source, destination, +// weight, callsite, and line number information. type IREdge struct { // Source and destination of the edge in IRNode. Src, Dst *IRNode @@ -81,7 +85,8 @@ type IREdge struct { CallSite int } -// NodeMapKey represents a hash key to identify unique call-edges in profile and in IR. Used for deduplication of call edges found in profile. +// NodeMapKey represents a hash key to identify unique call-edges in profile +// and in IR. Used for deduplication of call edges found in profile. type NodeMapKey struct { CallerName string CalleeName string @@ -102,84 +107,93 @@ type CallSiteInfo struct { Callee *ir.Func } -var ( - // Aggregated NodeWeights and EdgeWeights across profiles. This helps us determine the percentage threshold for hot/cold partitioning. - GlobalTotalNodeWeight = int64(0) - GlobalTotalEdgeWeight = int64(0) - - // Global node and their aggregated weight information. - GlobalNodeMap = make(map[NodeMapKey]*Weights) - - // WeightedCG represents the IRGraph built from profile, which we will update as part of inlining. - WeightedCG *IRGraph - +// Profile contains the processed PGO profile and weighted call graph used for +// PGO optimizations. +type Profile struct { // Original profile-graph. ProfileGraph *Graph - // Per-caller data structure to track the list of hot call sites. This gets rewritten every caller leaving it to GC for cleanup. + // Aggregated NodeWeights and EdgeWeights across the profile. This + // helps us determine the percentage threshold for hot/cold + // partitioning. + TotalNodeWeight int64 + TotalEdgeWeight int64 + + // NodeMap contains all unique call-edges in the profile and their + // aggregated weight. + NodeMap map[NodeMapKey]*Weights + + // WeightedCG represents the IRGraph built from profile, which we will + // update as part of inlining. + WeightedCG *IRGraph +} + +var ( + // Per-caller data structure to track the list of hot call sites. This + // gets rewritten every caller leaving it to GC for cleanup. + // + // TODO(prattmic): Make this non-global. Use of this seems to assume + // inline.CanInline is called immediately before inline.InlineCalls, + // which isn't necessarily true? ListOfHotCallSites = make(map[CallSiteInfo]struct{}) ) -// BuildProfileGraph generates a profile-graph from the profile. -func BuildProfileGraph(profileFile string) { - - // if possible, we should cache the profile-graph. - if ProfileGraph != nil { - return - } - - // open the profile file. +// New generates a profile-graph from the profile. +func New(profileFile string) *Profile { f, err := os.Open(profileFile) if err != nil { log.Fatal("failed to open file " + profileFile) - return + return nil } defer f.Close() - p, err := profile.Parse(f) + profile, err := profile.Parse(f) if err != nil { log.Fatal("failed to Parse profile file.") - return + return nil } - // Build the options. - opt := &Options{ + + g, _ := newGraph(profile, &Options{ CallTree: false, SampleValue: func(v []int64) int64 { return v[1] }, + }) + + p := &Profile{ + NodeMap: make(map[NodeMapKey]*Weights), + ProfileGraph: g, + WeightedCG: &IRGraph{ + IRNodes: make(map[string]*IRNode), + }, } - // Build the graph using profile package. - ProfileGraph = New(p, opt) - // Build various global maps from profile. - preprocessProfileGraph() - -} - -// BuildWeightedCallGraph generates a weighted callgraph from the profile for the current package. -func BuildWeightedCallGraph() { - - // Bail if there is no profile-graph available. - if ProfileGraph == nil { - return - } + // Build the node map and totals from the profile graph. + p.preprocessProfileGraph() // Create package-level call graph with weights from profile and IR. - WeightedCG = createIRGraph() + p.initializeIRGraph() + + return p } -// preprocessProfileGraph builds various maps from the profile-graph. It builds GlobalNodeMap and other information based on the name and callsite to compute node and edge weights which will be used later on to create edges for WeightedCG. -func preprocessProfileGraph() { +// preprocessProfileGraph builds various maps from the profile-graph. +// +// It initializes NodeMap and Total{Node,Edge}Weight based on the name and +// callsite to compute node and edge weights which will be used later on to +// create edges for WeightedCG. +func (p *Profile) preprocessProfileGraph() { nFlat := make(map[string]int64) nCum := make(map[string]int64) // Accummulate weights for the same node. - for _, n := range ProfileGraph.Nodes { + for _, n := range p.ProfileGraph.Nodes { canonicalName := n.Info.Name nFlat[canonicalName] += n.FlatValue() nCum[canonicalName] += n.CumValue() } - // Process ProfileGraph and build various node and edge maps which will be consumed by AST walk. - for _, n := range ProfileGraph.Nodes { - GlobalTotalNodeWeight += n.FlatValue() + // Process ProfileGraph and build various node and edge maps which will + // be consumed by AST walk. + for _, n := range p.ProfileGraph.Nodes { + p.TotalNodeWeight += n.FlatValue() canonicalName := n.Info.Name // Create the key to the NodeMapKey. nodeinfo := NodeMapKey{ @@ -188,35 +202,37 @@ func preprocessProfileGraph() { } for _, e := range n.Out { - GlobalTotalEdgeWeight += e.WeightValue() + p.TotalEdgeWeight += e.WeightValue() nodeinfo.CalleeName = e.Dest.Info.Name - if w, ok := GlobalNodeMap[nodeinfo]; ok { + if w, ok := p.NodeMap[nodeinfo]; ok { w.EWeight += e.WeightValue() } else { weights := new(Weights) weights.NFlat = nFlat[canonicalName] weights.NCum = nCum[canonicalName] weights.EWeight = e.WeightValue() - GlobalNodeMap[nodeinfo] = weights + p.NodeMap[nodeinfo] = weights } } } } -// createIRGraph builds the IRGraph by visting all the ir.Func in decl list of a package. -func createIRGraph() *IRGraph { - var g IRGraph +// initializeIRGraph builds the IRGraph by visting all the ir.Func in decl list +// of a package. +func (p *Profile) initializeIRGraph() { // Bottomup walk over the function to create IRGraph. ir.VisitFuncsBottomUp(typecheck.Target.Decls, func(list []*ir.Func, recursive bool) { for _, n := range list { - g.Visit(n, recursive) + p.VisitIR(n, recursive) } }) - return &g } -// Visit traverses the body of each ir.Func and use GlobalNodeMap to determine if we need to add an edge from ir.Func and any node in the ir.Func body. -func (g *IRGraph) Visit(fn *ir.Func, recursive bool) { +// VisitIR traverses the body of each ir.Func and use NodeMap to determine if +// we need to add an edge from ir.Func and any node in the ir.Func body. +func (p *Profile) VisitIR(fn *ir.Func, recursive bool) { + g := p.WeightedCG + if g.IRNodes == nil { g.IRNodes = make(map[string]*IRNode) } @@ -239,17 +255,19 @@ func (g *IRGraph) Visit(fn *ir.Func, recursive bool) { CallSite: -1, } // If the node exists, then update its node weight. - if weights, ok := GlobalNodeMap[nodeinfo]; ok { + if weights, ok := p.NodeMap[nodeinfo]; ok { g.IRNodes[name].Flat = weights.NFlat g.IRNodes[name].Cum = weights.NCum } // Recursively walk over the body of the function to create IRGraph edges. - g.createIRGraphEdge(fn, g.IRNodes[name], name) + p.createIRGraphEdge(fn, g.IRNodes[name], name) } -// addEdge adds an edge between caller and new node that points to `callee` based on the profile-graph and GlobalNodeMap. -func (g *IRGraph) addEdge(caller *IRNode, callee *ir.Func, n *ir.Node, callername string, line int) { +// addIREdge adds an edge between caller and new node that points to `callee` +// based on the profile-graph and NodeMap. +func (p *Profile) addIREdge(caller *IRNode, callee *ir.Func, n *ir.Node, callername string, line int) { + g := p.WeightedCG // Create an IRNode for the callee. calleenode := new(IRNode) @@ -271,13 +289,13 @@ func (g *IRGraph) addEdge(caller *IRNode, callee *ir.Func, n *ir.Node, callernam CalleeName: "", CallSite: -1, } - if weights, ok := GlobalNodeMap[nodeinfo2]; ok { + if weights, ok := p.NodeMap[nodeinfo2]; ok { g.IRNodes[calleename].Flat = weights.NFlat g.IRNodes[calleename].Cum = weights.NCum } } - if weights, ok := GlobalNodeMap[nodeinfo]; ok { + if weights, ok := p.NodeMap[nodeinfo]; ok { caller.Flat = weights.NFlat caller.Cum = weights.NCum @@ -288,7 +306,7 @@ func (g *IRGraph) addEdge(caller *IRNode, callee *ir.Func, n *ir.Node, callernam } else { nodeinfo.CalleeName = "" nodeinfo.CallSite = -1 - if weights, ok := GlobalNodeMap[nodeinfo]; ok { + if weights, ok := p.NodeMap[nodeinfo]; ok { caller.Flat = weights.NFlat caller.Cum = weights.NCum info := &IREdge{Src: caller, Dst: g.IRNodes[calleename], Weight: 0, CallSite: line} @@ -303,7 +321,7 @@ func (g *IRGraph) addEdge(caller *IRNode, callee *ir.Func, n *ir.Node, callernam } // createIRGraphEdge traverses the nodes in the body of ir.Func and add edges between callernode which points to the ir.Func and the nodes in the body. -func (g *IRGraph) createIRGraphEdge(fn *ir.Func, callernode *IRNode, name string) { +func (p *Profile) createIRGraphEdge(fn *ir.Func, callernode *IRNode, name string) { var doNode func(ir.Node) bool doNode = func(n ir.Node) bool { switch n.Op() { @@ -315,14 +333,14 @@ func (g *IRGraph) createIRGraphEdge(fn *ir.Func, callernode *IRNode, name string // Find the callee function from the call site and add the edge. f := inlCallee(call.X) if f != nil { - g.addEdge(callernode, f, &n, name, line) + p.addIREdge(callernode, f, &n, name, line) } case ir.OCALLMETH: call := n.(*ir.CallExpr) // Find the callee method from the call site and add the edge. fn2 := ir.MethodExprName(call.X).Func line := int(base.Ctxt.InnermostPos(n.Pos()).RelLine()) - g.addEdge(callernode, fn2, &n, name, line) + p.addIREdge(callernode, fn2, &n, name, line) } return false } @@ -339,7 +357,7 @@ func WeightInPercentage(value int64, total int64) float64 { } // PrintWeightedCallGraphDOT prints IRGraph in DOT format. -func PrintWeightedCallGraphDOT(nodeThreshold float64, edgeThreshold float64) { +func (p *Profile) PrintWeightedCallGraphDOT(nodeThreshold float64, edgeThreshold float64) { fmt.Printf("\ndigraph G {\n") fmt.Printf("forcelabels=true;\n") @@ -355,8 +373,8 @@ func PrintWeightedCallGraphDOT(nodeThreshold float64, edgeThreshold float64) { // Determine nodes of DOT. nodes := make(map[string]*ir.Func) for name, _ := range funcs { - if n, ok := WeightedCG.IRNodes[name]; ok { - for _, e := range WeightedCG.OutEdges[n] { + if n, ok := p.WeightedCG.IRNodes[name]; ok { + for _, e := range p.WeightedCG.OutEdges[n] { if _, ok := nodes[ir.PkgFuncName(e.Src.AST)]; !ok { nodes[ir.PkgFuncName(e.Src.AST)] = e.Src.AST } @@ -372,8 +390,8 @@ func PrintWeightedCallGraphDOT(nodeThreshold float64, edgeThreshold float64) { // Print nodes. for name, ast := range nodes { - if n, ok := WeightedCG.IRNodes[name]; ok { - nodeweight := WeightInPercentage(n.Flat, GlobalTotalNodeWeight) + if n, ok := p.WeightedCG.IRNodes[name]; ok { + nodeweight := WeightInPercentage(n.Flat, p.TotalNodeWeight) color := "black" if nodeweight > nodeThreshold { color = "red" @@ -389,9 +407,9 @@ func PrintWeightedCallGraphDOT(nodeThreshold float64, edgeThreshold float64) { ir.VisitFuncsBottomUp(typecheck.Target.Decls, func(list []*ir.Func, recursive bool) { for _, f := range list { name := ir.PkgFuncName(f) - if n, ok := WeightedCG.IRNodes[name]; ok { - for _, e := range WeightedCG.OutEdges[n] { - edgepercent := WeightInPercentage(e.Weight, GlobalTotalEdgeWeight) + if n, ok := p.WeightedCG.IRNodes[name]; ok { + for _, e := range p.WeightedCG.OutEdges[n] { + edgepercent := WeightInPercentage(e.Weight, p.TotalEdgeWeight) if edgepercent > edgeThreshold { fmt.Printf("edge [color=red, style=solid];\n") } else { @@ -406,8 +424,28 @@ func PrintWeightedCallGraphDOT(nodeThreshold float64, edgeThreshold float64) { fmt.Printf("}\n") } -// redirectEdges deletes the cur node out-edges and redirect them so now these edges are the parent node out-edges. -func redirectEdges(g *IRGraph, parent *IRNode, cur *IRNode) { +// RedirectEdges deletes and redirects out-edges from node cur based on inlining information via inlinedCallSites. +func (p *Profile) RedirectEdges(cur *IRNode, inlinedCallSites map[CallSiteInfo]struct{}) { + g := p.WeightedCG + + for i, outEdge := range g.OutEdges[cur] { + if _, found := inlinedCallSites[CallSiteInfo{Line: outEdge.CallSite, Caller: cur.AST}]; !found { + for _, InEdge := range g.InEdges[cur] { + if _, ok := inlinedCallSites[CallSiteInfo{Line: InEdge.CallSite, Caller: InEdge.Src.AST}]; ok { + weight := g.calculateWeight(InEdge.Src, cur) + g.redirectEdge(InEdge.Src, cur, outEdge, weight, i) + } + } + } else { + g.remove(cur, i, outEdge.Dst.AST.Nname) + } + } + g.removeall(cur) +} + +// redirectEdges deletes the cur node out-edges and redirect them so now these +// edges are the parent node out-edges. +func (g *IRGraph) redirectEdges(parent *IRNode, cur *IRNode) { for _, outEdge := range g.OutEdges[cur] { outEdge.Src = parent g.OutEdges[parent] = append(g.OutEdges[parent], outEdge) @@ -415,26 +453,36 @@ func redirectEdges(g *IRGraph, parent *IRNode, cur *IRNode) { delete(g.OutEdges, cur) } -// RedirectEdges deletes and redirects out-edges from node cur based on inlining information via inlinedCallSites. -func RedirectEdges(cur *IRNode, inlinedCallSites map[CallSiteInfo]struct{}) { - g := WeightedCG - for i, outEdge := range g.OutEdges[cur] { - if _, found := inlinedCallSites[CallSiteInfo{Line: outEdge.CallSite, Caller: cur.AST}]; !found { - for _, InEdge := range g.InEdges[cur] { - if _, ok := inlinedCallSites[CallSiteInfo{Line: InEdge.CallSite, Caller: InEdge.Src.AST}]; ok { - weight := calculateweight(g, InEdge.Src, cur) - redirectEdge(g, InEdge.Src, cur, outEdge, weight, i) - } - } - } else { - remove(g, cur, i, outEdge.Dst.AST.Nname) - } - } - removeall(g, cur) +// redirectEdge deletes the cur-node's out-edges and redirect them so now these +// edges are the parent node out-edges. +func (g *IRGraph) redirectEdge(parent *IRNode, cur *IRNode, outEdge *IREdge, weight int64, idx int) { + outEdge.Src = parent + outEdge.Weight = weight * outEdge.Weight + g.OutEdges[parent] = append(g.OutEdges[parent], outEdge) + g.remove(cur, idx, outEdge.Dst.AST.Nname) } -// calculateweight calculates the weight of the new redirected edge. -func calculateweight(g *IRGraph, parent *IRNode, cur *IRNode) int64 { +// remove deletes the cur-node's out-edges at index idx. +func (g *IRGraph) remove(cur *IRNode, idx int, name *ir.Name) { + if len(g.OutEdges[cur]) >= 2 { + g.OutEdges[cur][idx] = &IREdge{CallSite: -1} + } else { + delete(g.OutEdges, cur) + } +} + +// removeall deletes all cur-node's out-edges that marked to be removed . +func (g *IRGraph) removeall(cur *IRNode) { + for i := len(g.OutEdges[cur]) - 1; i >= 0; i-- { + if g.OutEdges[cur][i].CallSite == -1 { + g.OutEdges[cur][i] = g.OutEdges[cur][len(g.OutEdges[cur])-1] + g.OutEdges[cur] = g.OutEdges[cur][:len(g.OutEdges[cur])-1] + } + } +} + +// calculateWeight calculates the weight of the new redirected edge. +func (g *IRGraph) calculateWeight(parent *IRNode, cur *IRNode) int64 { sum := int64(0) pw := int64(0) for _, InEdge := range g.InEdges[cur] { @@ -452,33 +500,6 @@ func calculateweight(g *IRGraph, parent *IRNode, cur *IRNode) int64 { return weight } -// redirectEdge deletes the cur-node's out-edges and redirect them so now these edges are the parent node out-edges. -func redirectEdge(g *IRGraph, parent *IRNode, cur *IRNode, outEdge *IREdge, weight int64, idx int) { - outEdge.Src = parent - outEdge.Weight = weight * outEdge.Weight - g.OutEdges[parent] = append(g.OutEdges[parent], outEdge) - remove(g, cur, idx, outEdge.Dst.AST.Nname) -} - -// remove deletes the cur-node's out-edges at index idx. -func remove(g *IRGraph, cur *IRNode, idx int, name *ir.Name) { - if len(g.OutEdges[cur]) >= 2 { - g.OutEdges[cur][idx] = &IREdge{CallSite: -1} - } else { - delete(g.OutEdges, cur) - } -} - -// removeall deletes all cur-node's out-edges that marked to be removed . -func removeall(g *IRGraph, cur *IRNode) { - for i := len(g.OutEdges[cur]) - 1; i >= 0; i-- { - if g.OutEdges[cur][i].CallSite == -1 { - g.OutEdges[cur][i] = g.OutEdges[cur][len(g.OutEdges[cur])-1] - g.OutEdges[cur] = g.OutEdges[cur][:len(g.OutEdges[cur])-1] - } - } -} - // inlCallee is same as the implementation for inl.go with one change. The change is that we do not invoke CanInline on a closure. func inlCallee(fn ir.Node) *ir.Func { fn = ir.StaticValue(fn) diff --git a/src/cmd/compile/internal/reflectdata/reflect.go b/src/cmd/compile/internal/reflectdata/reflect.go index 0f0c4051672..f4996668b6c 100644 --- a/src/cmd/compile/internal/reflectdata/reflect.go +++ b/src/cmd/compile/internal/reflectdata/reflect.go @@ -2010,7 +2010,8 @@ func methodWrapper(rcvr *types.Type, method *types.Field, forItab bool) *obj.LSy } } if canInline { - inline.InlineCalls(fn) + // TODO(prattmic): plumb PGO. + inline.InlineCalls(fn, nil) } escape.Batch([]*ir.Func{fn}, false) }