// TODO(prattmic): Make this non-global.
candHotEdgeMap = make(map[pgo.CallSiteInfo]struct{})
- // List of inlined call sites. CallSiteInfo.Callee is always nil.
- // TODO(prattmic): Make this non-global.
- inlinedCallSites = make(map[pgo.CallSiteInfo]struct{})
-
// Threshold in percentage for hot callsite inlining.
inlineHotCallSiteThresholdPercent float64
return 0, nodes
}
-// pgoInlineEpilogue updates IRGraph after inlining.
-func pgoInlineEpilogue(p *pgo.Profile, decls []ir.Node) {
- if base.Debug.PGOInline >= 2 {
- ir.VisitFuncsBottomUp(decls, func(list []*ir.Func, recursive bool) {
- for _, f := range list {
- name := ir.LinkFuncName(f)
- if n, ok := p.WeightedCG.IRNodes[name]; ok {
- p.RedirectEdges(n, inlinedCallSites)
- }
- }
- })
- // Print the call-graph after inlining. This is a debugging feature.
- fmt.Printf("hot-cg after inline in dot:")
- p.PrintWeightedCallGraphDOT(inlineHotCallSiteThresholdPercent)
- }
-}
-
// InlinePackage finds functions that can be inlined and clones them before walk expands them.
func InlinePackage(p *pgo.Profile) {
InlineDecls(p, typecheck.Target.Decls, true)
}
}
})
-
- if p != nil {
- pgoInlineEpilogue(p, decls)
- }
}
// garbageCollectUnreferencedHiddenClosures makes a pass over all the
fmt.Printf("%v: Before inlining: %+v\n", ir.Line(n), n)
}
- if base.Debug.PGOInline > 0 {
- csi := pgo.CallSiteInfo{LineOffset: pgo.NodeLineOffset(n, fn), Caller: ir.CurFunc}
- if _, ok := inlinedCallSites[csi]; !ok {
- inlinedCallSites[csi] = struct{}{}
- }
- }
-
res := InlineCall(n, fn, inlIndex)
if res == nil {
fmt.Printf("}\n")
}
-// RedirectEdges deletes and redirects out-edges from node cur based on
-// inlining information via inlinedCallSites.
-//
-// CallSiteInfo.Callee must be nil.
-func (p *Profile) RedirectEdges(cur *IRNode, inlinedCallSites map[CallSiteInfo]struct{}) {
- g := p.WeightedCG
-
- i := 0
- outs := g.OutEdges[cur]
- for i < len(outs) {
- outEdge := outs[i]
- redirected := false
- _, found := inlinedCallSites[CallSiteInfo{LineOffset: outEdge.CallSiteOffset, Caller: cur.AST}]
- if !found {
- for _, InEdge := range g.InEdges[cur] {
- if _, ok := inlinedCallSites[CallSiteInfo{LineOffset: InEdge.CallSiteOffset, Caller: InEdge.Src.AST}]; ok {
- weight := g.calculateWeight(InEdge.Src, cur)
- g.redirectEdge(InEdge.Src, outEdge, weight)
- redirected = true
- }
- }
- }
- if found || redirected {
- g.remove(cur, i)
- outs = g.OutEdges[cur]
- continue
- }
- i++
- }
-}
-
-// redirectEdge redirects a node's out-edge to one of its parent nodes, cloning is
-// required as the node might be inlined in multiple call-sites.
-// TODO: adjust the in-edges of outEdge.Dst if necessary
-func (g *IRGraph) redirectEdge(parent *IRNode, outEdge *IREdge, weight int64) {
- edge := &IREdge{Src: parent, Dst: outEdge.Dst, Weight: weight * outEdge.Weight, CallSiteOffset: outEdge.CallSiteOffset}
- g.OutEdges[parent] = append(g.OutEdges[parent], edge)
-}
-
-// remove deletes the cur-node's out-edges at index idx.
-func (g *IRGraph) remove(cur *IRNode, i int) {
- if len(g.OutEdges[cur]) >= 2 {
- g.OutEdges[cur][i] = g.OutEdges[cur][len(g.OutEdges[cur])-1]
- g.OutEdges[cur] = g.OutEdges[cur][:len(g.OutEdges[cur])-1]
- } else {
- delete(g.OutEdges, cur)
- }
-}
-
-// calculateWeight calculates the weight of the new redirected edge.
-func (g *IRGraph) calculateWeight(parent *IRNode, cur *IRNode) int64 {
- sum := int64(0)
- pw := int64(0)
- for _, InEdge := range g.InEdges[cur] {
- sum += InEdge.Weight
- if InEdge.Src == parent {
- pw = InEdge.Weight
- }
- }
- weight := int64(0)
- if sum != 0 {
- weight = pw / sum
- } else {
- weight = pw
- }
- return weight
-}
-
// inlCallee is same as the implementation for inl.go with one change. The change is that we do not invoke CanInline on a closure.
func inlCallee(fn ir.Node) *ir.Func {
fn = ir.StaticValue(fn)