cmd/compile: use edge weights to decide inlineability in PGO

Currently, with PGO, the inliner uses node weights to decide if a
function is inlineable (with a larger budget). But the actual
inlining is determined by the weight of the call edge. There is a
discrepancy that, if a callee node is hot but the call edge is not,
it would not inlined, and marking the callee inlineable would of
no use.

Instead of using two kinds of weights, we just use the edge
weights to decide inlineability. If a function is the callee of a
hot call edge, its inlineability is determined with a larger
threshold. For a function that exceeds the regular inlining budget,
it is still inlined only when the call edge is hot, as it would
exceed the regular inlining cost for non-hot call sites, even if
it is marked inlineable.

For #55022.

Change-Id: I93fa9919fc6bcbb394e6cfe54ec96a96eede08f7
Reviewed-on: https://go-review.googlesource.com/c/go/+/447015
Run-TryBot: Cherry Mui <cherryyz@google.com>
Reviewed-by: Michael Pratt <mpratt@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>

diff --git a/src/cmd/compile/internal/base/debug.go b/src/cmd/compile/internal/base/debug.go
index 682c0dd518118d9ca248aaf03f3fd6565ae4ca8d..18422256ab61cfd7a99a006c47e37745b80cc4c3 100644 (file)
--- a/src/cmd/compile/internal/base/debug.go
+++ b/src/cmd/compile/internal/base/debug.go
@@ -46,7 +46,6 @@ type DebugFlags struct {
        WB                         int    `help:"print information about write barriers"`
        ABIWrap                    int    `help:"print information about ABI wrapper generation"`
        MayMoreStack               string `help:"call named function before all stack growth checks"`
-       InlineHotFuncThreshold     string `help:"threshold percentage for determining functions as hot candidates for inlining"`
        InlineHotCallSiteThreshold string `help:"threshold percentage for determining call sites as hot candidates for inlining"`
        InlineHotBudget            int    `help:"inline budget for hot functions"`
        PGOInline                  int    `help:"debug profile-guided inlining"`

diff --git a/src/cmd/compile/internal/inline/inl.go b/src/cmd/compile/internal/inline/inl.go
index 335bb23ecb91fa3236b69b4a75e5900bf14becd8..98bfb4e382ca1fe552c47110cf45513b4845e599 100644 (file)
--- a/src/cmd/compile/internal/inline/inl.go
+++ b/src/cmd/compile/internal/inline/inl.go
@@ -56,9 +56,9 @@ const (
 )
 
 var (
-       // List of all hot nodes.
+       // List of all hot callee nodes.
        // TODO(prattmic): Make this non-global.
-       candHotNodeMap = make(map[*pgo.IRNode]struct{})
+       candHotCalleeMap = make(map[*pgo.IRNode]struct{})
 
        // List of all hot call sites. CallSiteInfo.Callee is always nil.
        // TODO(prattmic): Make this non-global.
@@ -68,9 +68,6 @@ var (
        // TODO(prattmic): Make this non-global.
        inlinedCallSites = make(map[pgo.CallSiteInfo]struct{})
 
-       // Threshold in percentage for hot function inlining.
-       inlineHotFuncThresholdPercent = float64(2)
-
        // Threshold in percentage for hot callsite inlining.
        inlineHotCallSiteThresholdPercent = float64(0.1)
 
@@ -80,13 +77,6 @@ var (
 
 // pgoInlinePrologue records the hot callsites from ir-graph.
 func pgoInlinePrologue(p *pgo.Profile) {
-       if s, err := strconv.ParseFloat(base.Debug.InlineHotFuncThreshold, 64); err == nil {
-               inlineHotFuncThresholdPercent = s
-               if base.Debug.PGOInline > 0 {
-                       fmt.Printf("hot-node-thres=%v\n", inlineHotFuncThresholdPercent)
-               }
-       }
-
        if s, err := strconv.ParseFloat(base.Debug.InlineHotCallSiteThreshold, 64); err == nil {
                inlineHotCallSiteThresholdPercent = s
                if base.Debug.PGOInline > 0 {
@@ -102,10 +92,6 @@ func pgoInlinePrologue(p *pgo.Profile) {
                for _, f := range list {
                        name := ir.PkgFuncName(f)
                        if n, ok := p.WeightedCG.IRNodes[name]; ok {
-                               nodeweight := pgo.WeightInPercentage(n.Flat, p.TotalNodeWeight)
-                               if nodeweight > inlineHotFuncThresholdPercent {
-                                       candHotNodeMap[n] = struct{}{}
-                               }
                                for _, e := range p.WeightedCG.OutEdges[n] {
                                        if e.Weight != 0 {
                                                edgeweightpercent := pgo.WeightInPercentage(e.Weight, p.TotalEdgeWeight)
@@ -113,6 +99,7 @@ func pgoInlinePrologue(p *pgo.Profile) {
                                                        csi := pgo.CallSiteInfo{Line: e.CallSite, Caller: n.AST}
                                                        if _, ok := candHotEdgeMap[csi]; !ok {
                                                                candHotEdgeMap[csi] = struct{}{}
+                                                               candHotCalleeMap[e.Dst] = struct{}{}
                                                        }
                                                }
                                        }
@@ -122,7 +109,7 @@ func pgoInlinePrologue(p *pgo.Profile) {
        })
        if base.Debug.PGOInline > 0 {
                fmt.Printf("hot-cg before inline in dot format:")
-               p.PrintWeightedCallGraphDOT(inlineHotFuncThresholdPercent, inlineHotCallSiteThresholdPercent)
+               p.PrintWeightedCallGraphDOT(inlineHotCallSiteThresholdPercent)
        }
 }
 
@@ -139,7 +126,7 @@ func pgoInlineEpilogue(p *pgo.Profile) {
                })
                // Print the call-graph after inlining. This is a debugging feature.
                fmt.Printf("hot-cg after inline in dot:")
-               p.PrintWeightedCallGraphDOT(inlineHotFuncThresholdPercent, inlineHotCallSiteThresholdPercent)
+               p.PrintWeightedCallGraphDOT(inlineHotCallSiteThresholdPercent)
        }
 }
 
@@ -270,7 +257,7 @@ func CanInline(fn *ir.Func, profile *pgo.Profile) {
        budget := int32(inlineMaxBudget)
        if profile != nil {
                if n, ok := profile.WeightedCG.IRNodes[ir.PkgFuncName(fn)]; ok {
-                       if _, ok := candHotNodeMap[n]; ok {
+                       if _, ok := candHotCalleeMap[n]; ok {
                                budget = int32(inlineHotMaxBudget)
                                if base.Debug.PGOInline > 0 {
                                        fmt.Printf("hot-node enabled increased budget=%v for func=%v\n", budget, ir.PkgFuncName(fn))

diff --git a/src/cmd/compile/internal/pgo/irgraph.go b/src/cmd/compile/internal/pgo/irgraph.go
index 56cfebf85e3ff9f92e54c8125b9e672a6af89943..f0932b51ea275b32bc636000817223c9ad8b1a3a 100644 (file)
--- a/src/cmd/compile/internal/pgo/irgraph.go
+++ b/src/cmd/compile/internal/pgo/irgraph.go
@@ -150,7 +150,7 @@ func New(profileFile string) *Profile {
        p := &Profile{
                NodeMap:      make(map[NodeMapKey]*Weights),
                ProfileGraph: g,
-               WeightedCG:   &IRGraph{
+               WeightedCG: &IRGraph{
                        IRNodes: make(map[string]*IRNode),
                },
        }
@@ -347,7 +347,7 @@ func WeightInPercentage(value int64, total int64) float64 {
 }
 
 // PrintWeightedCallGraphDOT prints IRGraph in DOT format.
-func (p *Profile) PrintWeightedCallGraphDOT(nodeThreshold float64, edgeThreshold float64) {
+func (p *Profile) PrintWeightedCallGraphDOT(edgeThreshold float64) {
        fmt.Printf("\ndigraph G {\n")
        fmt.Printf("forcelabels=true;\n")
 
@@ -383,9 +383,6 @@ func (p *Profile) PrintWeightedCallGraphDOT(nodeThreshold float64, edgeThreshold
                if n, ok := p.WeightedCG.IRNodes[name]; ok {
                        nodeweight := WeightInPercentage(n.Flat, p.TotalNodeWeight)
                        color := "black"
-                       if nodeweight > nodeThreshold {
-                               color = "red"
-                       }
                        if ast.Inl != nil {
                                fmt.Printf("\"%v\" [color=%v,label=\"%v,freq=%.2f,inl_cost=%d\"];\n", ir.PkgFuncName(ast), color, ir.PkgFuncName(ast), nodeweight, ast.Inl.Cost)
                        } else {