"fmt"
"go/constant"
"internal/goexperiment"
- "sort"
"strconv"
"cmd/compile/internal/base"
base.Fatalf("invalid PGOInlineCDFThreshold, must be between 0 and 100")
}
}
- var hotCallsites []pgo.NodeMapKey
+ var hotCallsites []pgo.NamedCallEdge
inlineHotCallSiteThresholdPercent, hotCallsites = hotNodesFromCDF(p)
if base.Debug.PGODebug > 0 {
fmt.Printf("hot-callsite-thres-from-CDF=%v\n", inlineHotCallSiteThresholdPercent)
// (currently only used in debug prints) (in case of equal weights,
// comparing with the threshold may not accurately reflect which nodes are
// considiered hot).
-func hotNodesFromCDF(p *pgo.Profile) (float64, []pgo.NodeMapKey) {
- nodes := make([]pgo.NodeMapKey, len(p.NodeMap))
- i := 0
- for n := range p.NodeMap {
- nodes[i] = n
- i++
- }
- sort.Slice(nodes, func(i, j int) bool {
- ni, nj := nodes[i], nodes[j]
- if wi, wj := p.NodeMap[ni].EWeight, p.NodeMap[nj].EWeight; wi != wj {
- return wi > wj // want larger weight first
- }
- // same weight, order by name/line number
- if ni.CallerName != nj.CallerName {
- return ni.CallerName < nj.CallerName
- }
- if ni.CalleeName != nj.CalleeName {
- return ni.CalleeName < nj.CalleeName
- }
- return ni.CallSiteOffset < nj.CallSiteOffset
- })
+func hotNodesFromCDF(p *pgo.Profile) (float64, []pgo.NamedCallEdge) {
cum := int64(0)
- for i, n := range nodes {
- w := p.NodeMap[n].EWeight
+ for i, n := range p.NamedEdgeMap.ByWeight {
+ w := p.NamedEdgeMap.Weight[n]
cum += w
- if pgo.WeightInPercentage(cum, p.TotalEdgeWeight) > inlineCDFHotCallSiteThresholdPercent {
+ if pgo.WeightInPercentage(cum, p.TotalWeight) > inlineCDFHotCallSiteThresholdPercent {
// nodes[:i+1] to include the very last node that makes it to go over the threshold.
// (Say, if the CDF threshold is 50% and one hot node takes 60% of weight, we want to
// include that node instead of excluding it.)
- return pgo.WeightInPercentage(w, p.TotalEdgeWeight), nodes[:i+1]
+ return pgo.WeightInPercentage(w, p.TotalWeight), p.NamedEdgeMap.ByWeight[:i+1]
}
}
- return 0, nodes
+ return 0, p.NamedEdgeMap.ByWeight
}
// InlinePackage finds functions that can be inlined and clones them before walk expands them.
garbageCollectUnreferencedHiddenClosures()
if base.Debug.DumpInlFuncProps != "" {
- inlheur.DumpFuncProps(nil, base.Debug.DumpInlFuncProps, nil)
+ inlheur.DumpFuncProps(nil, base.Debug.DumpInlFuncProps, nil, inlineMaxBudget)
}
if goexperiment.NewInliner {
postProcessCallSites(p)
var funcProps *inlheur.FuncProps
if goexperiment.NewInliner || inlheur.UnitTesting() {
- funcProps = inlheur.AnalyzeFunc(fn,
- func(fn *ir.Func) { CanInline(fn, profile) })
+ callCanInline := func(fn *ir.Func) { CanInline(fn, profile) }
+ funcProps = inlheur.AnalyzeFunc(fn, callCanInline, inlineMaxBudget)
}
var reason string // reason, if any, that the function was not inlined
visitor := hairyVisitor{
curFunc: fn,
+ isBigFunc: isBigFunc(fn),
budget: budget,
maxBudget: budget,
extraCallCost: cc,
type hairyVisitor struct {
// This is needed to access the current caller in the doNode function.
curFunc *ir.Func
+ isBigFunc bool
budget int32
maxBudget int32
reason string
//
// runtime.throw is a "cheap call" like panic in normal code.
var cheap bool
- if n.X.Op() == ir.ONAME {
- name := n.X.(*ir.Name)
+ if n.Fun.Op() == ir.ONAME {
+ name := n.Fun.(*ir.Name)
if name.Class == ir.PFUNC {
switch fn := types.RuntimeSymName(name.Sym()); fn {
case "getcallerpc", "getcallersp":
return false
}
}
- if n.X.Op() == ir.OMETHEXPR {
- if meth := ir.MethodExprName(n.X); meth != nil {
+ if n.Fun.Op() == ir.OMETHEXPR {
+ if meth := ir.MethodExprName(n.Fun); meth != nil {
if fn := meth.Func; fn != nil {
s := fn.Sym()
if types.RuntimeSymName(s) == "heapBits.nextArena" {
break // treat like any other node, that is, cost of 1
}
- // Determine if the callee edge is for an inlinable hot callee or not.
- if v.profile != nil && v.curFunc != nil {
- if fn := inlCallee(v.curFunc, n.X, v.profile); fn != nil && typecheck.HaveInlineBody(fn) {
- lineOffset := pgo.NodeLineOffset(n, fn)
- csi := pgo.CallSiteInfo{LineOffset: lineOffset, Caller: v.curFunc}
- if _, o := candHotEdgeMap[csi]; o {
- if base.Debug.PGODebug > 0 {
- fmt.Printf("hot-callsite identified at line=%v for func=%v\n", ir.Line(n), ir.PkgFuncName(v.curFunc))
- }
- }
- }
- }
-
if ir.IsIntrinsicCall(n) {
// Treat like any other node.
break
}
- if fn := inlCallee(v.curFunc, n.X, v.profile); fn != nil && typecheck.HaveInlineBody(fn) {
- // In the existing inliner, it makes sense to use fn.Inl.Cost
- // here due to the fact that an "inline F everywhere if F inlinable"
- // strategy is used. With the new inliner, however, it is not
- // a given that we'll inline a specific callsite -- it depends
- // on what score we assign to the callsite. For now, use the
- // computed cost if lower than the call cost, otherwise
- // use call cost (we can eventually do away with this when
- // we move to the "min-heap of callsites" scheme.
- if !goexperiment.NewInliner {
- v.budget -= fn.Inl.Cost
+ if callee := inlCallee(v.curFunc, n.Fun, v.profile); callee != nil && typecheck.HaveInlineBody(callee) {
+ // Check whether we'd actually inline this call. Set
+ // log == false since we aren't actually doing inlining
+ // yet.
+ if canInlineCallExpr(v.curFunc, n, callee, v.isBigFunc, false) {
+ // mkinlcall would inline this call [1], so use
+ // the cost of the inline body as the cost of
+ // the call, as that is what will actually
+ // appear in the code.
+ //
+ // [1] This is almost a perfect match to the
+ // mkinlcall logic, except that
+ // canInlineCallExpr considers inlining cycles
+ // by looking at what has already been inlined.
+ // Since we haven't done any inlining yet we
+ // will miss those.
+ v.budget -= callee.Inl.Cost
break
- } else {
- if fn.Inl.Cost < inlineExtraCallCost {
- v.budget -= fn.Inl.Cost
- break
- }
}
}
}
if base.Debug.DumpInlFuncProps != "" && !fn.Wrapper() {
inlheur.DumpFuncProps(fn, base.Debug.DumpInlFuncProps,
- func(fn *ir.Func) { CanInline(fn, profile) })
+ func(fn *ir.Func) { CanInline(fn, profile) }, inlineMaxBudget)
}
savefn := ir.CurFunc
ir.CurFunc = fn
base.FatalfAt(n.Pos(), "OCALLMETH missed by typecheck")
case ir.OCALLFUNC:
n := n.(*ir.CallExpr)
- if n.X.Op() == ir.OMETHEXPR {
+ if n.Fun.Op() == ir.OMETHEXPR {
// Prevent inlining some reflect.Value methods when using checkptr,
// even when package reflect was compiled without it (#35073).
- if meth := ir.MethodExprName(n.X); meth != nil {
+ if meth := ir.MethodExprName(n.Fun); meth != nil {
s := meth.Sym()
if base.Debug.Checkptr != 0 {
switch types.ReflectSymName(s) {
break
}
if base.Flag.LowerM > 3 {
- fmt.Printf("%v:call to func %+v\n", ir.Line(n), call.X)
+ fmt.Printf("%v:call to func %+v\n", ir.Line(n), call.Fun)
}
if ir.IsIntrinsicCall(call) {
break
}
- if fn := inlCallee(callerfn, call.X, profile); fn != nil && typecheck.HaveInlineBody(fn) {
+ if fn := inlCallee(callerfn, call.Fun, profile); fn != nil && typecheck.HaveInlineBody(fn) {
n = mkinlcall(callerfn, call, fn, bigCaller, inlCalls)
}
}
return false, inlineHotMaxBudget
}
+ if !base.PGOHash.MatchPosWithInfo(n.Pos(), "inline", nil) {
+ // De-selected by PGO Hash.
+ return false, maxCost
+ }
+
if base.Debug.PGODebug > 0 {
fmt.Printf("hot-budget check allows inlining for call %s (cost %d) at %v in function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller))
}
return true, 0
}
-// If n is a OCALLFUNC node, and fn is an ONAME node for a
-// function with an inlinable body, return an OINLCALL node that can replace n.
-// The returned node's Ninit has the parameter assignments, the Nbody is the
-// inlined function body, and (List, Rlist) contain the (input, output)
-// parameters.
-// The result of mkinlcall MUST be assigned back to n, e.g.
+// canInlineCallsite returns true if the call n from caller to callee can be
+// inlined. bigCaller indicates that caller is a big function. log indicates
+// that the 'cannot inline' reason should be logged.
//
-// n.Left = mkinlcall(n.Left, fn, isddd)
-func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller bool, inlCalls *[]*ir.InlinedCallExpr) ir.Node {
- if fn.Inl == nil {
- if logopt.Enabled() {
+// Preconditions: CanInline(callee) has already been called.
+func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCaller bool, log bool) bool {
+ if callee.Inl == nil {
+ // callee is never inlinable.
+ if log && logopt.Enabled() {
logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
- fmt.Sprintf("%s cannot be inlined", ir.PkgFuncName(fn)))
+ fmt.Sprintf("%s cannot be inlined", ir.PkgFuncName(callee)))
}
- return n
+ return false
}
- if ok, maxCost := inlineCostOK(n, callerfn, fn, bigCaller); !ok {
- if logopt.Enabled() {
+ if ok, maxCost := inlineCostOK(n, callerfn, callee, bigCaller); !ok {
+ // callee cost too high for this call site.
+ if log && logopt.Enabled() {
logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
- fmt.Sprintf("cost %d of %s exceeds max caller cost %d", fn.Inl.Cost, ir.PkgFuncName(fn), maxCost))
+ fmt.Sprintf("cost %d of %s exceeds max caller cost %d", callee.Inl.Cost, ir.PkgFuncName(callee), maxCost))
}
- return n
+ return false
}
- if fn == callerfn {
+ if callee == callerfn {
// Can't recursively inline a function into itself.
- if logopt.Enabled() {
+ if log && logopt.Enabled() {
logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to %s", ir.FuncName(callerfn)))
}
- return n
+ return false
}
- if base.Flag.Cfg.Instrumenting && types.IsNoInstrumentPkg(fn.Sym().Pkg) {
+ if base.Flag.Cfg.Instrumenting && types.IsNoInstrumentPkg(callee.Sym().Pkg) {
// Runtime package must not be instrumented.
// Instrument skips runtime package. However, some runtime code can be
// inlined into other packages and instrumented there. To avoid this,
// we disable inlining of runtime functions when instrumenting.
// The example that we observed is inlining of LockOSThread,
// which lead to false race reports on m contents.
- return n
- }
- if base.Flag.Race && types.IsNoRacePkg(fn.Sym().Pkg) {
- return n
+ if log && logopt.Enabled() {
+ logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
+ fmt.Sprintf("call to runtime function %s in instrumented build", ir.PkgFuncName(callee)))
+ }
+ return false
}
- parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()
- sym := fn.Linksym()
+ if base.Flag.Race && types.IsNoRacePkg(callee.Sym().Pkg) {
+ if log && logopt.Enabled() {
+ logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
+ fmt.Sprintf(`call to into "no-race" package function %s in race build`, ir.PkgFuncName(callee)))
+ }
+ return false
+ }
// Check if we've already inlined this function at this particular
// call site, in order to stop inlining when we reach the beginning
// many functions. Most likely, the inlining will stop before we
// even hit the beginning of the cycle again, but this catches the
// unusual case.
+ parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()
+ sym := callee.Linksym()
for inlIndex := parent; inlIndex >= 0; inlIndex = base.Ctxt.InlTree.Parent(inlIndex) {
if base.Ctxt.InlTree.InlinedFunction(inlIndex) == sym {
- if base.Flag.LowerM > 1 {
- fmt.Printf("%v: cannot inline %v into %v: repeated recursive cycle\n", ir.Line(n), fn, ir.FuncName(callerfn))
+ if log {
+ if base.Flag.LowerM > 1 {
+ fmt.Printf("%v: cannot inline %v into %v: repeated recursive cycle\n", ir.Line(n), callee, ir.FuncName(callerfn))
+ }
+ if logopt.Enabled() {
+ logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
+ fmt.Sprintf("repeated recursive cycle to %s", ir.PkgFuncName(callee)))
+ }
}
- return n
+ return false
}
}
+ return true
+}
+
+// If n is a OCALLFUNC node, and fn is an ONAME node for a
+// function with an inlinable body, return an OINLCALL node that can replace n.
+// The returned node's Ninit has the parameter assignments, the Nbody is the
+// inlined function body, and (List, Rlist) contain the (input, output)
+// parameters.
+// The result of mkinlcall MUST be assigned back to n, e.g.
+//
+// n.Left = mkinlcall(n.Left, fn, isddd)
+func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller bool, inlCalls *[]*ir.InlinedCallExpr) ir.Node {
+ if !canInlineCallExpr(callerfn, n, fn, bigCaller, true) {
+ return n
+ }
typecheck.AssertFixedCall(n)
+ parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()
+ sym := fn.Linksym()
inlIndex := base.Ctxt.InlTree.Add(parent, n.Pos(), sym, ir.FuncName(fn))
closureInitLSym := func(n *ir.CallExpr, fn *ir.Func) {
// Not a standard call.
return
}
- if n.X.Op() != ir.OCLOSURE {
+ if n.Fun.Op() != ir.OCLOSURE {
// Not a direct closure call.
return
}
- clo := n.X.(*ir.ClosureExpr)
+ clo := n.Fun.(*ir.ClosureExpr)
if ir.IsTrivialClosure(clo) {
// enqueueFunc will handle trivial closures anyways.
return
// determine whether it represents a call to sync/atomic.AddUint32 to
// increment a coverage counter.
func isAtomicCoverageCounterUpdate(cn *ir.CallExpr) bool {
- if cn.X.Op() != ir.ONAME {
+ if cn.Fun.Op() != ir.ONAME {
return false
}
- name := cn.X.(*ir.Name)
+ name := cn.Fun.(*ir.Name)
if name.Class != ir.PFUNC {
return false
}