import (
"fmt"
"go/constant"
- "sort"
+ "internal/goexperiment"
"strconv"
"cmd/compile/internal/base"
+ "cmd/compile/internal/inline/inlheur"
"cmd/compile/internal/ir"
"cmd/compile/internal/logopt"
"cmd/compile/internal/pgo"
)
// pgoInlinePrologue records the hot callsites from ir-graph.
-func pgoInlinePrologue(p *pgo.Profile, decls []ir.Node) {
+func pgoInlinePrologue(p *pgo.Profile, funcs []*ir.Func) {
if base.Debug.PGOInlineCDFThreshold != "" {
if s, err := strconv.ParseFloat(base.Debug.PGOInlineCDFThreshold, 64); err == nil && s >= 0 && s <= 100 {
inlineCDFHotCallSiteThresholdPercent = s
base.Fatalf("invalid PGOInlineCDFThreshold, must be between 0 and 100")
}
}
- var hotCallsites []pgo.NodeMapKey
+ var hotCallsites []pgo.NamedCallEdge
inlineHotCallSiteThresholdPercent, hotCallsites = hotNodesFromCDF(p)
if base.Debug.PGODebug > 0 {
fmt.Printf("hot-callsite-thres-from-CDF=%v\n", inlineHotCallSiteThresholdPercent)
candHotCalleeMap[callee] = struct{}{}
}
// mark hot call sites
- if caller := p.WeightedCG.IRNodes[n.CallerName]; caller != nil {
+ if caller := p.WeightedCG.IRNodes[n.CallerName]; caller != nil && caller.AST != nil {
csi := pgo.CallSiteInfo{LineOffset: n.CallSiteOffset, Caller: caller.AST}
candHotEdgeMap[csi] = struct{}{}
}
}
- if base.Debug.PGODebug >= 2 {
+ if base.Debug.PGODebug >= 3 {
fmt.Printf("hot-cg before inline in dot format:")
p.PrintWeightedCallGraphDOT(inlineHotCallSiteThresholdPercent)
}
// (currently only used in debug prints) (in case of equal weights,
// comparing with the threshold may not accurately reflect which nodes are
// considiered hot).
-func hotNodesFromCDF(p *pgo.Profile) (float64, []pgo.NodeMapKey) {
- nodes := make([]pgo.NodeMapKey, len(p.NodeMap))
- i := 0
- for n := range p.NodeMap {
- nodes[i] = n
- i++
- }
- sort.Slice(nodes, func(i, j int) bool {
- ni, nj := nodes[i], nodes[j]
- if wi, wj := p.NodeMap[ni].EWeight, p.NodeMap[nj].EWeight; wi != wj {
- return wi > wj // want larger weight first
- }
- // same weight, order by name/line number
- if ni.CallerName != nj.CallerName {
- return ni.CallerName < nj.CallerName
- }
- if ni.CalleeName != nj.CalleeName {
- return ni.CalleeName < nj.CalleeName
- }
- return ni.CallSiteOffset < nj.CallSiteOffset
- })
+func hotNodesFromCDF(p *pgo.Profile) (float64, []pgo.NamedCallEdge) {
cum := int64(0)
- for i, n := range nodes {
- w := p.NodeMap[n].EWeight
+ for i, n := range p.NamedEdgeMap.ByWeight {
+ w := p.NamedEdgeMap.Weight[n]
cum += w
- if pgo.WeightInPercentage(cum, p.TotalEdgeWeight) > inlineCDFHotCallSiteThresholdPercent {
+ if pgo.WeightInPercentage(cum, p.TotalWeight) > inlineCDFHotCallSiteThresholdPercent {
// nodes[:i+1] to include the very last node that makes it to go over the threshold.
// (Say, if the CDF threshold is 50% and one hot node takes 60% of weight, we want to
// include that node instead of excluding it.)
- return pgo.WeightInPercentage(w, p.TotalEdgeWeight), nodes[:i+1]
+ return pgo.WeightInPercentage(w, p.TotalWeight), p.NamedEdgeMap.ByWeight[:i+1]
}
}
- return 0, nodes
+ return 0, p.NamedEdgeMap.ByWeight
}
// InlinePackage finds functions that can be inlined and clones them before walk expands them.
p = nil
}
- InlineDecls(p, typecheck.Target.Decls, true)
+ InlineDecls(p, typecheck.Target.Funcs, true)
// Perform a garbage collection of hidden closures functions that
// are no longer reachable from top-level functions following
// inlining. See #59404 and #59638 for more context.
garbageCollectUnreferencedHiddenClosures()
+
+ if base.Debug.DumpInlFuncProps != "" {
+ inlheur.DumpFuncProps(nil, base.Debug.DumpInlFuncProps, nil, inlineMaxBudget)
+ }
+ if goexperiment.NewInliner {
+ postProcessCallSites(p)
+ }
}
// InlineDecls applies inlining to the given batch of declarations.
-func InlineDecls(p *pgo.Profile, decls []ir.Node, doInline bool) {
+func InlineDecls(p *pgo.Profile, funcs []*ir.Func, doInline bool) {
if p != nil {
- pgoInlinePrologue(p, decls)
+ pgoInlinePrologue(p, funcs)
}
doCanInline := func(n *ir.Func, recursive bool, numfns int) {
}
}
- ir.VisitFuncsBottomUp(decls, func(list []*ir.Func, recursive bool) {
+ ir.VisitFuncsBottomUp(funcs, func(list []*ir.Func, recursive bool) {
numfns := numNonClosures(list)
// We visit functions within an SCC in fairly arbitrary order,
// so by computing inlinability for all functions in the SCC
})
}
- for i := 0; i < len(typecheck.Target.Decls); i++ {
- if fn, ok := typecheck.Target.Decls[i].(*ir.Func); ok {
- if fn.IsHiddenClosure() {
- continue
- }
- markLiveFuncs(fn)
+ for i := 0; i < len(typecheck.Target.Funcs); i++ {
+ fn := typecheck.Target.Funcs[i]
+ if fn.IsHiddenClosure() {
+ continue
}
+ markLiveFuncs(fn)
}
- for i := 0; i < len(typecheck.Target.Decls); i++ {
- if fn, ok := typecheck.Target.Decls[i].(*ir.Func); ok {
- if !fn.IsHiddenClosure() {
- continue
- }
- if fn.IsDeadcodeClosure() {
- continue
- }
- if liveFuncs[fn] {
- continue
- }
- fn.SetIsDeadcodeClosure(true)
- if base.Flag.LowerM > 2 {
- fmt.Printf("%v: unreferenced closure %v marked as dead\n", ir.Line(fn), fn)
- }
- if fn.Inl != nil && fn.LSym == nil {
- ir.InitLSym(fn, true)
+ for i := 0; i < len(typecheck.Target.Funcs); i++ {
+ fn := typecheck.Target.Funcs[i]
+ if !fn.IsHiddenClosure() {
+ continue
+ }
+ if fn.IsDeadcodeClosure() {
+ continue
+ }
+ if liveFuncs[fn] {
+ continue
+ }
+ fn.SetIsDeadcodeClosure(true)
+ if base.Flag.LowerM > 2 {
+ fmt.Printf("%v: unreferenced closure %v marked as dead\n", ir.Line(fn), fn)
+ }
+ if fn.Inl != nil && fn.LSym == nil {
+ ir.InitLSym(fn, true)
+ }
+ }
+}
+
+// inlineBudget determines the max budget for function 'fn' prior to
+// analyzing the hairyness of the body of 'fn'. We pass in the pgo
+// profile if available (which can change the budget), also a
+// 'relaxed' flag, which expands the budget slightly to allow for the
+// possibility that a call to the function might have its score
+// adjusted downwards. If 'verbose' is set, then print a remark where
+// we boost the budget due to PGO.
+func inlineBudget(fn *ir.Func, profile *pgo.Profile, relaxed bool, verbose bool) int32 {
+ // Update the budget for profile-guided inlining.
+ budget := int32(inlineMaxBudget)
+ if profile != nil {
+ if n, ok := profile.WeightedCG.IRNodes[ir.LinkFuncName(fn)]; ok {
+ if _, ok := candHotCalleeMap[n]; ok {
+ budget = int32(inlineHotMaxBudget)
+ if verbose {
+ fmt.Printf("hot-node enabled increased budget=%v for func=%v\n", budget, ir.PkgFuncName(fn))
+ }
}
}
}
+ if relaxed {
+ budget += inlineMaxBudget
+ }
+ return budget
}
// CanInline determines whether fn is inlineable.
base.Fatalf("CanInline no nname %+v", fn)
}
+ var funcProps *inlheur.FuncProps
+ if goexperiment.NewInliner || inlheur.UnitTesting() {
+ callCanInline := func(fn *ir.Func) { CanInline(fn, profile) }
+ funcProps = inlheur.AnalyzeFunc(fn, callCanInline, inlineMaxBudget)
+ }
+
var reason string // reason, if any, that the function was not inlined
if base.Flag.LowerM > 1 || logopt.Enabled() {
defer func() {
}()
}
- // If marked "go:noinline", don't inline
- if fn.Pragma&ir.Noinline != 0 {
- reason = "marked go:noinline"
- return
- }
-
- // If marked "go:norace" and -race compilation, don't inline.
- if base.Flag.Race && fn.Pragma&ir.Norace != 0 {
- reason = "marked go:norace with -race compilation"
- return
- }
-
- // If marked "go:nocheckptr" and -d checkptr compilation, don't inline.
- if base.Debug.Checkptr != 0 && fn.Pragma&ir.NoCheckPtr != 0 {
- reason = "marked go:nocheckptr"
- return
- }
-
- // If marked "go:cgo_unsafe_args", don't inline, since the
- // function makes assumptions about its argument frame layout.
- if fn.Pragma&ir.CgoUnsafeArgs != 0 {
- reason = "marked go:cgo_unsafe_args"
+ reason = InlineImpossible(fn)
+ if reason != "" {
return
}
-
- // If marked as "go:uintptrkeepalive", don't inline, since the
- // keep alive information is lost during inlining.
- //
- // TODO(prattmic): This is handled on calls during escape analysis,
- // which is after inlining. Move prior to inlining so the keep-alive is
- // maintained after inlining.
- if fn.Pragma&ir.UintptrKeepAlive != 0 {
- reason = "marked as having a keep-alive uintptr argument"
- return
- }
-
- // If marked as "go:uintptrescapes", don't inline, since the
- // escape information is lost during inlining.
- if fn.Pragma&ir.UintptrEscapes != 0 {
- reason = "marked as having an escaping uintptr argument"
- return
- }
-
- // The nowritebarrierrec checker currently works at function
- // granularity, so inlining yeswritebarrierrec functions can
- // confuse it (#22342). As a workaround, disallow inlining
- // them for now.
- if fn.Pragma&ir.Yeswritebarrierrec != 0 {
- reason = "marked go:yeswritebarrierrec"
- return
- }
-
- // If fn has no body (is defined outside of Go), cannot inline it.
- if len(fn.Body) == 0 {
- reason = "no function body"
- return
- }
-
- // If fn is synthetic hash or eq function, cannot inline it.
- // The function is not generated in Unified IR frontend at this moment.
- if ir.IsEqOrHashFunc(fn) {
- reason = "type eq/hash function"
- return
- }
-
if fn.Typecheck() == 0 {
base.Fatalf("CanInline on non-typechecked function %v", fn)
}
cc = 1 // this appears to yield better performance than 0.
}
- // Update the budget for profile-guided inlining.
- budget := int32(inlineMaxBudget)
- if profile != nil {
- if n, ok := profile.WeightedCG.IRNodes[ir.LinkFuncName(fn)]; ok {
- if _, ok := candHotCalleeMap[n]; ok {
- budget = int32(inlineHotMaxBudget)
- if base.Debug.PGODebug > 0 {
- fmt.Printf("hot-node enabled increased budget=%v for func=%v\n", budget, ir.PkgFuncName(fn))
- }
- }
- }
- }
+ // Used a "relaxed" inline budget if goexperiment.NewInliner is in
+ // effect, or if we're producing a debugging dump.
+ relaxed := goexperiment.NewInliner ||
+ (base.Debug.DumpInlFuncProps != "" ||
+ base.Debug.DumpInlCallSiteScores != 0)
+
+ // Compute the inline budget for this func.
+ budget := inlineBudget(fn, profile, relaxed, base.Debug.PGODebug > 0)
// At this point in the game the function we're looking at may
// have "stale" autos, vars that still appear in the Dcl list, but
// when creating the "Inline.Dcl" field below; to accomplish this,
// the hairyVisitor below builds up a map of used/referenced
// locals, and we use this map to produce a pruned Inline.Dcl
- // list. See issue 25249 for more context.
+ // list. See issue 25459 for more context.
visitor := hairyVisitor{
curFunc: fn,
+ isBigFunc: isBigFunc(fn),
budget: budget,
maxBudget: budget,
extraCallCost: cc,
}
n.Func.Inl = &ir.Inline{
- Cost: budget - visitor.budget,
- Dcl: pruneUnusedAutos(n.Defn.(*ir.Func).Dcl, &visitor),
- Body: inlcopylist(fn.Body),
+ Cost: budget - visitor.budget,
+ Dcl: pruneUnusedAutos(n.Func.Dcl, &visitor),
+ HaveDcl: true,
CanDelayResults: canDelayResults(fn),
}
+ if goexperiment.NewInliner {
+ n.Func.Inl.Properties = funcProps.SerializeToString()
+ }
if base.Flag.LowerM > 1 {
- fmt.Printf("%v: can inline %v with cost %d as: %v { %v }\n", ir.Line(fn), n, budget-visitor.budget, fn.Type(), ir.Nodes(n.Func.Inl.Body))
+ fmt.Printf("%v: can inline %v with cost %d as: %v { %v }\n", ir.Line(fn), n, budget-visitor.budget, fn.Type(), ir.Nodes(fn.Body))
} else if base.Flag.LowerM != 0 {
fmt.Printf("%v: can inline %v\n", ir.Line(fn), n)
}
}
}
+// InlineImpossible returns a non-empty reason string if fn is impossible to
+// inline regardless of cost or contents.
+func InlineImpossible(fn *ir.Func) string {
+ var reason string // reason, if any, that the function can not be inlined.
+ if fn.Nname == nil {
+ reason = "no name"
+ return reason
+ }
+
+ // If marked "go:noinline", don't inline.
+ if fn.Pragma&ir.Noinline != 0 {
+ reason = "marked go:noinline"
+ return reason
+ }
+
+ // If marked "go:norace" and -race compilation, don't inline.
+ if base.Flag.Race && fn.Pragma&ir.Norace != 0 {
+ reason = "marked go:norace with -race compilation"
+ return reason
+ }
+
+ // If marked "go:nocheckptr" and -d checkptr compilation, don't inline.
+ if base.Debug.Checkptr != 0 && fn.Pragma&ir.NoCheckPtr != 0 {
+ reason = "marked go:nocheckptr"
+ return reason
+ }
+
+ // If marked "go:cgo_unsafe_args", don't inline, since the function
+ // makes assumptions about its argument frame layout.
+ if fn.Pragma&ir.CgoUnsafeArgs != 0 {
+ reason = "marked go:cgo_unsafe_args"
+ return reason
+ }
+
+ // If marked as "go:uintptrkeepalive", don't inline, since the keep
+ // alive information is lost during inlining.
+ //
+ // TODO(prattmic): This is handled on calls during escape analysis,
+ // which is after inlining. Move prior to inlining so the keep-alive is
+ // maintained after inlining.
+ if fn.Pragma&ir.UintptrKeepAlive != 0 {
+ reason = "marked as having a keep-alive uintptr argument"
+ return reason
+ }
+
+ // If marked as "go:uintptrescapes", don't inline, since the escape
+ // information is lost during inlining.
+ if fn.Pragma&ir.UintptrEscapes != 0 {
+ reason = "marked as having an escaping uintptr argument"
+ return reason
+ }
+
+ // The nowritebarrierrec checker currently works at function
+ // granularity, so inlining yeswritebarrierrec functions can confuse it
+ // (#22342). As a workaround, disallow inlining them for now.
+ if fn.Pragma&ir.Yeswritebarrierrec != 0 {
+ reason = "marked go:yeswritebarrierrec"
+ return reason
+ }
+
+ // If a local function has no fn.Body (is defined outside of Go), cannot inline it.
+ // Imported functions don't have fn.Body but might have inline body in fn.Inl.
+ if len(fn.Body) == 0 && !typecheck.HaveInlineBody(fn) {
+ reason = "no function body"
+ return reason
+ }
+
+ return ""
+}
+
// canDelayResults reports whether inlined calls to fn can delay
// declaring the result parameter until the "return" statement.
func canDelayResults(fn *ir.Func) bool {
}
// temporaries for return values.
- for _, param := range fn.Type().Results().FieldSlice() {
- if sym := types.OrigSym(param.Sym); sym != nil && !sym.IsBlank() {
+ for _, param := range fn.Type().Results() {
+ if sym := param.Sym; sym != nil && !sym.IsBlank() {
return false // found a named result parameter (case 3)
}
}
type hairyVisitor struct {
// This is needed to access the current caller in the doNode function.
curFunc *ir.Func
+ isBigFunc bool
budget int32
maxBudget int32
reason string
if n == nil {
return false
}
+opSwitch:
switch n.Op() {
// Call is okay if inlinable and we have the budget for the body.
case ir.OCALLFUNC:
//
// runtime.throw is a "cheap call" like panic in normal code.
var cheap bool
- if n.X.Op() == ir.ONAME {
- name := n.X.(*ir.Name)
- if name.Class == ir.PFUNC && types.IsRuntimePkg(name.Sym().Pkg) {
- fn := name.Sym().Name
- if fn == "getcallerpc" || fn == "getcallersp" {
+ if n.Fun.Op() == ir.ONAME {
+ name := n.Fun.(*ir.Name)
+ if name.Class == ir.PFUNC {
+ switch fn := types.RuntimeSymName(name.Sym()); fn {
+ case "getcallerpc", "getcallersp":
v.reason = "call to " + fn
return true
- }
- if fn == "throw" {
+ case "throw":
v.budget -= inlineExtraThrowCost
- break
+ break opSwitch
}
- }
- // Special case for reflect.noescpae. It does just type
- // conversions to appease the escape analysis, and doesn't
- // generate code.
- if name.Class == ir.PFUNC && types.IsReflectPkg(name.Sym().Pkg) {
- if name.Sym().Name == "noescape" {
+ // Special case for reflect.noescape. It does just type
+ // conversions to appease the escape analysis, and doesn't
+ // generate code.
+ if types.ReflectSymName(name.Sym()) == "noescape" {
cheap = true
}
}
return false
}
}
- if n.X.Op() == ir.OMETHEXPR {
- if meth := ir.MethodExprName(n.X); meth != nil {
+ if n.Fun.Op() == ir.OMETHEXPR {
+ if meth := ir.MethodExprName(n.Fun); meth != nil {
if fn := meth.Func; fn != nil {
s := fn.Sym()
- if types.IsRuntimePkg(s.Pkg) && s.Name == "heapBits.nextArena" {
+ if types.RuntimeSymName(s) == "heapBits.nextArena" {
// Special case: explicitly allow mid-stack inlining of
// runtime.heapBits.next even though it calls slow-path
// runtime.heapBits.nextArena.
break // treat like any other node, that is, cost of 1
}
- // Determine if the callee edge is for an inlinable hot callee or not.
- if v.profile != nil && v.curFunc != nil {
- if fn := inlCallee(n.X, v.profile); fn != nil && typecheck.HaveInlineBody(fn) {
- lineOffset := pgo.NodeLineOffset(n, fn)
- csi := pgo.CallSiteInfo{LineOffset: lineOffset, Caller: v.curFunc}
- if _, o := candHotEdgeMap[csi]; o {
- if base.Debug.PGODebug > 0 {
- fmt.Printf("hot-callsite identified at line=%v for func=%v\n", ir.Line(n), ir.PkgFuncName(v.curFunc))
- }
- }
- }
- }
-
if ir.IsIntrinsicCall(n) {
// Treat like any other node.
break
}
- if fn := inlCallee(n.X, v.profile); fn != nil && typecheck.HaveInlineBody(fn) {
- v.budget -= fn.Inl.Cost
- break
+ if callee := inlCallee(v.curFunc, n.Fun, v.profile); callee != nil && typecheck.HaveInlineBody(callee) {
+ // Check whether we'd actually inline this call. Set
+ // log == false since we aren't actually doing inlining
+ // yet.
+ if canInlineCallExpr(v.curFunc, n, callee, v.isBigFunc, false) {
+ // mkinlcall would inline this call [1], so use
+ // the cost of the inline body as the cost of
+ // the call, as that is what will actually
+ // appear in the code.
+ //
+ // [1] This is almost a perfect match to the
+ // mkinlcall logic, except that
+ // canInlineCallExpr considers inlining cycles
+ // by looking at what has already been inlined.
+ // Since we haven't done any inlining yet we
+ // will miss those.
+ v.budget -= callee.Inl.Cost
+ break
+ }
}
// Call cost for non-leaf inlining.
v.budget -= inlineExtraPanicCost
case ir.ORECOVER:
+ base.FatalfAt(n.Pos(), "ORECOVER missed typecheck")
+ case ir.ORECOVERFP:
// recover matches the argument frame pointer to find
// the right panic value, so it needs an argument frame.
v.reason = "call to recover"
// should try to account for that if we're going to account for captures.
v.budget -= 15
- case ir.OGO,
- ir.ODEFER,
- ir.ODCLTYPE, // can't print yet
- ir.OTAILCALL:
+ case ir.OGO, ir.ODEFER, ir.OTAILCALL:
v.reason = "unhandled op " + n.Op().String()
return true
// This doesn't produce code, but the children might.
v.budget++ // undo default cost
- case ir.ODCLCONST, ir.OFALL, ir.OTYPE:
+ case ir.OFALL, ir.OTYPE:
// These nodes don't produce code; omit from inlining budget.
return false
})
}
-// inlcopylist (together with inlcopy) recursively copies a list of nodes, except
-// that it keeps the same ONAME, OTYPE, and OLITERAL nodes. It is used for copying
-// the body and dcls of an inlineable function.
-func inlcopylist(ll []ir.Node) []ir.Node {
- s := make([]ir.Node, len(ll))
- for i, n := range ll {
- s[i] = inlcopy(n)
- }
- return s
-}
-
-// inlcopy is like DeepCopy(), but does extra work to copy closures.
-func inlcopy(n ir.Node) ir.Node {
- var edit func(ir.Node) ir.Node
- edit = func(x ir.Node) ir.Node {
- switch x.Op() {
- case ir.ONAME, ir.OTYPE, ir.OLITERAL, ir.ONIL:
- return x
- }
- m := ir.Copy(x)
- ir.EditChildren(m, edit)
- if x.Op() == ir.OCLOSURE {
- x := x.(*ir.ClosureExpr)
- // Need to save/duplicate x.Func.Nname,
- // x.Func.Nname.Ntype, x.Func.Dcl, x.Func.ClosureVars, and
- // x.Func.Body for iexport and local inlining.
- oldfn := x.Func
- newfn := ir.NewFunc(oldfn.Pos())
- m.(*ir.ClosureExpr).Func = newfn
- newfn.Nname = ir.NewNameAt(oldfn.Nname.Pos(), oldfn.Nname.Sym())
- // XXX OK to share fn.Type() ??
- newfn.Nname.SetType(oldfn.Nname.Type())
- newfn.Body = inlcopylist(oldfn.Body)
- // Make shallow copy of the Dcl and ClosureVar slices
- newfn.Dcl = append([]*ir.Name(nil), oldfn.Dcl...)
- newfn.ClosureVars = append([]*ir.Name(nil), oldfn.ClosureVars...)
- }
- return m
- }
- return edit(n)
-}
-
// InlineCalls/inlnode walks fn's statements and expressions and substitutes any
// calls made to inlineable functions. This is the external entry point.
func InlineCalls(fn *ir.Func, profile *pgo.Profile) {
+ if goexperiment.NewInliner && !fn.Wrapper() {
+ inlheur.ScoreCalls(fn)
+ }
+ if base.Debug.DumpInlFuncProps != "" && !fn.Wrapper() {
+ inlheur.DumpFuncProps(fn, base.Debug.DumpInlFuncProps,
+ func(fn *ir.Func) { CanInline(fn, profile) }, inlineMaxBudget)
+ }
savefn := ir.CurFunc
ir.CurFunc = fn
bigCaller := isBigFunc(fn)
var inlCalls []*ir.InlinedCallExpr
var edit func(ir.Node) ir.Node
edit = func(n ir.Node) ir.Node {
- return inlnode(n, bigCaller, &inlCalls, edit, profile)
+ return inlnode(fn, n, bigCaller, &inlCalls, edit, profile)
}
ir.EditChildren(fn, edit)
// The result of inlnode MUST be assigned back to n, e.g.
//
// n.Left = inlnode(n.Left)
-func inlnode(n ir.Node, bigCaller bool, inlCalls *[]*ir.InlinedCallExpr, edit func(ir.Node) ir.Node, profile *pgo.Profile) ir.Node {
+func inlnode(callerfn *ir.Func, n ir.Node, bigCaller bool, inlCalls *[]*ir.InlinedCallExpr, edit func(ir.Node) ir.Node, profile *pgo.Profile) ir.Node {
if n == nil {
return n
}
base.FatalfAt(n.Pos(), "OCALLMETH missed by typecheck")
case ir.OCALLFUNC:
n := n.(*ir.CallExpr)
- if n.X.Op() == ir.OMETHEXPR {
+ if n.Fun.Op() == ir.OMETHEXPR {
// Prevent inlining some reflect.Value methods when using checkptr,
// even when package reflect was compiled without it (#35073).
- if meth := ir.MethodExprName(n.X); meth != nil {
+ if meth := ir.MethodExprName(n.Fun); meth != nil {
s := meth.Sym()
- if base.Debug.Checkptr != 0 && types.IsReflectPkg(s.Pkg) && (s.Name == "Value.UnsafeAddr" || s.Name == "Value.Pointer") {
- return n
+ if base.Debug.Checkptr != 0 {
+ switch types.ReflectSymName(s) {
+ case "Value.UnsafeAddr", "Value.Pointer":
+ return n
+ }
}
}
}
break
}
if base.Flag.LowerM > 3 {
- fmt.Printf("%v:call to func %+v\n", ir.Line(n), call.X)
+ fmt.Printf("%v:call to func %+v\n", ir.Line(n), call.Fun)
}
if ir.IsIntrinsicCall(call) {
break
}
- if fn := inlCallee(call.X, profile); fn != nil && typecheck.HaveInlineBody(fn) {
- n = mkinlcall(call, fn, bigCaller, inlCalls)
+ if fn := inlCallee(callerfn, call.Fun, profile); fn != nil && typecheck.HaveInlineBody(fn) {
+ n = mkinlcall(callerfn, call, fn, bigCaller, inlCalls)
}
}
// inlCallee takes a function-typed expression and returns the underlying function ONAME
// that it refers to if statically known. Otherwise, it returns nil.
-func inlCallee(fn ir.Node, profile *pgo.Profile) *ir.Func {
+func inlCallee(caller *ir.Func, fn ir.Node, profile *pgo.Profile) (res *ir.Func) {
fn = ir.StaticValue(fn)
switch fn.Op() {
case ir.OMETHEXPR:
case ir.OCLOSURE:
fn := fn.(*ir.ClosureExpr)
c := fn.Func
+ if len(c.ClosureVars) != 0 && c.ClosureVars[0].Outer.Curfn != caller {
+ return nil // inliner doesn't support inlining across closure frames
+ }
CanInline(c, profile)
return c
}
// InlineCall allows the inliner implementation to be overridden.
// If it returns nil, the function will not be inlined.
-var InlineCall = func(call *ir.CallExpr, fn *ir.Func, inlIndex int) *ir.InlinedCallExpr {
+var InlineCall = func(callerfn *ir.Func, call *ir.CallExpr, fn *ir.Func, inlIndex int) *ir.InlinedCallExpr {
base.Fatalf("inline.InlineCall not overridden")
panic("unreachable")
}
maxCost = inlineBigFunctionMaxCost
}
- if callee.Inl.Cost <= maxCost {
+ metric := callee.Inl.Cost
+ if goexperiment.NewInliner {
+ ok, score := inlheur.GetCallSiteScore(n)
+ if ok {
+ metric = int32(score)
+ }
+
+ }
+
+ if metric <= maxCost {
// Simple case. Function is already cheap enough.
return true, 0
}
return false, maxCost
}
- if callee.Inl.Cost > inlineHotMaxBudget {
+ if metric > inlineHotMaxBudget {
return false, inlineHotMaxBudget
}
+ if !base.PGOHash.MatchPosWithInfo(n.Pos(), "inline", nil) {
+ // De-selected by PGO Hash.
+ return false, maxCost
+ }
+
if base.Debug.PGODebug > 0 {
fmt.Printf("hot-budget check allows inlining for call %s (cost %d) at %v in function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller))
}
return true, 0
}
-// If n is a OCALLFUNC node, and fn is an ONAME node for a
-// function with an inlinable body, return an OINLCALL node that can replace n.
-// The returned node's Ninit has the parameter assignments, the Nbody is the
-// inlined function body, and (List, Rlist) contain the (input, output)
-// parameters.
-// The result of mkinlcall MUST be assigned back to n, e.g.
+// canInlineCallsite returns true if the call n from caller to callee can be
+// inlined. bigCaller indicates that caller is a big function. log indicates
+// that the 'cannot inline' reason should be logged.
//
-// n.Left = mkinlcall(n.Left, fn, isddd)
-func mkinlcall(n *ir.CallExpr, fn *ir.Func, bigCaller bool, inlCalls *[]*ir.InlinedCallExpr) ir.Node {
- if fn.Inl == nil {
- if logopt.Enabled() {
- logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(ir.CurFunc),
- fmt.Sprintf("%s cannot be inlined", ir.PkgFuncName(fn)))
+// Preconditions: CanInline(callee) has already been called.
+func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCaller bool, log bool) bool {
+ if callee.Inl == nil {
+ // callee is never inlinable.
+ if log && logopt.Enabled() {
+ logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
+ fmt.Sprintf("%s cannot be inlined", ir.PkgFuncName(callee)))
}
- return n
+ return false
}
- if ok, maxCost := inlineCostOK(n, ir.CurFunc, fn, bigCaller); !ok {
- if logopt.Enabled() {
- logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(ir.CurFunc),
- fmt.Sprintf("cost %d of %s exceeds max caller cost %d", fn.Inl.Cost, ir.PkgFuncName(fn), maxCost))
+ if ok, maxCost := inlineCostOK(n, callerfn, callee, bigCaller); !ok {
+ // callee cost too high for this call site.
+ if log && logopt.Enabled() {
+ logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
+ fmt.Sprintf("cost %d of %s exceeds max caller cost %d", callee.Inl.Cost, ir.PkgFuncName(callee), maxCost))
}
- return n
+ return false
}
- if fn == ir.CurFunc {
+ if callee == callerfn {
// Can't recursively inline a function into itself.
- if logopt.Enabled() {
- logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to %s", ir.FuncName(ir.CurFunc)))
+ if log && logopt.Enabled() {
+ logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to %s", ir.FuncName(callerfn)))
}
- return n
+ return false
}
- if base.Flag.Cfg.Instrumenting && types.IsNoInstrumentPkg(fn.Sym().Pkg) {
+ if base.Flag.Cfg.Instrumenting && types.IsNoInstrumentPkg(callee.Sym().Pkg) {
// Runtime package must not be instrumented.
// Instrument skips runtime package. However, some runtime code can be
// inlined into other packages and instrumented there. To avoid this,
// we disable inlining of runtime functions when instrumenting.
// The example that we observed is inlining of LockOSThread,
// which lead to false race reports on m contents.
- return n
- }
- if base.Flag.Race && types.IsNoRacePkg(fn.Sym().Pkg) {
- return n
+ if log && logopt.Enabled() {
+ logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
+ fmt.Sprintf("call to runtime function %s in instrumented build", ir.PkgFuncName(callee)))
+ }
+ return false
}
- parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()
- sym := fn.Linksym()
+ if base.Flag.Race && types.IsNoRacePkg(callee.Sym().Pkg) {
+ if log && logopt.Enabled() {
+ logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
+ fmt.Sprintf(`call to into "no-race" package function %s in race build`, ir.PkgFuncName(callee)))
+ }
+ return false
+ }
// Check if we've already inlined this function at this particular
// call site, in order to stop inlining when we reach the beginning
// many functions. Most likely, the inlining will stop before we
// even hit the beginning of the cycle again, but this catches the
// unusual case.
+ parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()
+ sym := callee.Linksym()
for inlIndex := parent; inlIndex >= 0; inlIndex = base.Ctxt.InlTree.Parent(inlIndex) {
if base.Ctxt.InlTree.InlinedFunction(inlIndex) == sym {
- if base.Flag.LowerM > 1 {
- fmt.Printf("%v: cannot inline %v into %v: repeated recursive cycle\n", ir.Line(n), fn, ir.FuncName(ir.CurFunc))
+ if log {
+ if base.Flag.LowerM > 1 {
+ fmt.Printf("%v: cannot inline %v into %v: repeated recursive cycle\n", ir.Line(n), callee, ir.FuncName(callerfn))
+ }
+ if logopt.Enabled() {
+ logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
+ fmt.Sprintf("repeated recursive cycle to %s", ir.PkgFuncName(callee)))
+ }
}
- return n
+ return false
}
}
+ return true
+}
+
+// If n is a OCALLFUNC node, and fn is an ONAME node for a
+// function with an inlinable body, return an OINLCALL node that can replace n.
+// The returned node's Ninit has the parameter assignments, the Nbody is the
+// inlined function body, and (List, Rlist) contain the (input, output)
+// parameters.
+// The result of mkinlcall MUST be assigned back to n, e.g.
+//
+// n.Left = mkinlcall(n.Left, fn, isddd)
+func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller bool, inlCalls *[]*ir.InlinedCallExpr) ir.Node {
+ if !canInlineCallExpr(callerfn, n, fn, bigCaller, true) {
+ return n
+ }
typecheck.AssertFixedCall(n)
- inlIndex := base.Ctxt.InlTree.Add(parent, n.Pos(), sym)
+ parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()
+ sym := fn.Linksym()
+ inlIndex := base.Ctxt.InlTree.Add(parent, n.Pos(), sym, ir.FuncName(fn))
closureInitLSym := func(n *ir.CallExpr, fn *ir.Func) {
// The linker needs FuncInfo metadata for all inlined
// Not a standard call.
return
}
- if n.X.Op() != ir.OCLOSURE {
+ if n.Fun.Op() != ir.OCLOSURE {
// Not a direct closure call.
return
}
- clo := n.X.(*ir.ClosureExpr)
+ clo := n.Fun.(*ir.ClosureExpr)
if ir.IsTrivialClosure(clo) {
// enqueueFunc will handle trivial closures anyways.
return
fmt.Printf("%v: Before inlining: %+v\n", ir.Line(n), n)
}
- res := InlineCall(n, fn, inlIndex)
+ res := InlineCall(callerfn, n, fn, inlIndex)
if res == nil {
base.FatalfAt(n.Pos(), "inlining call to %v failed", fn)
for _, n := range ll {
if n.Class == ir.PAUTO {
if !vis.usedLocals.Has(n) {
+ // TODO(mdempsky): Simplify code after confident that this
+ // never happens anymore.
+ base.FatalfAt(n.Pos(), "unused auto: %v", n)
continue
}
}
// determine whether it represents a call to sync/atomic.AddUint32 to
// increment a coverage counter.
func isAtomicCoverageCounterUpdate(cn *ir.CallExpr) bool {
- if cn.X.Op() != ir.ONAME {
+ if cn.Fun.Op() != ir.ONAME {
return false
}
- name := cn.X.(*ir.Name)
+ name := cn.Fun.(*ir.Name)
if name.Class != ir.PFUNC {
return false
}
v := isIndexingCoverageCounter(adn.X)
return v
}
+
+func postProcessCallSites(profile *pgo.Profile) {
+ if base.Debug.DumpInlCallSiteScores != 0 {
+ budgetCallback := func(fn *ir.Func, prof *pgo.Profile) (int32, bool) {
+ v := inlineBudget(fn, prof, false, false)
+ return v, v == inlineHotMaxBudget
+ }
+ inlheur.DumpInlCallSiteScores(profile, budgetCallback)
+ }
+}