import (
"fmt"
"go/constant"
- "sort"
+ "internal/goexperiment"
"strconv"
"cmd/compile/internal/base"
+ "cmd/compile/internal/inline/inlheur"
"cmd/compile/internal/ir"
"cmd/compile/internal/logopt"
"cmd/compile/internal/pgo"
// TODO(prattmic): Make this non-global.
candHotEdgeMap = make(map[pgo.CallSiteInfo]struct{})
- // List of inlined call sites. CallSiteInfo.Callee is always nil.
- // TODO(prattmic): Make this non-global.
- inlinedCallSites = make(map[pgo.CallSiteInfo]struct{})
-
// Threshold in percentage for hot callsite inlining.
inlineHotCallSiteThresholdPercent float64
)
// pgoInlinePrologue records the hot callsites from ir-graph.
-func pgoInlinePrologue(p *pgo.Profile, decls []ir.Node) {
+func pgoInlinePrologue(p *pgo.Profile, funcs []*ir.Func) {
if base.Debug.PGOInlineCDFThreshold != "" {
if s, err := strconv.ParseFloat(base.Debug.PGOInlineCDFThreshold, 64); err == nil && s >= 0 && s <= 100 {
inlineCDFHotCallSiteThresholdPercent = s
base.Fatalf("invalid PGOInlineCDFThreshold, must be between 0 and 100")
}
}
- var hotCallsites []pgo.NodeMapKey
+ var hotCallsites []pgo.NamedCallEdge
inlineHotCallSiteThresholdPercent, hotCallsites = hotNodesFromCDF(p)
- if base.Debug.PGOInline > 0 {
+ if base.Debug.PGODebug > 0 {
fmt.Printf("hot-callsite-thres-from-CDF=%v\n", inlineHotCallSiteThresholdPercent)
}
candHotCalleeMap[callee] = struct{}{}
}
// mark hot call sites
- if caller := p.WeightedCG.IRNodes[n.CallerName]; caller != nil {
+ if caller := p.WeightedCG.IRNodes[n.CallerName]; caller != nil && caller.AST != nil {
csi := pgo.CallSiteInfo{LineOffset: n.CallSiteOffset, Caller: caller.AST}
candHotEdgeMap[csi] = struct{}{}
}
}
- if base.Debug.PGOInline >= 2 {
+ if base.Debug.PGODebug >= 3 {
fmt.Printf("hot-cg before inline in dot format:")
p.PrintWeightedCallGraphDOT(inlineHotCallSiteThresholdPercent)
}
// (currently only used in debug prints) (in case of equal weights,
// comparing with the threshold may not accurately reflect which nodes are
// considiered hot).
-func hotNodesFromCDF(p *pgo.Profile) (float64, []pgo.NodeMapKey) {
- nodes := make([]pgo.NodeMapKey, len(p.NodeMap))
- i := 0
- for n := range p.NodeMap {
- nodes[i] = n
- i++
- }
- sort.Slice(nodes, func(i, j int) bool {
- ni, nj := nodes[i], nodes[j]
- if wi, wj := p.NodeMap[ni].EWeight, p.NodeMap[nj].EWeight; wi != wj {
- return wi > wj // want larger weight first
- }
- // same weight, order by name/line number
- if ni.CallerName != nj.CallerName {
- return ni.CallerName < nj.CallerName
- }
- if ni.CalleeName != nj.CalleeName {
- return ni.CalleeName < nj.CalleeName
- }
- return ni.CallSiteOffset < nj.CallSiteOffset
- })
+func hotNodesFromCDF(p *pgo.Profile) (float64, []pgo.NamedCallEdge) {
cum := int64(0)
- for i, n := range nodes {
- w := p.NodeMap[n].EWeight
+ for i, n := range p.NamedEdgeMap.ByWeight {
+ w := p.NamedEdgeMap.Weight[n]
cum += w
- if pgo.WeightInPercentage(cum, p.TotalEdgeWeight) > inlineCDFHotCallSiteThresholdPercent {
+ if pgo.WeightInPercentage(cum, p.TotalWeight) > inlineCDFHotCallSiteThresholdPercent {
// nodes[:i+1] to include the very last node that makes it to go over the threshold.
// (Say, if the CDF threshold is 50% and one hot node takes 60% of weight, we want to
// include that node instead of excluding it.)
- return pgo.WeightInPercentage(w, p.TotalEdgeWeight), nodes[:i+1]
+ return pgo.WeightInPercentage(w, p.TotalWeight), p.NamedEdgeMap.ByWeight[:i+1]
}
}
- return 0, nodes
-}
-
-// pgoInlineEpilogue updates IRGraph after inlining.
-func pgoInlineEpilogue(p *pgo.Profile, decls []ir.Node) {
- if base.Debug.PGOInline >= 2 {
- ir.VisitFuncsBottomUp(decls, func(list []*ir.Func, recursive bool) {
- for _, f := range list {
- name := ir.PkgFuncName(f)
- if n, ok := p.WeightedCG.IRNodes[name]; ok {
- p.RedirectEdges(n, inlinedCallSites)
- }
- }
- })
- // Print the call-graph after inlining. This is a debugging feature.
- fmt.Printf("hot-cg after inline in dot:")
- p.PrintWeightedCallGraphDOT(inlineHotCallSiteThresholdPercent)
- }
+ return 0, p.NamedEdgeMap.ByWeight
}
// InlinePackage finds functions that can be inlined and clones them before walk expands them.
func InlinePackage(p *pgo.Profile) {
- InlineDecls(p, typecheck.Target.Decls, true)
+ if base.Debug.PGOInline == 0 {
+ p = nil
+ }
+
+ InlineDecls(p, typecheck.Target.Funcs, true)
+
+ // Perform a garbage collection of hidden closures functions that
+ // are no longer reachable from top-level functions following
+ // inlining. See #59404 and #59638 for more context.
+ garbageCollectUnreferencedHiddenClosures()
+
+ if base.Debug.DumpInlFuncProps != "" {
+ inlheur.DumpFuncProps(nil, base.Debug.DumpInlFuncProps, nil, inlineMaxBudget)
+ }
+ if goexperiment.NewInliner {
+ postProcessCallSites(p)
+ }
}
// InlineDecls applies inlining to the given batch of declarations.
-func InlineDecls(p *pgo.Profile, decls []ir.Node, doInline bool) {
+func InlineDecls(p *pgo.Profile, funcs []*ir.Func, doInline bool) {
if p != nil {
- pgoInlinePrologue(p, decls)
+ pgoInlinePrologue(p, funcs)
}
- ir.VisitFuncsBottomUp(decls, func(list []*ir.Func, recursive bool) {
+ doCanInline := func(n *ir.Func, recursive bool, numfns int) {
+ if !recursive || numfns > 1 {
+ // We allow inlining if there is no
+ // recursion, or the recursion cycle is
+ // across more than one function.
+ CanInline(n, p)
+ } else {
+ if base.Flag.LowerM > 1 && n.OClosure == nil {
+ fmt.Printf("%v: cannot inline %v: recursive\n", ir.Line(n), n.Nname)
+ }
+ }
+ }
+
+ ir.VisitFuncsBottomUp(funcs, func(list []*ir.Func, recursive bool) {
numfns := numNonClosures(list)
+ // We visit functions within an SCC in fairly arbitrary order,
+ // so by computing inlinability for all functions in the SCC
+ // before performing any inlining, the results are less
+ // sensitive to the order within the SCC (see #58905 for an
+ // example).
+
+ // First compute inlinability for all functions in the SCC ...
for _, n := range list {
- if !recursive || numfns > 1 {
- // We allow inlining if there is no
- // recursion, or the recursion cycle is
- // across more than one function.
- CanInline(n, p)
- } else {
- if base.Flag.LowerM > 1 {
- fmt.Printf("%v: cannot inline %v: recursive\n", ir.Line(n), n.Nname)
- }
- }
- if doInline {
+ doCanInline(n, recursive, numfns)
+ }
+ // ... then make a second pass to do inlining of calls.
+ if doInline {
+ for _, n := range list {
InlineCalls(n, p)
}
}
})
+}
- if p != nil {
- pgoInlineEpilogue(p, decls)
+// garbageCollectUnreferencedHiddenClosures makes a pass over all the
+// top-level (non-hidden-closure) functions looking for nested closure
+// functions that are reachable, then sweeps through the Target.Decls
+// list and marks any non-reachable hidden closure function as dead.
+// See issues #59404 and #59638 for more context.
+func garbageCollectUnreferencedHiddenClosures() {
+
+ liveFuncs := make(map[*ir.Func]bool)
+
+ var markLiveFuncs func(fn *ir.Func)
+ markLiveFuncs = func(fn *ir.Func) {
+ if liveFuncs[fn] {
+ return
+ }
+ liveFuncs[fn] = true
+ ir.Visit(fn, func(n ir.Node) {
+ if clo, ok := n.(*ir.ClosureExpr); ok {
+ markLiveFuncs(clo.Func)
+ }
+ })
+ }
+
+ for i := 0; i < len(typecheck.Target.Funcs); i++ {
+ fn := typecheck.Target.Funcs[i]
+ if fn.IsHiddenClosure() {
+ continue
+ }
+ markLiveFuncs(fn)
+ }
+
+ for i := 0; i < len(typecheck.Target.Funcs); i++ {
+ fn := typecheck.Target.Funcs[i]
+ if !fn.IsHiddenClosure() {
+ continue
+ }
+ if fn.IsDeadcodeClosure() {
+ continue
+ }
+ if liveFuncs[fn] {
+ continue
+ }
+ fn.SetIsDeadcodeClosure(true)
+ if base.Flag.LowerM > 2 {
+ fmt.Printf("%v: unreferenced closure %v marked as dead\n", ir.Line(fn), fn)
+ }
+ if fn.Inl != nil && fn.LSym == nil {
+ ir.InitLSym(fn, true)
+ }
+ }
+}
+
+// inlineBudget determines the max budget for function 'fn' prior to
+// analyzing the hairyness of the body of 'fn'. We pass in the pgo
+// profile if available (which can change the budget), also a
+// 'relaxed' flag, which expands the budget slightly to allow for the
+// possibility that a call to the function might have its score
+// adjusted downwards. If 'verbose' is set, then print a remark where
+// we boost the budget due to PGO.
+func inlineBudget(fn *ir.Func, profile *pgo.Profile, relaxed bool, verbose bool) int32 {
+ // Update the budget for profile-guided inlining.
+ budget := int32(inlineMaxBudget)
+ if profile != nil {
+ if n, ok := profile.WeightedCG.IRNodes[ir.LinkFuncName(fn)]; ok {
+ if _, ok := candHotCalleeMap[n]; ok {
+ budget = int32(inlineHotMaxBudget)
+ if verbose {
+ fmt.Printf("hot-node enabled increased budget=%v for func=%v\n", budget, ir.PkgFuncName(fn))
+ }
+ }
+ }
}
+ if relaxed {
+ budget += inlineMaxBudget
+ }
+ return budget
}
// CanInline determines whether fn is inlineable.
base.Fatalf("CanInline no nname %+v", fn)
}
+ var funcProps *inlheur.FuncProps
+ if goexperiment.NewInliner || inlheur.UnitTesting() {
+ callCanInline := func(fn *ir.Func) { CanInline(fn, profile) }
+ funcProps = inlheur.AnalyzeFunc(fn, callCanInline, inlineMaxBudget)
+ }
+
var reason string // reason, if any, that the function was not inlined
if base.Flag.LowerM > 1 || logopt.Enabled() {
defer func() {
}()
}
- // If marked "go:noinline", don't inline
- if fn.Pragma&ir.Noinline != 0 {
- reason = "marked go:noinline"
- return
- }
-
- // If marked "go:norace" and -race compilation, don't inline.
- if base.Flag.Race && fn.Pragma&ir.Norace != 0 {
- reason = "marked go:norace with -race compilation"
- return
- }
-
- // If marked "go:nocheckptr" and -d checkptr compilation, don't inline.
- if base.Debug.Checkptr != 0 && fn.Pragma&ir.NoCheckPtr != 0 {
- reason = "marked go:nocheckptr"
- return
- }
-
- // If marked "go:cgo_unsafe_args", don't inline, since the
- // function makes assumptions about its argument frame layout.
- if fn.Pragma&ir.CgoUnsafeArgs != 0 {
- reason = "marked go:cgo_unsafe_args"
- return
- }
-
- // If marked as "go:uintptrkeepalive", don't inline, since the
- // keep alive information is lost during inlining.
- //
- // TODO(prattmic): This is handled on calls during escape analysis,
- // which is after inlining. Move prior to inlining so the keep-alive is
- // maintained after inlining.
- if fn.Pragma&ir.UintptrKeepAlive != 0 {
- reason = "marked as having a keep-alive uintptr argument"
- return
- }
-
- // If marked as "go:uintptrescapes", don't inline, since the
- // escape information is lost during inlining.
- if fn.Pragma&ir.UintptrEscapes != 0 {
- reason = "marked as having an escaping uintptr argument"
- return
- }
-
- // The nowritebarrierrec checker currently works at function
- // granularity, so inlining yeswritebarrierrec functions can
- // confuse it (#22342). As a workaround, disallow inlining
- // them for now.
- if fn.Pragma&ir.Yeswritebarrierrec != 0 {
- reason = "marked go:yeswritebarrierrec"
+ reason = InlineImpossible(fn)
+ if reason != "" {
return
}
-
- // If fn has no body (is defined outside of Go), cannot inline it.
- if len(fn.Body) == 0 {
- reason = "no function body"
- return
- }
-
if fn.Typecheck() == 0 {
base.Fatalf("CanInline on non-typechecked function %v", fn)
}
cc = 1 // this appears to yield better performance than 0.
}
- // Update the budget for profile-guided inlining.
- budget := int32(inlineMaxBudget)
- if profile != nil {
- if n, ok := profile.WeightedCG.IRNodes[ir.PkgFuncName(fn)]; ok {
- if _, ok := candHotCalleeMap[n]; ok {
- budget = int32(inlineHotMaxBudget)
- if base.Debug.PGOInline > 0 {
- fmt.Printf("hot-node enabled increased budget=%v for func=%v\n", budget, ir.PkgFuncName(fn))
- }
- }
- }
- }
+ // Used a "relaxed" inline budget if goexperiment.NewInliner is in
+ // effect, or if we're producing a debugging dump.
+ relaxed := goexperiment.NewInliner ||
+ (base.Debug.DumpInlFuncProps != "" ||
+ base.Debug.DumpInlCallSiteScores != 0)
+
+ // Compute the inline budget for this func.
+ budget := inlineBudget(fn, profile, relaxed, base.Debug.PGODebug > 0)
// At this point in the game the function we're looking at may
// have "stale" autos, vars that still appear in the Dcl list, but
// when creating the "Inline.Dcl" field below; to accomplish this,
// the hairyVisitor below builds up a map of used/referenced
// locals, and we use this map to produce a pruned Inline.Dcl
- // list. See issue 25249 for more context.
+ // list. See issue 25459 for more context.
visitor := hairyVisitor{
curFunc: fn,
+ isBigFunc: isBigFunc(fn),
budget: budget,
maxBudget: budget,
extraCallCost: cc,
}
n.Func.Inl = &ir.Inline{
- Cost: budget - visitor.budget,
- Dcl: pruneUnusedAutos(n.Defn.(*ir.Func).Dcl, &visitor),
- Body: inlcopylist(fn.Body),
+ Cost: budget - visitor.budget,
+ Dcl: pruneUnusedAutos(n.Func.Dcl, &visitor),
+ HaveDcl: true,
CanDelayResults: canDelayResults(fn),
}
+ if goexperiment.NewInliner {
+ n.Func.Inl.Properties = funcProps.SerializeToString()
+ }
if base.Flag.LowerM > 1 {
- fmt.Printf("%v: can inline %v with cost %d as: %v { %v }\n", ir.Line(fn), n, budget-visitor.budget, fn.Type(), ir.Nodes(n.Func.Inl.Body))
+ fmt.Printf("%v: can inline %v with cost %d as: %v { %v }\n", ir.Line(fn), n, budget-visitor.budget, fn.Type(), ir.Nodes(fn.Body))
} else if base.Flag.LowerM != 0 {
fmt.Printf("%v: can inline %v\n", ir.Line(fn), n)
}
}
}
+// InlineImpossible returns a non-empty reason string if fn is impossible to
+// inline regardless of cost or contents.
+func InlineImpossible(fn *ir.Func) string {
+ var reason string // reason, if any, that the function can not be inlined.
+ if fn.Nname == nil {
+ reason = "no name"
+ return reason
+ }
+
+ // If marked "go:noinline", don't inline.
+ if fn.Pragma&ir.Noinline != 0 {
+ reason = "marked go:noinline"
+ return reason
+ }
+
+ // If marked "go:norace" and -race compilation, don't inline.
+ if base.Flag.Race && fn.Pragma&ir.Norace != 0 {
+ reason = "marked go:norace with -race compilation"
+ return reason
+ }
+
+ // If marked "go:nocheckptr" and -d checkptr compilation, don't inline.
+ if base.Debug.Checkptr != 0 && fn.Pragma&ir.NoCheckPtr != 0 {
+ reason = "marked go:nocheckptr"
+ return reason
+ }
+
+ // If marked "go:cgo_unsafe_args", don't inline, since the function
+ // makes assumptions about its argument frame layout.
+ if fn.Pragma&ir.CgoUnsafeArgs != 0 {
+ reason = "marked go:cgo_unsafe_args"
+ return reason
+ }
+
+ // If marked as "go:uintptrkeepalive", don't inline, since the keep
+ // alive information is lost during inlining.
+ //
+ // TODO(prattmic): This is handled on calls during escape analysis,
+ // which is after inlining. Move prior to inlining so the keep-alive is
+ // maintained after inlining.
+ if fn.Pragma&ir.UintptrKeepAlive != 0 {
+ reason = "marked as having a keep-alive uintptr argument"
+ return reason
+ }
+
+ // If marked as "go:uintptrescapes", don't inline, since the escape
+ // information is lost during inlining.
+ if fn.Pragma&ir.UintptrEscapes != 0 {
+ reason = "marked as having an escaping uintptr argument"
+ return reason
+ }
+
+ // The nowritebarrierrec checker currently works at function
+ // granularity, so inlining yeswritebarrierrec functions can confuse it
+ // (#22342). As a workaround, disallow inlining them for now.
+ if fn.Pragma&ir.Yeswritebarrierrec != 0 {
+ reason = "marked go:yeswritebarrierrec"
+ return reason
+ }
+
+ // If a local function has no fn.Body (is defined outside of Go), cannot inline it.
+ // Imported functions don't have fn.Body but might have inline body in fn.Inl.
+ if len(fn.Body) == 0 && !typecheck.HaveInlineBody(fn) {
+ reason = "no function body"
+ return reason
+ }
+
+ return ""
+}
+
// canDelayResults reports whether inlined calls to fn can delay
// declaring the result parameter until the "return" statement.
func canDelayResults(fn *ir.Func) bool {
}
// temporaries for return values.
- for _, param := range fn.Type().Results().FieldSlice() {
- if sym := types.OrigSym(param.Sym); sym != nil && !sym.IsBlank() {
+ for _, param := range fn.Type().Results() {
+ if sym := param.Sym; sym != nil && !sym.IsBlank() {
return false // found a named result parameter (case 3)
}
}
type hairyVisitor struct {
// This is needed to access the current caller in the doNode function.
curFunc *ir.Func
+ isBigFunc bool
budget int32
maxBudget int32
reason string
return false
}
+// doNode visits n and its children, updates the state in v, and returns true if
+// n makes the current function too hairy for inlining.
func (v *hairyVisitor) doNode(n ir.Node) bool {
if n == nil {
return false
}
+opSwitch:
switch n.Op() {
// Call is okay if inlinable and we have the budget for the body.
case ir.OCALLFUNC:
// because getcaller{pc,sp} expect a pointer to the caller's first argument.
//
// runtime.throw is a "cheap call" like panic in normal code.
- if n.X.Op() == ir.ONAME {
- name := n.X.(*ir.Name)
- if name.Class == ir.PFUNC && types.IsRuntimePkg(name.Sym().Pkg) {
- fn := name.Sym().Name
- if fn == "getcallerpc" || fn == "getcallersp" {
+ var cheap bool
+ if n.Fun.Op() == ir.ONAME {
+ name := n.Fun.(*ir.Name)
+ if name.Class == ir.PFUNC {
+ switch fn := types.RuntimeSymName(name.Sym()); fn {
+ case "getcallerpc", "getcallersp":
v.reason = "call to " + fn
return true
- }
- if fn == "throw" {
+ case "throw":
v.budget -= inlineExtraThrowCost
- break
+ break opSwitch
+ }
+ // Special case for reflect.noescape. It does just type
+ // conversions to appease the escape analysis, and doesn't
+ // generate code.
+ if types.ReflectSymName(name.Sym()) == "noescape" {
+ cheap = true
}
}
// Special case for coverage counter updates; although
return false
}
}
- if n.X.Op() == ir.OMETHEXPR {
- if meth := ir.MethodExprName(n.X); meth != nil {
+ if n.Fun.Op() == ir.OMETHEXPR {
+ if meth := ir.MethodExprName(n.Fun); meth != nil {
if fn := meth.Func; fn != nil {
s := fn.Sym()
- var cheap bool
- if types.IsRuntimePkg(s.Pkg) && s.Name == "heapBits.nextArena" {
+ if types.RuntimeSymName(s) == "heapBits.nextArena" {
// Special case: explicitly allow mid-stack inlining of
// runtime.heapBits.next even though it calls slow-path
// runtime.heapBits.nextArena.
cheap = true
}
}
- if cheap {
- break // treat like any other node, that is, cost of 1
- }
}
}
}
-
- // Determine if the callee edge is for an inlinable hot callee or not.
- if v.profile != nil && v.curFunc != nil {
- if fn := inlCallee(n.X, v.profile); fn != nil && typecheck.HaveInlineBody(fn) {
- lineOffset := pgo.NodeLineOffset(n, fn)
- csi := pgo.CallSiteInfo{LineOffset: lineOffset, Caller: v.curFunc}
- if _, o := candHotEdgeMap[csi]; o {
- if base.Debug.PGOInline > 0 {
- fmt.Printf("hot-callsite identified at line=%v for func=%v\n", ir.Line(n), ir.PkgFuncName(v.curFunc))
- }
- }
- }
+ if cheap {
+ break // treat like any other node, that is, cost of 1
}
if ir.IsIntrinsicCall(n) {
break
}
- if fn := inlCallee(n.X, v.profile); fn != nil && typecheck.HaveInlineBody(fn) {
- v.budget -= fn.Inl.Cost
- break
+ if callee := inlCallee(v.curFunc, n.Fun, v.profile); callee != nil && typecheck.HaveInlineBody(callee) {
+ // Check whether we'd actually inline this call. Set
+ // log == false since we aren't actually doing inlining
+ // yet.
+ if canInlineCallExpr(v.curFunc, n, callee, v.isBigFunc, false) {
+ // mkinlcall would inline this call [1], so use
+ // the cost of the inline body as the cost of
+ // the call, as that is what will actually
+ // appear in the code.
+ //
+ // [1] This is almost a perfect match to the
+ // mkinlcall logic, except that
+ // canInlineCallExpr considers inlining cycles
+ // by looking at what has already been inlined.
+ // Since we haven't done any inlining yet we
+ // will miss those.
+ v.budget -= callee.Inl.Cost
+ break
+ }
}
// Call cost for non-leaf inlining.
v.budget -= inlineExtraPanicCost
case ir.ORECOVER:
+ base.FatalfAt(n.Pos(), "ORECOVER missed typecheck")
+ case ir.ORECOVERFP:
// recover matches the argument frame pointer to find
// the right panic value, so it needs an argument frame.
v.reason = "call to recover"
// TODO(danscales): Maybe make budget proportional to number of closure
// variables, e.g.:
//v.budget -= int32(len(n.(*ir.ClosureExpr).Func.ClosureVars) * 3)
+ // TODO(austin): However, if we're able to inline this closure into
+ // v.curFunc, then we actually pay nothing for the closure captures. We
+ // should try to account for that if we're going to account for captures.
v.budget -= 15
- // Scan body of closure (which DoChildren doesn't automatically
- // do) to check for disallowed ops in the body and include the
- // body in the budget.
- if doList(n.(*ir.ClosureExpr).Func.Body, v.do) {
- return true
- }
- case ir.OGO,
- ir.ODEFER,
- ir.ODCLTYPE, // can't print yet
- ir.OTAILCALL:
+ case ir.OGO, ir.ODEFER, ir.OTAILCALL:
v.reason = "unhandled op " + n.Op().String()
return true
// This doesn't produce code, but the children might.
v.budget++ // undo default cost
- case ir.ODCLCONST, ir.OFALL:
+ case ir.OFALL, ir.OTYPE:
// These nodes don't produce code; omit from inlining budget.
return false
})
}
-// inlcopylist (together with inlcopy) recursively copies a list of nodes, except
-// that it keeps the same ONAME, OTYPE, and OLITERAL nodes. It is used for copying
-// the body and dcls of an inlineable function.
-func inlcopylist(ll []ir.Node) []ir.Node {
- s := make([]ir.Node, len(ll))
- for i, n := range ll {
- s[i] = inlcopy(n)
- }
- return s
-}
-
-// inlcopy is like DeepCopy(), but does extra work to copy closures.
-func inlcopy(n ir.Node) ir.Node {
- var edit func(ir.Node) ir.Node
- edit = func(x ir.Node) ir.Node {
- switch x.Op() {
- case ir.ONAME, ir.OTYPE, ir.OLITERAL, ir.ONIL:
- return x
- }
- m := ir.Copy(x)
- ir.EditChildren(m, edit)
- if x.Op() == ir.OCLOSURE {
- x := x.(*ir.ClosureExpr)
- // Need to save/duplicate x.Func.Nname,
- // x.Func.Nname.Ntype, x.Func.Dcl, x.Func.ClosureVars, and
- // x.Func.Body for iexport and local inlining.
- oldfn := x.Func
- newfn := ir.NewFunc(oldfn.Pos())
- m.(*ir.ClosureExpr).Func = newfn
- newfn.Nname = ir.NewNameAt(oldfn.Nname.Pos(), oldfn.Nname.Sym())
- // XXX OK to share fn.Type() ??
- newfn.Nname.SetType(oldfn.Nname.Type())
- newfn.Body = inlcopylist(oldfn.Body)
- // Make shallow copy of the Dcl and ClosureVar slices
- newfn.Dcl = append([]*ir.Name(nil), oldfn.Dcl...)
- newfn.ClosureVars = append([]*ir.Name(nil), oldfn.ClosureVars...)
- }
- return m
- }
- return edit(n)
-}
-
// InlineCalls/inlnode walks fn's statements and expressions and substitutes any
// calls made to inlineable functions. This is the external entry point.
func InlineCalls(fn *ir.Func, profile *pgo.Profile) {
+ if goexperiment.NewInliner && !fn.Wrapper() {
+ inlheur.ScoreCalls(fn)
+ }
+ if base.Debug.DumpInlFuncProps != "" && !fn.Wrapper() {
+ inlheur.DumpFuncProps(fn, base.Debug.DumpInlFuncProps,
+ func(fn *ir.Func) { CanInline(fn, profile) }, inlineMaxBudget)
+ }
savefn := ir.CurFunc
ir.CurFunc = fn
- maxCost := int32(inlineMaxBudget)
- if isBigFunc(fn) {
- if base.Flag.LowerM > 1 {
- fmt.Printf("%v: function %v considered 'big'; revising maxCost from %d to %d\n", ir.Line(fn), fn, maxCost, inlineBigFunctionMaxCost)
- }
- maxCost = inlineBigFunctionMaxCost
+ bigCaller := isBigFunc(fn)
+ if bigCaller && base.Flag.LowerM > 1 {
+ fmt.Printf("%v: function %v considered 'big'; reducing max cost of inlinees\n", ir.Line(fn), fn)
}
var inlCalls []*ir.InlinedCallExpr
var edit func(ir.Node) ir.Node
edit = func(n ir.Node) ir.Node {
- return inlnode(n, maxCost, &inlCalls, edit, profile)
+ return inlnode(fn, n, bigCaller, &inlCalls, edit, profile)
}
ir.EditChildren(fn, edit)
// The result of inlnode MUST be assigned back to n, e.g.
//
// n.Left = inlnode(n.Left)
-func inlnode(n ir.Node, maxCost int32, inlCalls *[]*ir.InlinedCallExpr, edit func(ir.Node) ir.Node, profile *pgo.Profile) ir.Node {
+func inlnode(callerfn *ir.Func, n ir.Node, bigCaller bool, inlCalls *[]*ir.InlinedCallExpr, edit func(ir.Node) ir.Node, profile *pgo.Profile) ir.Node {
if n == nil {
return n
}
base.FatalfAt(n.Pos(), "OCALLMETH missed by typecheck")
case ir.OCALLFUNC:
n := n.(*ir.CallExpr)
- if n.X.Op() == ir.OMETHEXPR {
+ if n.Fun.Op() == ir.OMETHEXPR {
// Prevent inlining some reflect.Value methods when using checkptr,
// even when package reflect was compiled without it (#35073).
- if meth := ir.MethodExprName(n.X); meth != nil {
+ if meth := ir.MethodExprName(n.Fun); meth != nil {
s := meth.Sym()
- if base.Debug.Checkptr != 0 && types.IsReflectPkg(s.Pkg) && (s.Name == "Value.UnsafeAddr" || s.Name == "Value.Pointer") {
- return n
+ if base.Debug.Checkptr != 0 {
+ switch types.ReflectSymName(s) {
+ case "Value.UnsafeAddr", "Value.Pointer":
+ return n
+ }
}
}
}
break
}
if base.Flag.LowerM > 3 {
- fmt.Printf("%v:call to func %+v\n", ir.Line(n), call.X)
+ fmt.Printf("%v:call to func %+v\n", ir.Line(n), call.Fun)
}
if ir.IsIntrinsicCall(call) {
break
}
- if fn := inlCallee(call.X, profile); fn != nil && typecheck.HaveInlineBody(fn) {
- n = mkinlcall(call, fn, maxCost, inlCalls, edit)
+ if fn := inlCallee(callerfn, call.Fun, profile); fn != nil && typecheck.HaveInlineBody(fn) {
+ n = mkinlcall(callerfn, call, fn, bigCaller, inlCalls)
}
}
// inlCallee takes a function-typed expression and returns the underlying function ONAME
// that it refers to if statically known. Otherwise, it returns nil.
-func inlCallee(fn ir.Node, profile *pgo.Profile) *ir.Func {
+func inlCallee(caller *ir.Func, fn ir.Node, profile *pgo.Profile) (res *ir.Func) {
fn = ir.StaticValue(fn)
switch fn.Op() {
case ir.OMETHEXPR:
case ir.OCLOSURE:
fn := fn.(*ir.ClosureExpr)
c := fn.Func
+ if len(c.ClosureVars) != 0 && c.ClosureVars[0].Outer.Curfn != caller {
+ return nil // inliner doesn't support inlining across closure frames
+ }
CanInline(c, profile)
return c
}
// InlineCall allows the inliner implementation to be overridden.
// If it returns nil, the function will not be inlined.
-var InlineCall = func(call *ir.CallExpr, fn *ir.Func, inlIndex int) *ir.InlinedCallExpr {
+var InlineCall = func(callerfn *ir.Func, call *ir.CallExpr, fn *ir.Func, inlIndex int) *ir.InlinedCallExpr {
base.Fatalf("inline.InlineCall not overridden")
panic("unreachable")
}
-// If n is a OCALLFUNC node, and fn is an ONAME node for a
-// function with an inlinable body, return an OINLCALL node that can replace n.
-// The returned node's Ninit has the parameter assignments, the Nbody is the
-// inlined function body, and (List, Rlist) contain the (input, output)
-// parameters.
-// The result of mkinlcall MUST be assigned back to n, e.g.
+// inlineCostOK returns true if call n from caller to callee is cheap enough to
+// inline. bigCaller indicates that caller is a big function.
//
-// n.Left = mkinlcall(n.Left, fn, isddd)
-func mkinlcall(n *ir.CallExpr, fn *ir.Func, maxCost int32, inlCalls *[]*ir.InlinedCallExpr, edit func(ir.Node) ir.Node) ir.Node {
- if fn.Inl == nil {
- if logopt.Enabled() {
- logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(ir.CurFunc),
- fmt.Sprintf("%s cannot be inlined", ir.PkgFuncName(fn)))
+// If inlineCostOK returns false, it also returns the max cost that the callee
+// exceeded.
+func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller bool) (bool, int32) {
+ maxCost := int32(inlineMaxBudget)
+ if bigCaller {
+ // We use this to restrict inlining into very big functions.
+ // See issue 26546 and 17566.
+ maxCost = inlineBigFunctionMaxCost
+ }
+
+ metric := callee.Inl.Cost
+ if goexperiment.NewInliner {
+ ok, score := inlheur.GetCallSiteScore(n)
+ if ok {
+ metric = int32(score)
}
- return n
+
}
- if fn.Inl.Cost > maxCost {
- // If the callsite is hot and it is under the inlineHotMaxBudget budget, then try to inline it, or else bail.
- lineOffset := pgo.NodeLineOffset(n, ir.CurFunc)
- csi := pgo.CallSiteInfo{LineOffset: lineOffset, Caller: ir.CurFunc}
- if _, ok := candHotEdgeMap[csi]; ok {
- if fn.Inl.Cost > inlineHotMaxBudget {
- if logopt.Enabled() {
- logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(ir.CurFunc),
- fmt.Sprintf("cost %d of %s exceeds max large caller cost %d", fn.Inl.Cost, ir.PkgFuncName(fn), inlineHotMaxBudget))
- }
- return n
- }
- if base.Debug.PGOInline > 0 {
- fmt.Printf("hot-budget check allows inlining for call %s at %v\n", ir.PkgFuncName(fn), ir.Line(n))
- }
- } else {
- // The inlined function body is too big. Typically we use this check to restrict
- // inlining into very big functions. See issue 26546 and 17566.
- if logopt.Enabled() {
- logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(ir.CurFunc),
- fmt.Sprintf("cost %d of %s exceeds max large caller cost %d", fn.Inl.Cost, ir.PkgFuncName(fn), maxCost))
- }
- return n
+
+ if metric <= maxCost {
+ // Simple case. Function is already cheap enough.
+ return true, 0
+ }
+
+ // We'll also allow inlining of hot functions below inlineHotMaxBudget,
+ // but only in small functions.
+
+ lineOffset := pgo.NodeLineOffset(n, caller)
+ csi := pgo.CallSiteInfo{LineOffset: lineOffset, Caller: caller}
+ if _, ok := candHotEdgeMap[csi]; !ok {
+ // Cold
+ return false, maxCost
+ }
+
+ // Hot
+
+ if bigCaller {
+ if base.Debug.PGODebug > 0 {
+ fmt.Printf("hot-big check disallows inlining for call %s (cost %d) at %v in big function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller))
+ }
+ return false, maxCost
+ }
+
+ if metric > inlineHotMaxBudget {
+ return false, inlineHotMaxBudget
+ }
+
+ if !base.PGOHash.MatchPosWithInfo(n.Pos(), "inline", nil) {
+ // De-selected by PGO Hash.
+ return false, maxCost
+ }
+
+ if base.Debug.PGODebug > 0 {
+ fmt.Printf("hot-budget check allows inlining for call %s (cost %d) at %v in function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller))
+ }
+
+ return true, 0
+}
+
+// canInlineCallsite returns true if the call n from caller to callee can be
+// inlined. bigCaller indicates that caller is a big function. log indicates
+// that the 'cannot inline' reason should be logged.
+//
+// Preconditions: CanInline(callee) has already been called.
+func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCaller bool, log bool) bool {
+ if callee.Inl == nil {
+ // callee is never inlinable.
+ if log && logopt.Enabled() {
+ logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
+ fmt.Sprintf("%s cannot be inlined", ir.PkgFuncName(callee)))
}
+ return false
}
- if fn == ir.CurFunc {
+ if ok, maxCost := inlineCostOK(n, callerfn, callee, bigCaller); !ok {
+ // callee cost too high for this call site.
+ if log && logopt.Enabled() {
+ logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
+ fmt.Sprintf("cost %d of %s exceeds max caller cost %d", callee.Inl.Cost, ir.PkgFuncName(callee), maxCost))
+ }
+ return false
+ }
+
+ if callee == callerfn {
// Can't recursively inline a function into itself.
- if logopt.Enabled() {
- logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to %s", ir.FuncName(ir.CurFunc)))
+ if log && logopt.Enabled() {
+ logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to %s", ir.FuncName(callerfn)))
}
- return n
+ return false
}
- if base.Flag.Cfg.Instrumenting && types.IsRuntimePkg(fn.Sym().Pkg) {
+ if base.Flag.Cfg.Instrumenting && types.IsNoInstrumentPkg(callee.Sym().Pkg) {
// Runtime package must not be instrumented.
// Instrument skips runtime package. However, some runtime code can be
// inlined into other packages and instrumented there. To avoid this,
// we disable inlining of runtime functions when instrumenting.
// The example that we observed is inlining of LockOSThread,
// which lead to false race reports on m contents.
- return n
+ if log && logopt.Enabled() {
+ logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
+ fmt.Sprintf("call to runtime function %s in instrumented build", ir.PkgFuncName(callee)))
+ }
+ return false
}
- parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()
- sym := fn.Linksym()
+ if base.Flag.Race && types.IsNoRacePkg(callee.Sym().Pkg) {
+ if log && logopt.Enabled() {
+ logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
+ fmt.Sprintf(`call to into "no-race" package function %s in race build`, ir.PkgFuncName(callee)))
+ }
+ return false
+ }
// Check if we've already inlined this function at this particular
// call site, in order to stop inlining when we reach the beginning
// many functions. Most likely, the inlining will stop before we
// even hit the beginning of the cycle again, but this catches the
// unusual case.
+ parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()
+ sym := callee.Linksym()
for inlIndex := parent; inlIndex >= 0; inlIndex = base.Ctxt.InlTree.Parent(inlIndex) {
if base.Ctxt.InlTree.InlinedFunction(inlIndex) == sym {
- if base.Flag.LowerM > 1 {
- fmt.Printf("%v: cannot inline %v into %v: repeated recursive cycle\n", ir.Line(n), fn, ir.FuncName(ir.CurFunc))
+ if log {
+ if base.Flag.LowerM > 1 {
+ fmt.Printf("%v: cannot inline %v into %v: repeated recursive cycle\n", ir.Line(n), callee, ir.FuncName(callerfn))
+ }
+ if logopt.Enabled() {
+ logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
+ fmt.Sprintf("repeated recursive cycle to %s", ir.PkgFuncName(callee)))
+ }
}
- return n
+ return false
}
}
- typecheck.FixVariadicCall(n)
+ return true
+}
+
+// If n is a OCALLFUNC node, and fn is an ONAME node for a
+// function with an inlinable body, return an OINLCALL node that can replace n.
+// The returned node's Ninit has the parameter assignments, the Nbody is the
+// inlined function body, and (List, Rlist) contain the (input, output)
+// parameters.
+// The result of mkinlcall MUST be assigned back to n, e.g.
+//
+// n.Left = mkinlcall(n.Left, fn, isddd)
+func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller bool, inlCalls *[]*ir.InlinedCallExpr) ir.Node {
+ if !canInlineCallExpr(callerfn, n, fn, bigCaller, true) {
+ return n
+ }
+ typecheck.AssertFixedCall(n)
- inlIndex := base.Ctxt.InlTree.Add(parent, n.Pos(), sym)
+ parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()
+ sym := fn.Linksym()
+ inlIndex := base.Ctxt.InlTree.Add(parent, n.Pos(), sym, ir.FuncName(fn))
closureInitLSym := func(n *ir.CallExpr, fn *ir.Func) {
// The linker needs FuncInfo metadata for all inlined
// Not a standard call.
return
}
- if n.X.Op() != ir.OCLOSURE {
+ if n.Fun.Op() != ir.OCLOSURE {
// Not a direct closure call.
return
}
- clo := n.X.(*ir.ClosureExpr)
+ clo := n.Fun.(*ir.ClosureExpr)
if ir.IsTrivialClosure(clo) {
// enqueueFunc will handle trivial closures anyways.
return
fmt.Printf("%v: Before inlining: %+v\n", ir.Line(n), n)
}
- if base.Debug.PGOInline > 0 {
- csi := pgo.CallSiteInfo{LineOffset: pgo.NodeLineOffset(n, fn), Caller: ir.CurFunc}
- if _, ok := inlinedCallSites[csi]; !ok {
- inlinedCallSites[csi] = struct{}{}
- }
- }
-
- res := InlineCall(n, fn, inlIndex)
+ res := InlineCall(callerfn, n, fn, inlIndex)
if res == nil {
base.FatalfAt(n.Pos(), "inlining call to %v failed", fn)
for _, n := range ll {
if n.Class == ir.PAUTO {
if !vis.usedLocals.Has(n) {
+ // TODO(mdempsky): Simplify code after confident that this
+ // never happens anymore.
+ base.FatalfAt(n.Pos(), "unused auto: %v", n)
continue
}
}
// determine whether it represents a call to sync/atomic.AddUint32 to
// increment a coverage counter.
func isAtomicCoverageCounterUpdate(cn *ir.CallExpr) bool {
- if cn.X.Op() != ir.ONAME {
+ if cn.Fun.Op() != ir.ONAME {
return false
}
- name := cn.X.(*ir.Name)
+ name := cn.Fun.(*ir.Name)
if name.Class != ir.PFUNC {
return false
}
v := isIndexingCoverageCounter(adn.X)
return v
}
+
+func postProcessCallSites(profile *pgo.Profile) {
+ if base.Debug.DumpInlCallSiteScores != 0 {
+ budgetCallback := func(fn *ir.Func, prof *pgo.Profile) (int32, bool) {
+ v := inlineBudget(fn, prof, false, false)
+ return v, v == inlineHotMaxBudget
+ }
+ inlheur.DumpInlCallSiteScores(profile, budgetCallback)
+ }
+}