TypeAssert int `help:"print information about type assertion inlining"`
TypecheckInl int `help:"eager typechecking of inline function bodies"`
WB int `help:"print information about write barriers"`
+ ABIWrap int `help:"print information about ABI wrapper generation"`
any bool // set when any of the values have been set
}
CompilingRuntime bool "flag:\"+\" help:\"compiling runtime\""
// Longer names
+ ABIWrap bool "help:\"enable generation of ABI wrappers\""
+ ABIWrapLimit int "help:\"emit at most N ABI wrappers (for debugging)\""
AsmHdr string "help:\"write assembly header to `file`\""
Bench string "help:\"append benchmark times to `file`\""
BlockProfile string "help:\"write block profile to `file`\""
Flag.LowerP = &Ctxt.Pkgpath
Flag.LowerV = &Ctxt.Debugvlog
+ Flag.ABIWrap = objabi.Regabi_enabled != 0
Flag.Dwarf = objabi.GOARCH != "wasm"
Flag.DwarfBASEntries = &Ctxt.UseBASEntries
Flag.DwarfLocationLists = &Ctxt.Flag_locationlists
"cmd/compile/internal/base"
"cmd/compile/internal/ir"
"cmd/compile/internal/ssa"
+ "cmd/compile/internal/types"
"cmd/internal/obj"
"cmd/internal/objabi"
"cmd/internal/src"
+ "fmt"
+ "os"
)
var sharedProgArray = new([10000]obj.Prog) // *T instead of T to work around issue 19839
ptxt.From.Sym = fn.LSym
}
+// makeABIWrapper creates a new function that wraps a cross-ABI call
+// to "f". The wrapper is marked as an ABIWRAPPER.
+func makeABIWrapper(f *ir.Func, wrapperABI obj.ABI) {
+
+ // Q: is this needed?
+ savepos := base.Pos
+ savedclcontext := dclcontext
+ savedcurfn := Curfn
+
+ base.Pos = autogeneratedPos
+ dclcontext = ir.PEXTERN
+
+ // At the moment we don't support wrapping a method, we'd need machinery
+ // below to handle the receiver. Panic if we see this scenario.
+ ft := f.Nname.Ntype.Type()
+ if ft.NumRecvs() != 0 {
+ panic("makeABIWrapper support for wrapping methods not implemented")
+ }
+
+ // Manufacture a new func type to use for the wrapper.
+ var noReceiver *ir.Field
+ tfn := ir.NewFuncType(base.Pos,
+ noReceiver,
+ structargs(ft.Params(), true),
+ structargs(ft.Results(), false))
+
+ // Reuse f's types.Sym to create a new ODCLFUNC/function.
+ fn := dclfunc(f.Nname.Sym(), tfn)
+ fn.SetDupok(true)
+ fn.SetWrapper(true) // ignore frame for panic+recover matching
+
+ // Select LSYM now.
+ asym := base.Ctxt.LookupABI(f.LSym.Name, wrapperABI)
+ asym.Type = objabi.STEXT
+ if fn.LSym != nil {
+ panic("unexpected")
+ }
+ fn.LSym = asym
+
+ // ABI0-to-ABIInternal wrappers will be mainly loading params from
+ // stack into registers (and/or storing stack locations back to
+ // registers after the wrapped call); in most cases they won't
+ // need to allocate stack space, so it should be OK to mark them
+ // as NOSPLIT in these cases. In addition, my assumption is that
+ // functions written in assembly are NOSPLIT in most (but not all)
+ // cases. In the case of an ABIInternal target that has too many
+ // parameters to fit into registers, the wrapper would need to
+ // allocate stack space, but this seems like an unlikely scenario.
+ // Hence: mark these wrappers NOSPLIT.
+ //
+ // ABIInternal-to-ABI0 wrappers on the other hand will be taking
+ // things in registers and pushing them onto the stack prior to
+ // the ABI0 call, meaning that they will always need to allocate
+ // stack space. If the compiler marks them as NOSPLIT this seems
+ // as though it could lead to situations where the the linker's
+ // nosplit-overflow analysis would trigger a link failure. On the
+ // other hand if they not tagged NOSPLIT then this could cause
+ // problems when building the runtime (since there may be calls to
+ // asm routine in cases where it's not safe to grow the stack). In
+ // most cases the wrapper would be (in effect) inlined, but are
+ // there (perhaps) indirect calls from the runtime that could run
+ // into trouble here.
+ // FIXME: at the moment all.bash does not pass when I leave out
+ // NOSPLIT for these wrappers, so all are currently tagged with NOSPLIT.
+ setupTextLSym(fn, obj.NOSPLIT|obj.ABIWRAPPER)
+
+ // Generate call. Use tail call if no params and no returns,
+ // but a regular call otherwise.
+ //
+ // Note: ideally we would be using a tail call in cases where
+ // there are params but no returns for ABI0->ABIInternal wrappers,
+ // provided that all params fit into registers (e.g. we don't have
+ // to allocate any stack space). Doing this will require some
+ // extra work in typecheck/walk/ssa, might want to add a new node
+ // OTAILCALL or something to this effect.
+ var call ir.Node
+ if tfn.Type().NumResults() == 0 && tfn.Type().NumParams() == 0 && tfn.Type().NumRecvs() == 0 {
+ call = nodSym(ir.ORETJMP, nil, f.Nname.Sym())
+ } else {
+ call = ir.Nod(ir.OCALL, f.Nname, nil)
+ call.PtrList().Set(paramNnames(tfn.Type()))
+ call.SetIsDDD(tfn.Type().IsVariadic())
+ if tfn.Type().NumResults() > 0 {
+ n := ir.Nod(ir.ORETURN, nil, nil)
+ n.PtrList().Set1(call)
+ call = n
+ }
+ }
+ fn.PtrBody().Append(call)
+
+ funcbody()
+ if base.Debug.DclStack != 0 {
+ testdclstack()
+ }
+
+ typecheckFunc(fn)
+ Curfn = fn
+ typecheckslice(fn.Body().Slice(), ctxStmt)
+
+ escapeFuncs([]*ir.Func{fn}, false)
+
+ Target.Decls = append(Target.Decls, fn)
+
+ // Restore previous context.
+ base.Pos = savepos
+ dclcontext = savedclcontext
+ Curfn = savedcurfn
+}
+
// initLSym defines f's obj.LSym and initializes it based on the
// properties of f. This includes setting the symbol flags and ABI and
// creating and initializing related DWARF symbols.
//
// initLSym must be called exactly once per function and must be
// called for both functions with bodies and functions without bodies.
+// For body-less functions, we only create the LSym; for functions
+// with bodies call a helper to setup up / populate the LSym.
func initLSym(f *ir.Func, hasBody bool) {
+ // FIXME: for new-style ABI wrappers, we set up the lsym at the
+ // point the wrapper is created.
+ if f.LSym != nil && base.Flag.ABIWrap {
+ return
+ }
+ selectLSym(f, hasBody)
+ if hasBody {
+ setupTextLSym(f, 0)
+ }
+}
+
+// selectLSym sets up the LSym for a given function, and
+// makes calls to helpers to create ABI wrappers if needed.
+func selectLSym(f *ir.Func, hasBody bool) {
if f.LSym != nil {
base.Fatalf("Func.initLSym called twice")
}
if nam := f.Nname; !ir.IsBlank(nam) {
- f.LSym = nam.Sym().Linksym()
- if f.Pragma&ir.Systemstack != 0 {
- f.LSym.Set(obj.AttrCFunc, true)
- }
- var aliasABI obj.ABI
- needABIAlias := false
- defABI, hasDefABI := symabiDefs[f.LSym.Name]
+ var wrapperABI obj.ABI
+ needABIWrapper := false
+ defABI, hasDefABI := symabiDefs[nam.Sym().LinksymName()]
if hasDefABI && defABI == obj.ABI0 {
// Symbol is defined as ABI0. Create an
// Internal -> ABI0 wrapper.
- f.LSym.SetABI(obj.ABI0)
- needABIAlias, aliasABI = true, obj.ABIInternal
+ f.LSym = nam.Sym().LinksymABI0()
+ needABIWrapper, wrapperABI = true, obj.ABIInternal
} else {
+ f.LSym = nam.Sym().Linksym()
// No ABI override. Check that the symbol is
// using the expected ABI.
want := obj.ABIInternal
base.Fatalf("function symbol %s has the wrong ABI %v, expected %v", f.LSym.Name, f.LSym.ABI(), want)
}
}
+ if f.Pragma&ir.Systemstack != 0 {
+ f.LSym.Set(obj.AttrCFunc, true)
+ }
isLinknameExported := nam.Sym().Linkname != "" && (hasBody || hasDefABI)
if abi, ok := symabiRefs[f.LSym.Name]; (ok && abi == obj.ABI0) || isLinknameExported {
// using linkname and we don't want to create
// duplicate ABI wrappers.
if f.LSym.ABI() != obj.ABI0 {
- needABIAlias, aliasABI = true, obj.ABI0
+ needABIWrapper, wrapperABI = true, obj.ABI0
}
}
- if needABIAlias {
- // These LSyms have the same name as the
- // native function, so we create them directly
- // rather than looking them up. The uniqueness
- // of f.lsym ensures uniqueness of asym.
- asym := &obj.LSym{
- Name: f.LSym.Name,
- Type: objabi.SABIALIAS,
- R: []obj.Reloc{{Sym: f.LSym}}, // 0 size, so "informational"
+ if needABIWrapper {
+ if !useABIWrapGen(f) {
+ // Fallback: use alias instead. FIXME.
+
+ // These LSyms have the same name as the
+ // native function, so we create them directly
+ // rather than looking them up. The uniqueness
+ // of f.lsym ensures uniqueness of asym.
+ asym := &obj.LSym{
+ Name: f.LSym.Name,
+ Type: objabi.SABIALIAS,
+ R: []obj.Reloc{{Sym: f.LSym}}, // 0 size, so "informational"
+ }
+ asym.SetABI(wrapperABI)
+ asym.Set(obj.AttrDuplicateOK, true)
+ base.Ctxt.ABIAliases = append(base.Ctxt.ABIAliases, asym)
+ } else {
+ if base.Debug.ABIWrap != 0 {
+ fmt.Fprintf(os.Stderr, "=-= %v to %v wrapper for %s.%s\n",
+ wrapperABI, 1-wrapperABI, types.LocalPkg.Path, f.LSym.Name)
+ }
+ makeABIWrapper(f, wrapperABI)
}
- asym.SetABI(aliasABI)
- asym.Set(obj.AttrDuplicateOK, true)
- base.Ctxt.ABIAliases = append(base.Ctxt.ABIAliases, asym)
}
}
+}
- if !hasBody {
- // For body-less functions, we only create the LSym.
- return
- }
-
- var flag int
+// setupTextLsym initializes the LSym for a with-body text symbol.
+func setupTextLSym(f *ir.Func, flag int) {
if f.Dupok() {
flag |= obj.DUPOK
}
initUniverse()
}
+
+// useNewABIWrapGen returns TRUE if the compiler should generate an
+// ABI wrapper for the function 'f'.
+func useABIWrapGen(f *ir.Func) bool {
+ if !base.Flag.ABIWrap {
+ return false
+ }
+
+ // Support limit option for bisecting.
+ if base.Flag.ABIWrapLimit == 1 {
+ return false
+ }
+ if base.Flag.ABIWrapLimit < 1 {
+ return true
+ }
+ base.Flag.ABIWrapLimit--
+ if base.Debug.ABIWrap != 0 && base.Flag.ABIWrapLimit == 1 {
+ fmt.Fprintf(os.Stderr, "=-= limit reached after new wrapper for %s\n",
+ f.LSym.Name)
+ }
+
+ return true
+}
return
}
lsym := base.Ctxt.Lookup(fn.LSym.Name + ".args_stackmap")
-
nptr := int(fn.Type().ArgWidth() / int64(Widthptr))
bv := bvalloc(int32(nptr) * 2)
nbitmap := 1
fn := curfn.(*ir.Func)
if fn.Nname != nil {
- if expect := fn.Sym().Linksym(); fnsym != expect {
+ expect := fn.Sym().Linksym()
+ if fnsym.ABI() == obj.ABI0 {
+ expect = fn.Sym().LinksymABI0()
+ }
+ if fnsym != expect {
base.Fatalf("unexpected fnsym: %v != %v", fnsym, expect)
}
}
}
func instrument(fn *ir.Func) {
- if fn.Pragma&ir.Norace != 0 {
+ if fn.Pragma&ir.Norace != 0 || (fn.Sym().Linksym() != nil && fn.Sym().Linksym().ABIWrapper()) {
return
}
case ir.ORETJMP:
b := s.exit()
b.Kind = ssa.BlockRetJmp // override BlockRet
- b.Aux = n.Sym().Linksym()
+ b.Aux = callTargetLSym(n.Sym(), s.curfn.LSym)
case ir.OCONTINUE, ir.OBREAK:
var to *ssa.Block
}
case sym != nil:
if testLateExpansion {
- aux := ssa.StaticAuxCall(sym.Linksym(), ACArgs, ACResults)
+ aux := ssa.StaticAuxCall(callTargetLSym(sym, s.curfn.LSym), ACArgs, ACResults)
call = s.newValue0A(ssa.OpStaticLECall, aux.LateExpansionResultType(), aux)
call.AddArgs(callArgs...)
} else {
- call = s.newValue1A(ssa.OpStaticCall, types.TypeMem, ssa.StaticAuxCall(sym.Linksym(), ACArgs, ACResults), s.mem())
+ call = s.newValue1A(ssa.OpStaticCall, types.TypeMem, ssa.StaticAuxCall(callTargetLSym(sym, s.curfn.LSym), ACArgs, ACResults), s.mem())
}
default:
s.Fatalf("bad call type %v %v", n.Op(), n)
}
return n
}
+
+// callTargetLSym determines the correct LSym for 'callee' when called
+// from function 'caller'. There are a couple of different scenarios
+// to contend with here:
+//
+// 1. if 'caller' is an ABI wrapper, then we always want to use the
+// LSym from the Func for the callee.
+//
+// 2. if 'caller' is not an ABI wrapper, then we looked at the callee
+// to see if it corresponds to a "known" ABI0 symbol (e.g. assembly
+// routine defined in the current package); if so, we want the call to
+// directly target the ABI0 symbol (effectively bypassing the
+// ABIInternal->ABI0 wrapper for 'callee').
+//
+// 3. in all other cases, want the regular ABIInternal linksym
+//
+func callTargetLSym(callee *types.Sym, callerLSym *obj.LSym) *obj.LSym {
+ lsym := callee.Linksym()
+ if !base.Flag.ABIWrap {
+ return lsym
+ }
+ if ir.AsNode(callee.Def) == nil {
+ return lsym
+ }
+ ndclfunc := ir.AsNode(callee.Def).Name().Defn
+ if ndclfunc == nil {
+ return lsym
+ }
+ // check for case 1 above
+ if callerLSym.ABIWrapper() {
+ if nlsym := ndclfunc.Func().LSym; nlsym != nil {
+ lsym = nlsym
+ }
+ } else {
+ // check for case 2 above
+ nam := ndclfunc.Func().Nname
+ defABI, hasDefABI := symabiDefs[nam.Sym().LinksymName()]
+ if hasDefABI && defABI == obj.ABI0 {
+ lsym = nam.Sym().LinksymABI0()
+ }
+ }
+ return lsym
+}
return base.Ctxt.LookupInit(sym.LinksymName(), initPkg)
}
+// LinksymABI0 looks up or creates an ABI0 linker symbol for "sym",
+// in cases where we want to specifically select the ABI0 version of
+// a symbol (typically used only for ABI wrappers).
+func (sym *Sym) LinksymABI0() *obj.LSym {
+ if sym == nil {
+ return nil
+ }
+ initPkg := func(r *obj.LSym) {
+ if sym.Linkname != "" {
+ r.Pkg = "_"
+ } else {
+ r.Pkg = sym.Pkg.Prefix
+ }
+ }
+ return base.Ctxt.LookupABIInit(sym.LinksymName(), obj.ABI0, initPkg)
+}
+
// Less reports whether symbol a is ordered before symbol b.
//
// Symbols are ordered exported before non-exported, then by name, and
// ContentAddressable indicates this is a content-addressable symbol.
AttrContentAddressable
+ // ABI wrapper is set for compiler-generated text symbols that
+ // convert between ABI0 and ABIInternal calling conventions.
+ AttrABIWrapper
+
// attrABIBase is the value at which the ABI is encoded in
// Attribute. This must be last; all bits after this are
// assumed to be an ABI value.
func (a Attribute) Indexed() bool { return a&AttrIndexed != 0 }
func (a Attribute) UsedInIface() bool { return a&AttrUsedInIface != 0 }
func (a Attribute) ContentAddressable() bool { return a&AttrContentAddressable != 0 }
+func (a Attribute) ABIWrapper() bool { return a&AttrABIWrapper != 0 }
func (a *Attribute) Set(flag Attribute, value bool) {
if value {
{bit: AttrTopFrame, s: "TOPFRAME"},
{bit: AttrIndexed, s: ""},
{bit: AttrContentAddressable, s: ""},
+ {bit: AttrABIWrapper, s: "ABIWRAPPER"},
}
// TextAttrString formats a for printing in as part of a TEXT prog.
if !strings.HasPrefix(s.Name, "\"\".") {
continue
}
+ if s.ABIWrapper() {
+ // Don't create an args_stackmap symbol reference for an ABI
+ // wrapper function
+ continue
+ }
found := false
for p := s.Func().Text; p != nil; p = p.Link {
if p.As == AFUNCDATA && p.From.Type == TYPE_CONST && p.From.Offset == objabi.FUNCDATA_ArgsPointerMaps {
s.Set(AttrNoSplit, flag&NOSPLIT != 0)
s.Set(AttrReflectMethod, flag&REFLECTMETHOD != 0)
s.Set(AttrWrapper, flag&WRAPPER != 0)
+ s.Set(AttrABIWrapper, flag&ABIWRAPPER != 0)
s.Set(AttrNeedCtxt, flag&NEEDCTXT != 0)
s.Set(AttrNoFrame, flag&NOFRAME != 0)
s.Set(AttrTopFrame, flag&TOPFRAME != 0)
// Function is the top of the call stack. Call stack unwinders should stop
// at this function.
TOPFRAME = 2048
+
+ // Function is an ABI wrapper.
+ ABIWRAPPER = 4096
)
}
}
- if !p.From.Sym.NoSplit() || p.From.Sym.Wrapper() {
+ if !p.From.Sym.NoSplit() || (p.From.Sym.Wrapper() && !p.From.Sym.ABIWrapper()) {
p = obj.Appendp(p, newprog)
p = load_g_cx(ctxt, p, newprog) // load g into CX
}
p.To.Reg = REG_BP
}
- if cursym.Func().Text.From.Sym.Wrapper() {
+ if cursym.Func().Text.From.Sym.Wrapper() && !cursym.Func().Text.From.Sym.ABIWrapper() {
// if g._panic != nil && g._panic.argp == FP {
// g._panic.argp = bottom-of-frame
// }
FlagRound = flag.Int("R", -1, "set address rounding `quantum`")
FlagTextAddr = flag.Int64("T", -1, "set text segment `address`")
flagEntrySymbol = flag.String("E", "", "set `entry` symbol name")
-
- cpuprofile = flag.String("cpuprofile", "", "write cpu profile to `file`")
- memprofile = flag.String("memprofile", "", "write memory profile to `file`")
- memprofilerate = flag.Int64("memprofilerate", 0, "set runtime.MemProfileRate to `rate`")
-
+ cpuprofile = flag.String("cpuprofile", "", "write cpu profile to `file`")
+ memprofile = flag.String("memprofile", "", "write memory profile to `file`")
+ memprofilerate = flag.Int64("memprofilerate", 0, "set runtime.MemProfileRate to `rate`")
+ flagAbiWrap = false
benchmarkFlag = flag.String("benchmark", "", "set to 'mem' or 'cpu' to enable phase benchmarking")
benchmarkFileFlag = flag.String("benchmarkprofile", "", "emit phase profiles to `base`_phase.{cpu,mem}prof")
)
objabi.Flagfn1("X", "add string value `definition` of the form importpath.name=value", func(s string) { addstrdata1(ctxt, s) })
objabi.Flagcount("v", "print link trace", &ctxt.Debugvlog)
objabi.Flagfn1("importcfg", "read import configuration from `file`", ctxt.readImportCfg)
+ if objabi.Regabi_enabled != 0 {
+ flag.BoolVar(&flagAbiWrap, "abiwrap", true, "support ABI wrapper functions")
+ }
objabi.Flagparse(usage)
elfshnum = xosect.Elfsect.(*ElfShdr).shnum
}
+ sname := ldr.SymExtname(x)
+
+ // For functions with ABI wrappers, we have to make sure that we
+ // don't wind up with two elf symbol table entries with the same
+ // name (since this will generated an error from the external
+ // linker). In the CgoExportStatic case, we want the ABI0 symbol
+ // to have the primary symbol table entry (since it's going to be
+ // called from C), so we rename the ABIInternal symbol. In all
+ // other cases, we rename the ABI0 symbol, since we want
+ // cross-load-module calls to target ABIInternal.
+ //
+ // TODO: generalize this for non-ELF (put the rename code in the
+ // loader, and store the rename result in SymExtname).
+ //
+ // TODO: avoid the ldr.Lookup calls below by instead using an aux
+ // sym or marker relocation to associate the wrapper with the
+ // wrapped function.
+ //
+ if flagAbiWrap {
+ if !ldr.IsExternal(x) && ldr.SymType(x) == sym.STEXT {
+ // First case
+ if ldr.SymVersion(x) == sym.SymVerABIInternal {
+ if s2 := ldr.Lookup(sname, sym.SymVerABI0); s2 != 0 && ldr.AttrCgoExportStatic(s2) && ldr.SymType(s2) == sym.STEXT {
+ sname = sname + ".abiinternal"
+ }
+ }
+ // Second case
+ if ldr.SymVersion(x) == sym.SymVerABI0 && !ldr.AttrCgoExportStatic(x) {
+ if s2 := ldr.Lookup(sname, sym.SymVerABIInternal); s2 != 0 && ldr.SymType(s2) == sym.STEXT {
+ sname = sname + ".abi0"
+ }
+ }
+ }
+ }
+
// One pass for each binding: elf.STB_LOCAL, elf.STB_GLOBAL,
// maybe one day elf.STB_WEAK.
bind := elf.STB_GLOBAL
other |= 3 << 5
}
- sname := ldr.SymExtname(x)
-
// When dynamically linking, we create Symbols by reading the names from
// the symbol tables of the shared libraries and so the names need to
// match exactly. Tools like DTrace will have to wait for now.
// Function is the top of the call stack. Call stack unwinders should stop
// at this function.
#define TOPFRAME 2048
+// Function is an ABI wrapper.
+#define ABIWRAPPER 4096
log.Fatal(err)
}
- cmd := exec.Command("go", "build")
+ // Turn off ABI0 wrapper generation for now. The problem here is
+ // that in these test cases main.main is an assembly routine,
+ // thus calls to it will have to go through an ABI wrapper. The
+ // ABI wrapper will consume some stack space, which throws off
+ // the numbers.
+ workaround := "-gcflags=-abiwrap=0"
+
+ cmd := exec.Command("go", "build", workaround)
cmd.Dir = dir
output, err := cmd.CombinedOutput()
if err == nil {