typVar = ssaMarker("typ")
okVar = ssaMarker("ok")
deferBitsVar = ssaMarker("deferBits")
+ hashVar = ssaMarker("hash")
)
// startBlock sets the current block we're generating code in to b.
case ir.OINTERFACESWITCH:
n := n.(*ir.InterfaceSwitchStmt)
+ typs := s.f.Config.Types
t := s.expr(n.RuntimeType)
- d := s.newValue1A(ssa.OpAddr, s.f.Config.Types.BytePtr, n.Descriptor, s.sb)
- r := s.rtcall(ir.Syms.InterfaceSwitch, true, []*types.Type{s.f.Config.Types.Int, s.f.Config.Types.BytePtr}, d, t)
+ d := s.newValue1A(ssa.OpAddr, typs.BytePtr, n.Descriptor, s.sb)
+
+ // Check the cache first.
+ var merge *ssa.Block
+ if base.Flag.N == 0 && rtabi.UseInterfaceSwitchCache(Arch.LinkArch.Name) {
+ // Note: we can only use the cache if we have the right atomic load instruction.
+ // Double-check that here.
+ if _, ok := intrinsics[intrinsicKey{Arch.LinkArch.Arch, "runtime/internal/atomic", "Loadp"}]; !ok {
+ s.Fatalf("atomic load not available")
+ }
+ merge = s.f.NewBlock(ssa.BlockPlain)
+ cacheHit := s.f.NewBlock(ssa.BlockPlain)
+ cacheMiss := s.f.NewBlock(ssa.BlockPlain)
+ loopHead := s.f.NewBlock(ssa.BlockPlain)
+ loopBody := s.f.NewBlock(ssa.BlockPlain)
+
+ // Pick right size ops.
+ var mul, and, add, zext ssa.Op
+ if s.config.PtrSize == 4 {
+ mul = ssa.OpMul32
+ and = ssa.OpAnd32
+ add = ssa.OpAdd32
+ zext = ssa.OpCopy
+ } else {
+ mul = ssa.OpMul64
+ and = ssa.OpAnd64
+ add = ssa.OpAdd64
+ zext = ssa.OpZeroExt32to64
+ }
+
+ // Load cache pointer out of descriptor, with an atomic load so
+ // we ensure that we see a fully written cache.
+ atomicLoad := s.newValue2(ssa.OpAtomicLoadPtr, types.NewTuple(typs.BytePtr, types.TypeMem), d, s.mem())
+ cache := s.newValue1(ssa.OpSelect0, typs.BytePtr, atomicLoad)
+ s.vars[memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, atomicLoad)
+
+ // Load hash from type.
+ hash := s.newValue2(ssa.OpLoad, typs.UInt32, s.newValue1I(ssa.OpOffPtr, typs.UInt32Ptr, 2*s.config.PtrSize, t), s.mem())
+ hash = s.newValue1(zext, typs.Uintptr, hash)
+ s.vars[hashVar] = hash
+ // Load mask from cache.
+ mask := s.newValue2(ssa.OpLoad, typs.Uintptr, cache, s.mem())
+ // Jump to loop head.
+ b := s.endBlock()
+ b.AddEdgeTo(loopHead)
+
+ // At loop head, get pointer to the cache entry.
+ // e := &cache.Entries[hash&mask]
+ s.startBlock(loopHead)
+ entries := s.newValue2(ssa.OpAddPtr, typs.UintptrPtr, cache, s.uintptrConstant(uint64(s.config.PtrSize)))
+ idx := s.newValue2(and, typs.Uintptr, s.variable(hashVar, typs.Uintptr), mask)
+ idx = s.newValue2(mul, typs.Uintptr, idx, s.uintptrConstant(uint64(3*s.config.PtrSize)))
+ e := s.newValue2(ssa.OpAddPtr, typs.UintptrPtr, entries, idx)
+ // hash++
+ s.vars[hashVar] = s.newValue2(add, typs.Uintptr, s.variable(hashVar, typs.Uintptr), s.uintptrConstant(1))
+
+ // Look for a cache hit.
+ // if e.Typ == t { goto hit }
+ eTyp := s.newValue2(ssa.OpLoad, typs.Uintptr, e, s.mem())
+ cmp1 := s.newValue2(ssa.OpEqPtr, typs.Bool, t, eTyp)
+ b = s.endBlock()
+ b.Kind = ssa.BlockIf
+ b.SetControl(cmp1)
+ b.AddEdgeTo(cacheHit)
+ b.AddEdgeTo(loopBody)
+
+ // Look for an empty entry, the tombstone for this hash table.
+ // if e.Typ == nil { goto miss }
+ s.startBlock(loopBody)
+ cmp2 := s.newValue2(ssa.OpEqPtr, typs.Bool, eTyp, s.constNil(typs.BytePtr))
+ b = s.endBlock()
+ b.Kind = ssa.BlockIf
+ b.SetControl(cmp2)
+ b.AddEdgeTo(cacheMiss)
+ b.AddEdgeTo(loopHead)
+
+ // On a hit, load the data fields of the cache entry.
+ // Case = e.Case
+ // Itab = e.Itab
+ s.startBlock(cacheHit)
+ eCase := s.newValue2(ssa.OpLoad, typs.Int, s.newValue1I(ssa.OpOffPtr, typs.IntPtr, s.config.PtrSize, e), s.mem())
+ eItab := s.newValue2(ssa.OpLoad, typs.BytePtr, s.newValue1I(ssa.OpOffPtr, typs.BytePtrPtr, 2*s.config.PtrSize, e), s.mem())
+ s.assign(n.Case, eCase, false, 0)
+ s.assign(n.Itab, eItab, false, 0)
+ b = s.endBlock()
+ b.AddEdgeTo(merge)
+
+ // On a miss, call into the runtime to get the answer.
+ s.startBlock(cacheMiss)
+ }
+
+ r := s.rtcall(ir.Syms.InterfaceSwitch, true, []*types.Type{typs.Int, typs.BytePtr}, d, t)
s.assign(n.Case, r[0], false, 0)
s.assign(n.Itab, r[1], false, 0)
+ if merge != nil {
+ // Cache hits merge in here.
+ b := s.endBlock()
+ b.Kind = ssa.BlockPlain
+ b.AddEdgeTo(merge)
+ s.startBlock(merge)
+ }
+
case ir.OCHECKNIL:
n := n.(*ir.UnaryExpr)
p := s.expr(n.X)