// If ha.noMorePtrs[i]>>j&1 is set, the entries in ha.bitmap[8*i+j+1] and
// beyond must all be zero until the start of the next object.
//
-// The bitmap for noscan spans is not maintained (can be junk). Code must
-// ensure that an object is scannable before consulting its bitmap by
-// checking either the noscan bit in the span or by consulting its
-// type's information.
+// The bitmap for noscan spans is set to all zero at span allocation time.
//
-// The bitmap for unallocated objects is also not maintained.
+// The bitmap for unallocated objects in scannable spans is not maintained
+// (can be junk).
package runtime
"unsafe"
)
+// heapArenaPtrScalar contains the per-heapArena pointer/scalar metadata for the GC.
+type heapArenaPtrScalar struct {
+ // bitmap stores the pointer/scalar bitmap for the words in
+ // this arena. See mbitmap.go for a description.
+ // This array uses 1 bit per word of heap, or 1.6% of the heap size (for 64-bit).
+ bitmap [heapArenaBitmapWords]uintptr
+
+ // If the ith bit of noMorePtrs is true, then there are no more
+ // pointers for the object containing the word described by the
+ // high bit of bitmap[i].
+ // In that case, bitmap[i+1], ... must be zero until the start
+ // of the next object.
+ // We never operate on these entries using bit-parallel techniques,
+ // so it is ok if they are small. Also, they can't be bigger than
+ // uint16 because at that size a single noMorePtrs entry
+ // represents 8K of memory, the minimum size of a span. Any larger
+ // and we'd have to worry about concurrent updates.
+ // This array uses 1 bit per word of bitmap, or .024% of the heap size (for 64-bit).
+ noMorePtrs [heapArenaBitmapWords / 8]uint8
+}
+
// addb returns the byte pointer p+n.
//
//go:nowritebarrier
// and negates them so that ctz (count trailing zeros) instructions
// can be used. It then places these 8 bytes into the cached 64 bit
// s.allocCache.
-func (s *mspan) refillAllocCache(whichByte uintptr) {
- bytes := (*[8]uint8)(unsafe.Pointer(s.allocBits.bytep(whichByte)))
+func (s *mspan) refillAllocCache(whichByte uint16) {
+ bytes := (*[8]uint8)(unsafe.Pointer(s.allocBits.bytep(uintptr(whichByte))))
aCache := uint64(0)
aCache |= uint64(bytes[0])
aCache |= uint64(bytes[1]) << (1 * 8)
// or after s.freeindex.
// There are hardware instructions that can be used to make this
// faster if profiling warrants it.
-func (s *mspan) nextFreeIndex() uintptr {
+func (s *mspan) nextFreeIndex() uint16 {
sfreeindex := s.freeindex
snelems := s.nelems
if sfreeindex == snelems {
aCache := s.allocCache
- bitIndex := sys.Ctz64(aCache)
+ bitIndex := sys.TrailingZeros64(aCache)
for bitIndex == 64 {
// Move index to start of next cached bits.
sfreeindex = (sfreeindex + 64) &^ (64 - 1)
// Refill s.allocCache with the next 64 alloc bits.
s.refillAllocCache(whichByte)
aCache = s.allocCache
- bitIndex = sys.Ctz64(aCache)
+ bitIndex = sys.TrailingZeros64(aCache)
// nothing available in cached bits
// grab the next 8 bytes and try again.
}
- result := sfreeindex + uintptr(bitIndex)
+ result := sfreeindex + uint16(bitIndex)
if result >= snelems {
s.freeindex = snelems
return snelems
// been no preemption points since ensuring this (which could allow a
// GC transition, which would allow the state to change).
func (s *mspan) isFree(index uintptr) bool {
- if index < s.freeindex {
+ if index < uintptr(s.freeIndexForScan) {
return false
}
bytep, mask := s.allocBits.bitp(index)
// n must be within [0, s.npages*_PageSize),
// or may be exactly s.npages*_PageSize
// if s.elemsize is from sizeclasses.go.
+//
+// nosplit, because it is called by objIndex, which is nosplit
+//
+//go:nosplit
func (s *mspan) divideByElemSize(n uintptr) uintptr {
const doubleCheck = false
return q
}
+// nosplit, because it is called by other nosplit code like findObject
+//
+//go:nosplit
func (s *mspan) objIndex(p uintptr) uintptr {
return s.divideByElemSize(p - s.base())
}
}
func (s *mspan) markBitsForBase() markBits {
- return markBits{(*uint8)(s.gcmarkBits), uint8(1), 0}
+ return markBits{&s.gcmarkBits.x, uint8(1), 0}
}
// isMarked reports whether mark bit m is set.
return
}
-// verifyNotInHeapPtr reports whether converting the not-in-heap pointer into a unsafe.Pointer is ok.
+// reflect_verifyNotInHeapPtr reports whether converting the not-in-heap pointer into a unsafe.Pointer is ok.
//
//go:linkname reflect_verifyNotInHeapPtr reflect.verifyNotInHeapPtr
func reflect_verifyNotInHeapPtr(p uintptr) bool {
if h.mask != 0 {
var i int
if goarch.PtrSize == 8 {
- i = sys.Ctz64(uint64(h.mask))
+ i = sys.TrailingZeros64(uint64(h.mask))
} else {
- i = sys.Ctz32(uint32(h.mask))
+ i = sys.TrailingZeros32(uint32(h.mask))
}
h.mask ^= uintptr(1) << (i & (ptrBits - 1))
return h, h.addr + uintptr(i)*goarch.PtrSize
// nextFast is like next, but can return 0 even when there are more pointers
// to be found. Callers should call next if nextFast returns 0 as its second
// return value.
-// if addr, h = h.nextFast(); addr == 0 {
-// if addr, h = h.next(); addr == 0 {
-// ... no more pointers ...
-// }
-// }
-// ... process pointer at addr ...
+//
+// if addr, h = h.nextFast(); addr == 0 {
+// if addr, h = h.next(); addr == 0 {
+// ... no more pointers ...
+// }
+// }
+// ... process pointer at addr ...
+//
// nextFast is designed to be inlineable.
//
//go:nosplit
// BSFQ
var i int
if goarch.PtrSize == 8 {
- i = sys.Ctz64(uint64(h.mask))
+ i = sys.TrailingZeros64(uint64(h.mask))
} else {
- i = sys.Ctz32(uint32(h.mask))
+ i = sys.TrailingZeros32(uint32(h.mask))
}
// BTCQ
h.mask ^= uintptr(1) << (i & (ptrBits - 1))
// The pointer bitmap is not maintained for allocations containing
// no pointers at all; any caller of bulkBarrierPreWrite must first
// make sure the underlying allocation contains pointers, usually
-// by checking typ.ptrdata.
+// by checking typ.PtrBytes.
//
-// Callers must perform cgo checks if writeBarrier.cgo.
+// Callers must perform cgo checks if goexperiment.CgoCheck2.
//
//go:nosplit
func bulkBarrierPreWrite(dst, src, size uintptr) {
if (dst|src|size)&(goarch.PtrSize-1) != 0 {
throw("bulkBarrierPreWrite: unaligned arguments")
}
- if !writeBarrier.needed {
+ if !writeBarrier.enabled {
return
}
if s := spanOf(dst); s == nil {
break
}
dstx := (*uintptr)(unsafe.Pointer(addr))
- if !buf.putFast(*dstx, 0) {
- wbBufFlush(nil, 0)
- }
+ p := buf.get1()
+ p[0] = *dstx
}
} else {
for {
}
dstx := (*uintptr)(unsafe.Pointer(addr))
srcx := (*uintptr)(unsafe.Pointer(src + (addr - dst)))
- if !buf.putFast(*dstx, *srcx) {
- wbBufFlush(nil, 0)
- }
+ p := buf.get2()
+ p[0] = *dstx
+ p[1] = *srcx
}
}
}
if (dst|src|size)&(goarch.PtrSize-1) != 0 {
throw("bulkBarrierPreWrite: unaligned arguments")
}
- if !writeBarrier.needed {
+ if !writeBarrier.enabled {
return
}
buf := &getg().m.p.ptr().wbBuf
break
}
srcx := (*uintptr)(unsafe.Pointer(addr - dst + src))
- if !buf.putFast(0, *srcx) {
- wbBufFlush(nil, 0)
- }
+ p := buf.get1()
+ p[0] = *srcx
}
}
if *bits&mask != 0 {
dstx := (*uintptr)(unsafe.Pointer(dst + i))
if src == 0 {
- if !buf.putFast(*dstx, 0) {
- wbBufFlush(nil, 0)
- }
+ p := buf.get1()
+ p[0] = *dstx
} else {
srcx := (*uintptr)(unsafe.Pointer(src + i))
- if !buf.putFast(*dstx, *srcx) {
- wbBufFlush(nil, 0)
- }
+ p := buf.get2()
+ p[0] = *dstx
+ p[1] = *srcx
}
}
mask <<= 1
// Must not be preempted because it typically runs right before memmove,
// and the GC must observe them as an atomic action.
//
-// Callers must perform cgo checks if writeBarrier.cgo.
+// Callers must perform cgo checks if goexperiment.CgoCheck2.
//
//go:nosplit
func typeBitsBulkBarrier(typ *_type, dst, src, size uintptr) {
if typ == nil {
throw("runtime: typeBitsBulkBarrier without type")
}
- if typ.size != size {
- println("runtime: typeBitsBulkBarrier with type ", typ.string(), " of size ", typ.size, " but memory size", size)
+ if typ.Size_ != size {
+ println("runtime: typeBitsBulkBarrier with type ", toRType(typ).string(), " of size ", typ.Size_, " but memory size", size)
throw("runtime: invalid typeBitsBulkBarrier")
}
- if typ.kind&kindGCProg != 0 {
- println("runtime: typeBitsBulkBarrier with type ", typ.string(), " with GC prog")
+ if typ.Kind_&kindGCProg != 0 {
+ println("runtime: typeBitsBulkBarrier with type ", toRType(typ).string(), " with GC prog")
throw("runtime: invalid typeBitsBulkBarrier")
}
- if !writeBarrier.needed {
+ if !writeBarrier.enabled {
return
}
- ptrmask := typ.gcdata
+ ptrmask := typ.GCData
buf := &getg().m.p.ptr().wbBuf
var bits uint32
- for i := uintptr(0); i < typ.ptrdata; i += goarch.PtrSize {
+ for i := uintptr(0); i < typ.PtrBytes; i += goarch.PtrSize {
if i&(goarch.PtrSize*8-1) == 0 {
bits = uint32(*ptrmask)
ptrmask = addb(ptrmask, 1)
if bits&1 != 0 {
dstx := (*uintptr)(unsafe.Pointer(dst + i))
srcx := (*uintptr)(unsafe.Pointer(src + i))
- if !buf.putFast(*dstx, *srcx) {
- wbBufFlush(nil, 0)
- }
+ p := buf.get2()
+ p[0] = *dstx
+ p[1] = *srcx
}
}
}
// initHeapBits initializes the heap bitmap for a span.
// If this is a span of single pointer allocations, it initializes all
-// words to pointer.
-func (s *mspan) initHeapBits() {
+// words to pointer. If force is true, clears all bits.
+func (s *mspan) initHeapBits(forceClear bool) {
+ if forceClear || s.spanclass.noscan() {
+ // Set all the pointer bits to zero. We do this once
+ // when the span is allocated so we don't have to do it
+ // for each object allocation.
+ base := s.base()
+ size := s.npages * pageSize
+ h := writeHeapBitsForAddr(base)
+ h.flush(base, size)
+ return
+ }
isPtrs := goarch.PtrSize == 8 && s.elemsize == goarch.PtrSize
if !isPtrs {
return // nothing to do
// scanning the allocation bitmap.
func (s *mspan) countAlloc() int {
count := 0
- bytes := divRoundUp(s.nelems, 8)
+ bytes := divRoundUp(uintptr(s.nelems), 8)
// Iterate over each 8-byte chunk and count allocations
// with an intrinsic. Note that newMarkBits guarantees that
// gcmarkBits will be 8-byte aligned, so we don't have to
// Continue on writing zeros for the rest of the object.
// For standard use of the ptr bits this is not required, as
// the bits are read from the beginning of the object. Some uses,
- // like oblets, bulk write barriers, and cgocheck, might
+ // like noscan spans, oblets, bulk write barriers, and cgocheck, might
// start mid-object, so these writes are still required.
for {
// Write zero bits.
// heapBitsSetType records that the new allocation [x, x+size)
// holds in [x, x+dataSize) one or more values of type typ.
-// (The number of values is given by dataSize / typ.size.)
+// (The number of values is given by dataSize / typ.Size.)
// If dataSize < size, the fragment [x+dataSize, x+size) is
// recorded as non-pointer data.
// It is known that the type has pointers somewhere;
func heapBitsSetType(x, size, dataSize uintptr, typ *_type) {
const doubleCheck = false // slow but helpful; enable to test modifications to this code
- if doubleCheck && dataSize%typ.size != 0 {
- throw("heapBitsSetType: dataSize not a multiple of typ.size")
+ if doubleCheck && dataSize%typ.Size_ != 0 {
+ throw("heapBitsSetType: dataSize not a multiple of typ.Size")
}
if goarch.PtrSize == 8 && size == goarch.PtrSize {
// It's one word and it has pointers, it must be a pointer.
// Since all allocated one-word objects are pointers
// (non-pointers are aggregated into tinySize allocations),
- // initSpan sets the pointer bits for us. Nothing to do here.
+ // (*mspan).initHeapBits sets the pointer bits for us.
+ // Nothing to do here.
if doubleCheck {
h, addr := heapBitsForAddr(x, size).next()
if addr != x {
h := writeHeapBitsForAddr(x)
// Handle GC program.
- if typ.kind&kindGCProg != 0 {
+ if typ.Kind_&kindGCProg != 0 {
// Expand the gc program into the storage we're going to use for the actual object.
obj := (*uint8)(unsafe.Pointer(x))
- n := runGCProg(addb(typ.gcdata, 4), obj)
+ n := runGCProg(addb(typ.GCData, 4), obj)
// Use the expanded program to set the heap bits.
- for i := uintptr(0); true; i += typ.size {
+ for i := uintptr(0); true; i += typ.Size_ {
// Copy expanded program to heap bitmap.
p := obj
j := n
}
h = h.write(uintptr(*p), j)
- if i+typ.size == dataSize {
+ if i+typ.Size_ == dataSize {
break // no padding after last element
}
// Pad with zeros to the start of the next element.
- h = h.pad(typ.size - n*goarch.PtrSize)
+ h = h.pad(typ.Size_ - n*goarch.PtrSize)
}
h.flush(x, size)
// Note about sizes:
//
- // typ.size is the number of words in the object,
- // and typ.ptrdata is the number of words in the prefix
+ // typ.Size is the number of words in the object,
+ // and typ.PtrBytes is the number of words in the prefix
// of the object that contains pointers. That is, the final
- // typ.size - typ.ptrdata words contain no pointers.
+ // typ.Size - typ.PtrBytes words contain no pointers.
// This allows optimization of a common pattern where
// an object has a small header followed by a large scalar
// buffer. If we know the pointers are over, we don't have
// to scan the buffer's heap bitmap at all.
// The 1-bit ptrmasks are sized to contain only bits for
- // the typ.ptrdata prefix, zero padded out to a full byte
+ // the typ.PtrBytes prefix, zero padded out to a full byte
// of bitmap. If there is more room in the allocated object,
// that space is pointerless. The noMorePtrs bitmap will prevent
// scanning large pointerless tails of an object.
// objects with scalar tails, all but the last tail does have to
// be initialized, because there is no way to say "skip forward".
- ptrs := typ.ptrdata / goarch.PtrSize
- if typ.size == dataSize { // Single element
+ ptrs := typ.PtrBytes / goarch.PtrSize
+ if typ.Size_ == dataSize { // Single element
if ptrs <= ptrBits { // Single small element
- m := readUintptr(typ.gcdata)
+ m := readUintptr(typ.GCData)
h = h.write(m, ptrs)
} else { // Single large element
- p := typ.gcdata
+ p := typ.GCData
for {
h = h.write(readUintptr(p), ptrBits)
p = addb(p, ptrBits/8)
h = h.write(m, ptrs)
}
} else { // Repeated element
- words := typ.size / goarch.PtrSize // total words, including scalar tail
- if words <= ptrBits { // Repeated small element
- n := dataSize / typ.size
- m := readUintptr(typ.gcdata)
+ words := typ.Size_ / goarch.PtrSize // total words, including scalar tail
+ if words <= ptrBits { // Repeated small element
+ n := dataSize / typ.Size_
+ m := readUintptr(typ.GCData)
// Make larger unit to repeat
for words <= ptrBits/2 {
if n&1 != 0 {
}
h = h.write(m, ptrs)
} else { // Repeated large element
- for i := uintptr(0); true; i += typ.size {
- p := typ.gcdata
+ for i := uintptr(0); true; i += typ.Size_ {
+ p := typ.GCData
j := ptrs
for j > ptrBits {
h = h.write(readUintptr(p), ptrBits)
}
m := readUintptr(p)
h = h.write(m, j)
- if i+typ.size == dataSize {
+ if i+typ.Size_ == dataSize {
break // don't need the trailing nonptr bits on the last element.
}
// Pad with zeros to the start of the next element.
- h = h.pad(typ.size - typ.ptrdata)
+ h = h.pad(typ.Size_ - typ.PtrBytes)
}
}
}
// Compute the pointer bit we want at offset i.
want := false
if i < dataSize {
- off := i % typ.size
- if off < typ.ptrdata {
+ off := i % typ.Size_
+ if off < typ.PtrBytes {
j := off / goarch.PtrSize
- want = *addb(typ.gcdata, j/8)>>(j%8)&1 != 0
+ want = *addb(typ.GCData, j/8)>>(j%8)&1 != 0
}
}
if want {
// Testing.
-func getgcmaskcb(frame *stkframe, ctxt unsafe.Pointer) bool {
- target := (*stkframe)(ctxt)
- if frame.sp <= target.sp && target.sp < frame.varp {
- *target = *frame
- return false
- }
- return true
-}
-
-// gcbits returns the GC type info for x, for testing.
+// reflect_gcbits returns the GC type info for x, for testing.
// The result is the bitmap entries (0 or 1), one entry per byte.
//
//go:linkname reflect_gcbits reflect.gcbits
// data
if datap.data <= uintptr(p) && uintptr(p) < datap.edata {
bitmap := datap.gcdatamask.bytedata
- n := (*ptrtype)(unsafe.Pointer(t)).elem.size
+ n := (*ptrtype)(unsafe.Pointer(t)).Elem.Size_
mask = make([]byte, n/goarch.PtrSize)
for i := uintptr(0); i < n; i += goarch.PtrSize {
off := (uintptr(p) + i - datap.data) / goarch.PtrSize
// bss
if datap.bss <= uintptr(p) && uintptr(p) < datap.ebss {
bitmap := datap.gcbssmask.bytedata
- n := (*ptrtype)(unsafe.Pointer(t)).elem.size
+ n := (*ptrtype)(unsafe.Pointer(t)).Elem.Size_
mask = make([]byte, n/goarch.PtrSize)
for i := uintptr(0); i < n; i += goarch.PtrSize {
off := (uintptr(p) + i - datap.bss) / goarch.PtrSize
// stack
if gp := getg(); gp.m.curg.stack.lo <= uintptr(p) && uintptr(p) < gp.m.curg.stack.hi {
- var frame stkframe
- frame.sp = uintptr(p)
- gentraceback(gp.m.curg.sched.pc, gp.m.curg.sched.sp, 0, gp.m.curg, 0, nil, 1000, getgcmaskcb, noescape(unsafe.Pointer(&frame)), 0)
- if frame.fn.valid() {
- locals, _, _ := getStackMap(&frame, nil, false)
+ found := false
+ var u unwinder
+ for u.initAt(gp.m.curg.sched.pc, gp.m.curg.sched.sp, 0, gp.m.curg, 0); u.valid(); u.next() {
+ if u.frame.sp <= uintptr(p) && uintptr(p) < u.frame.varp {
+ found = true
+ break
+ }
+ }
+ if found {
+ locals, _, _ := u.frame.getStackMap(false)
if locals.n == 0 {
return
}
size := uintptr(locals.n) * goarch.PtrSize
- n := (*ptrtype)(unsafe.Pointer(t)).elem.size
+ n := (*ptrtype)(unsafe.Pointer(t)).Elem.Size_
mask = make([]byte, n/goarch.PtrSize)
for i := uintptr(0); i < n; i += goarch.PtrSize {
- off := (uintptr(p) + i - frame.varp + size) / goarch.PtrSize
+ off := (uintptr(p) + i - u.frame.varp + size) / goarch.PtrSize
mask[i/goarch.PtrSize] = locals.ptrbit(off)
}
}
// must not have pointers
return
}
+
+// userArenaHeapBitsSetType is the equivalent of heapBitsSetType but for
+// non-slice-backing-store Go values allocated in a user arena chunk. It
+// sets up the heap bitmap for the value with type typ allocated at address ptr.
+// base is the base address of the arena chunk.
+func userArenaHeapBitsSetType(typ *_type, ptr unsafe.Pointer, base uintptr) {
+ h := writeHeapBitsForAddr(uintptr(ptr))
+
+ // Our last allocation might have ended right at a noMorePtrs mark,
+ // which we would not have erased. We need to erase that mark here,
+ // because we're going to start adding new heap bitmap bits.
+ // We only need to clear one mark, because below we make sure to
+ // pad out the bits with zeroes and only write one noMorePtrs bit
+ // for each new object.
+ // (This is only necessary at noMorePtrs boundaries, as noMorePtrs
+ // marks within an object allocated with newAt will be erased by
+ // the normal writeHeapBitsForAddr mechanism.)
+ //
+ // Note that we skip this if this is the first allocation in the
+ // arena because there's definitely no previous noMorePtrs mark
+ // (in fact, we *must* do this, because we're going to try to back
+ // up a pointer to fix this up).
+ if uintptr(ptr)%(8*goarch.PtrSize*goarch.PtrSize) == 0 && uintptr(ptr) != base {
+ // Back up one pointer and rewrite that pointer. That will
+ // cause the writeHeapBits implementation to clear the
+ // noMorePtrs bit we need to clear.
+ r := heapBitsForAddr(uintptr(ptr)-goarch.PtrSize, goarch.PtrSize)
+ _, p := r.next()
+ b := uintptr(0)
+ if p == uintptr(ptr)-goarch.PtrSize {
+ b = 1
+ }
+ h = writeHeapBitsForAddr(uintptr(ptr) - goarch.PtrSize)
+ h = h.write(b, 1)
+ }
+
+ p := typ.GCData // start of 1-bit pointer mask (or GC program)
+ var gcProgBits uintptr
+ if typ.Kind_&kindGCProg != 0 {
+ // Expand gc program, using the object itself for storage.
+ gcProgBits = runGCProg(addb(p, 4), (*byte)(ptr))
+ p = (*byte)(ptr)
+ }
+ nb := typ.PtrBytes / goarch.PtrSize
+
+ for i := uintptr(0); i < nb; i += ptrBits {
+ k := nb - i
+ if k > ptrBits {
+ k = ptrBits
+ }
+ h = h.write(readUintptr(addb(p, i/8)), k)
+ }
+ // Note: we call pad here to ensure we emit explicit 0 bits
+ // for the pointerless tail of the object. This ensures that
+ // there's only a single noMorePtrs mark for the next object
+ // to clear. We don't need to do this to clear stale noMorePtrs
+ // markers from previous uses because arena chunk pointer bitmaps
+ // are always fully cleared when reused.
+ h = h.pad(typ.Size_ - typ.PtrBytes)
+ h.flush(uintptr(ptr), typ.Size_)
+
+ if typ.Kind_&kindGCProg != 0 {
+ // Zero out temporary ptrmask buffer inside object.
+ memclrNoHeapPointers(ptr, (gcProgBits+7)/8)
+ }
+
+ // Double-check that the bitmap was written out correctly.
+ //
+ // Derived from heapBitsSetType.
+ const doubleCheck = false
+ if doubleCheck {
+ size := typ.Size_
+ x := uintptr(ptr)
+ h := heapBitsForAddr(x, size)
+ for i := uintptr(0); i < size; i += goarch.PtrSize {
+ // Compute the pointer bit we want at offset i.
+ want := false
+ off := i % typ.Size_
+ if off < typ.PtrBytes {
+ j := off / goarch.PtrSize
+ want = *addb(typ.GCData, j/8)>>(j%8)&1 != 0
+ }
+ if want {
+ var addr uintptr
+ h, addr = h.next()
+ if addr != x+i {
+ throw("userArenaHeapBitsSetType: pointer entry not correct")
+ }
+ }
+ }
+ if _, addr := h.next(); addr != 0 {
+ throw("userArenaHeapBitsSetType: extra pointer")
+ }
+ }
+}