"unsafe"
)
+// heapArenaPtrScalar contains the per-heapArena pointer/scalar metadata for the GC.
+type heapArenaPtrScalar struct {
+ // bitmap stores the pointer/scalar bitmap for the words in
+ // this arena. See mbitmap.go for a description.
+ // This array uses 1 bit per word of heap, or 1.6% of the heap size (for 64-bit).
+ bitmap [heapArenaBitmapWords]uintptr
+
+ // If the ith bit of noMorePtrs is true, then there are no more
+ // pointers for the object containing the word described by the
+ // high bit of bitmap[i].
+ // In that case, bitmap[i+1], ... must be zero until the start
+ // of the next object.
+ // We never operate on these entries using bit-parallel techniques,
+ // so it is ok if they are small. Also, they can't be bigger than
+ // uint16 because at that size a single noMorePtrs entry
+ // represents 8K of memory, the minimum size of a span. Any larger
+ // and we'd have to worry about concurrent updates.
+ // This array uses 1 bit per word of bitmap, or .024% of the heap size (for 64-bit).
+ noMorePtrs [heapArenaBitmapWords / 8]uint8
+}
+
// addb returns the byte pointer p+n.
//
//go:nowritebarrier
// and negates them so that ctz (count trailing zeros) instructions
// can be used. It then places these 8 bytes into the cached 64 bit
// s.allocCache.
-func (s *mspan) refillAllocCache(whichByte uintptr) {
- bytes := (*[8]uint8)(unsafe.Pointer(s.allocBits.bytep(whichByte)))
+func (s *mspan) refillAllocCache(whichByte uint16) {
+ bytes := (*[8]uint8)(unsafe.Pointer(s.allocBits.bytep(uintptr(whichByte))))
aCache := uint64(0)
aCache |= uint64(bytes[0])
aCache |= uint64(bytes[1]) << (1 * 8)
// or after s.freeindex.
// There are hardware instructions that can be used to make this
// faster if profiling warrants it.
-func (s *mspan) nextFreeIndex() uintptr {
+func (s *mspan) nextFreeIndex() uint16 {
sfreeindex := s.freeindex
snelems := s.nelems
if sfreeindex == snelems {
// nothing available in cached bits
// grab the next 8 bytes and try again.
}
- result := sfreeindex + uintptr(bitIndex)
+ result := sfreeindex + uint16(bitIndex)
if result >= snelems {
s.freeindex = snelems
return snelems
// been no preemption points since ensuring this (which could allow a
// GC transition, which would allow the state to change).
func (s *mspan) isFree(index uintptr) bool {
- if index < s.freeIndexForScan {
+ if index < uintptr(s.freeIndexForScan) {
return false
}
bytep, mask := s.allocBits.bitp(index)
// n must be within [0, s.npages*_PageSize),
// or may be exactly s.npages*_PageSize
// if s.elemsize is from sizeclasses.go.
+//
+// nosplit, because it is called by objIndex, which is nosplit
+//
+//go:nosplit
func (s *mspan) divideByElemSize(n uintptr) uintptr {
const doubleCheck = false
return q
}
+// nosplit, because it is called by other nosplit code like findObject
+//
+//go:nosplit
func (s *mspan) objIndex(p uintptr) uintptr {
return s.divideByElemSize(p - s.base())
}
if (dst|src|size)&(goarch.PtrSize-1) != 0 {
throw("bulkBarrierPreWrite: unaligned arguments")
}
- if !writeBarrier.needed {
+ if !writeBarrier.enabled {
return
}
if s := spanOf(dst); s == nil {
if (dst|src|size)&(goarch.PtrSize-1) != 0 {
throw("bulkBarrierPreWrite: unaligned arguments")
}
- if !writeBarrier.needed {
+ if !writeBarrier.enabled {
return
}
buf := &getg().m.p.ptr().wbBuf
throw("runtime: typeBitsBulkBarrier without type")
}
if typ.Size_ != size {
- println("runtime: typeBitsBulkBarrier with type ", typ.string(), " of size ", typ.Size_, " but memory size", size)
+ println("runtime: typeBitsBulkBarrier with type ", toRType(typ).string(), " of size ", typ.Size_, " but memory size", size)
throw("runtime: invalid typeBitsBulkBarrier")
}
if typ.Kind_&kindGCProg != 0 {
- println("runtime: typeBitsBulkBarrier with type ", typ.string(), " with GC prog")
+ println("runtime: typeBitsBulkBarrier with type ", toRType(typ).string(), " with GC prog")
throw("runtime: invalid typeBitsBulkBarrier")
}
- if !writeBarrier.needed {
+ if !writeBarrier.enabled {
return
}
ptrmask := typ.GCData
// scanning the allocation bitmap.
func (s *mspan) countAlloc() int {
count := 0
- bytes := divRoundUp(s.nelems, 8)
+ bytes := divRoundUp(uintptr(s.nelems), 8)
// Iterate over each 8-byte chunk and count allocations
// with an intrinsic. Note that newMarkBits guarantees that
// gcmarkBits will be 8-byte aligned, so we don't have to
// data
if datap.data <= uintptr(p) && uintptr(p) < datap.edata {
bitmap := datap.gcdatamask.bytedata
- n := (*ptrtype)(unsafe.Pointer(t)).elem.Size_
+ n := (*ptrtype)(unsafe.Pointer(t)).Elem.Size_
mask = make([]byte, n/goarch.PtrSize)
for i := uintptr(0); i < n; i += goarch.PtrSize {
off := (uintptr(p) + i - datap.data) / goarch.PtrSize
// bss
if datap.bss <= uintptr(p) && uintptr(p) < datap.ebss {
bitmap := datap.gcbssmask.bytedata
- n := (*ptrtype)(unsafe.Pointer(t)).elem.Size_
+ n := (*ptrtype)(unsafe.Pointer(t)).Elem.Size_
mask = make([]byte, n/goarch.PtrSize)
for i := uintptr(0); i < n; i += goarch.PtrSize {
off := (uintptr(p) + i - datap.bss) / goarch.PtrSize
}
}
if found {
- locals, _, _ := u.frame.getStackMap(nil, false)
+ locals, _, _ := u.frame.getStackMap(false)
if locals.n == 0 {
return
}
size := uintptr(locals.n) * goarch.PtrSize
- n := (*ptrtype)(unsafe.Pointer(t)).elem.Size_
+ n := (*ptrtype)(unsafe.Pointer(t)).Elem.Size_
mask = make([]byte, n/goarch.PtrSize)
for i := uintptr(0); i < n; i += goarch.PtrSize {
off := (uintptr(p) + i - u.frame.varp + size) / goarch.PtrSize
// must not have pointers
return
}
+
+// userArenaHeapBitsSetType is the equivalent of heapBitsSetType but for
+// non-slice-backing-store Go values allocated in a user arena chunk. It
+// sets up the heap bitmap for the value with type typ allocated at address ptr.
+// base is the base address of the arena chunk.
+func userArenaHeapBitsSetType(typ *_type, ptr unsafe.Pointer, base uintptr) {
+ h := writeHeapBitsForAddr(uintptr(ptr))
+
+ // Our last allocation might have ended right at a noMorePtrs mark,
+ // which we would not have erased. We need to erase that mark here,
+ // because we're going to start adding new heap bitmap bits.
+ // We only need to clear one mark, because below we make sure to
+ // pad out the bits with zeroes and only write one noMorePtrs bit
+ // for each new object.
+ // (This is only necessary at noMorePtrs boundaries, as noMorePtrs
+ // marks within an object allocated with newAt will be erased by
+ // the normal writeHeapBitsForAddr mechanism.)
+ //
+ // Note that we skip this if this is the first allocation in the
+ // arena because there's definitely no previous noMorePtrs mark
+ // (in fact, we *must* do this, because we're going to try to back
+ // up a pointer to fix this up).
+ if uintptr(ptr)%(8*goarch.PtrSize*goarch.PtrSize) == 0 && uintptr(ptr) != base {
+ // Back up one pointer and rewrite that pointer. That will
+ // cause the writeHeapBits implementation to clear the
+ // noMorePtrs bit we need to clear.
+ r := heapBitsForAddr(uintptr(ptr)-goarch.PtrSize, goarch.PtrSize)
+ _, p := r.next()
+ b := uintptr(0)
+ if p == uintptr(ptr)-goarch.PtrSize {
+ b = 1
+ }
+ h = writeHeapBitsForAddr(uintptr(ptr) - goarch.PtrSize)
+ h = h.write(b, 1)
+ }
+
+ p := typ.GCData // start of 1-bit pointer mask (or GC program)
+ var gcProgBits uintptr
+ if typ.Kind_&kindGCProg != 0 {
+ // Expand gc program, using the object itself for storage.
+ gcProgBits = runGCProg(addb(p, 4), (*byte)(ptr))
+ p = (*byte)(ptr)
+ }
+ nb := typ.PtrBytes / goarch.PtrSize
+
+ for i := uintptr(0); i < nb; i += ptrBits {
+ k := nb - i
+ if k > ptrBits {
+ k = ptrBits
+ }
+ h = h.write(readUintptr(addb(p, i/8)), k)
+ }
+ // Note: we call pad here to ensure we emit explicit 0 bits
+ // for the pointerless tail of the object. This ensures that
+ // there's only a single noMorePtrs mark for the next object
+ // to clear. We don't need to do this to clear stale noMorePtrs
+ // markers from previous uses because arena chunk pointer bitmaps
+ // are always fully cleared when reused.
+ h = h.pad(typ.Size_ - typ.PtrBytes)
+ h.flush(uintptr(ptr), typ.Size_)
+
+ if typ.Kind_&kindGCProg != 0 {
+ // Zero out temporary ptrmask buffer inside object.
+ memclrNoHeapPointers(ptr, (gcProgBits+7)/8)
+ }
+
+ // Double-check that the bitmap was written out correctly.
+ //
+ // Derived from heapBitsSetType.
+ const doubleCheck = false
+ if doubleCheck {
+ size := typ.Size_
+ x := uintptr(ptr)
+ h := heapBitsForAddr(x, size)
+ for i := uintptr(0); i < size; i += goarch.PtrSize {
+ // Compute the pointer bit we want at offset i.
+ want := false
+ off := i % typ.Size_
+ if off < typ.PtrBytes {
+ j := off / goarch.PtrSize
+ want = *addb(typ.GCData, j/8)>>(j%8)&1 != 0
+ }
+ if want {
+ var addr uintptr
+ h, addr = h.next()
+ if addr != x+i {
+ throw("userArenaHeapBitsSetType: pointer entry not correct")
+ }
+ }
+ }
+ if _, addr := h.next(); addr != 0 {
+ throw("userArenaHeapBitsSetType: extra pointer")
+ }
+ }
+}