+// heapBits returns the heap ptr/scalar bits stored at the end of the span for
+// small object spans.
+//
+// heapBitsInSpan(span.elemsize) or span.isUserArenaChunk must be true.
+//
+//go:nosplit
+func (span *mspan) heapBits() []uintptr {
+ const doubleCheck = false
+
+ if doubleCheck && !span.isUserArenaChunk {
+ if span.spanclass.noscan() {
+ throw("heapBits called for noscan")
+ }
+ if span.elemsize > minSizeForMallocHeader {
+ throw("heapBits called for span class that should have a malloc header")
+ }
+ }
+ // Find the bitmap at the end of the span.
+ //
+ // Nearly every span with heap bits is exactly one page in size. Arenas are the only exception.
+ if span.npages == 1 {
+ // This will be inlined and constant-folded down.
+ return heapBitsSlice(span.base(), pageSize)
+ }
+ return heapBitsSlice(span.base(), span.npages*pageSize)
+}
+
+// Helper for constructing a slice for the span's heap bits.
+//
+//go:nosplit
+func heapBitsSlice(spanBase, spanSize uintptr) []uintptr {
+ bitmapSize := spanSize / goarch.PtrSize / 8
+ elems := int(bitmapSize / goarch.PtrSize)
+ var sl notInHeapSlice
+ sl = notInHeapSlice{(*notInHeap)(unsafe.Pointer(spanBase + spanSize - bitmapSize)), elems, elems}
+ return *(*[]uintptr)(unsafe.Pointer(&sl))
+}
+
+// heapBitsSmallForAddr loads the heap bits for the object stored at addr from span.heapBits.
+//
+// addr must be the base pointer of an object in the span. heapBitsInSpan(span.elemsize)
+// must be true.
+//
+//go:nosplit
+func (span *mspan) heapBitsSmallForAddr(addr uintptr) uintptr {
+ spanSize := span.npages * pageSize
+ bitmapSize := spanSize / goarch.PtrSize / 8
+ hbits := (*byte)(unsafe.Pointer(span.base() + spanSize - bitmapSize))
+
+ // These objects are always small enough that their bitmaps
+ // fit in a single word, so just load the word or two we need.
+ //
+ // Mirrors mspan.writeHeapBitsSmall.
+ //
+ // We should be using heapBits(), but unfortunately it introduces
+ // both bounds checks panics and throw which causes us to exceed
+ // the nosplit limit in quite a few cases.
+ i := (addr - span.base()) / goarch.PtrSize / ptrBits
+ j := (addr - span.base()) / goarch.PtrSize % ptrBits
+ bits := span.elemsize / goarch.PtrSize
+ word0 := (*uintptr)(unsafe.Pointer(addb(hbits, goarch.PtrSize*(i+0))))
+ word1 := (*uintptr)(unsafe.Pointer(addb(hbits, goarch.PtrSize*(i+1))))
+
+ var read uintptr
+ if j+bits > ptrBits {
+ // Two reads.
+ bits0 := ptrBits - j
+ bits1 := bits - bits0
+ read = *word0 >> j
+ read |= (*word1 & ((1 << bits1) - 1)) << bits0
+ } else {
+ // One read.
+ read = (*word0 >> j) & ((1 << bits) - 1)
+ }
+ return read
+}
+
+// writeHeapBitsSmall writes the heap bits for small objects whose ptr/scalar data is
+// stored as a bitmap at the end of the span.
+//
+// Assumes dataSize is <= ptrBits*goarch.PtrSize. x must be a pointer into the span.
+// heapBitsInSpan(dataSize) must be true. dataSize must be >= typ.Size_.
+//
+//go:nosplit
+func (span *mspan) writeHeapBitsSmall(x, dataSize uintptr, typ *_type) (scanSize uintptr) {
+ // The objects here are always really small, so a single load is sufficient.
+ src0 := readUintptr(typ.GCData)
+
+ // Create repetitions of the bitmap if we have a small array.
+ bits := span.elemsize / goarch.PtrSize
+ scanSize = typ.PtrBytes
+ src := src0
+ switch typ.Size_ {
+ case goarch.PtrSize:
+ src = (1 << (dataSize / goarch.PtrSize)) - 1
+ default:
+ for i := typ.Size_; i < dataSize; i += typ.Size_ {
+ src |= src0 << (i / goarch.PtrSize)
+ scanSize += typ.Size_
+ }
+ }
+
+ // Since we're never writing more than one uintptr's worth of bits, we're either going
+ // to do one or two writes.
+ dst := span.heapBits()
+ o := (x - span.base()) / goarch.PtrSize
+ i := o / ptrBits
+ j := o % ptrBits
+ if j+bits > ptrBits {
+ // Two writes.
+ bits0 := ptrBits - j
+ bits1 := bits - bits0
+ dst[i+0] = dst[i+0]&(^uintptr(0)>>bits0) | (src << j)
+ dst[i+1] = dst[i+1]&^((1<<bits1)-1) | (src >> bits0)
+ } else {
+ // One write.
+ dst[i] = (dst[i] &^ (((1 << bits) - 1) << j)) | (src << j)
+ }
+
+ const doubleCheck = false
+ if doubleCheck {
+ srcRead := span.heapBitsSmallForAddr(x)
+ if srcRead != src {
+ print("runtime: x=", hex(x), " i=", i, " j=", j, " bits=", bits, "\n")
+ print("runtime: dataSize=", dataSize, " typ.Size_=", typ.Size_, " typ.PtrBytes=", typ.PtrBytes, "\n")
+ print("runtime: src0=", hex(src0), " src=", hex(src), " srcRead=", hex(srcRead), "\n")
+ throw("bad pointer bits written for small object")
+ }
+ }
+ return
+}
+
+// For !goexperiment.AllocHeaders.
+func heapBitsSetType(x, size, dataSize uintptr, typ *_type) {
+}
+
+// heapSetType records that the new allocation [x, x+size)