package runtime
import (
+ "internal/goarch"
+ "internal/goexperiment"
"runtime/internal/atomic"
"runtime/internal/math"
"unsafe"
lockInit(&userArenaState.lock, lockRankUserArenaState)
}
+// userArenaChunkReserveBytes returns the amount of additional bytes to reserve for
+// heap metadata.
+func userArenaChunkReserveBytes() uintptr {
+ if goexperiment.AllocHeaders {
+ // In the allocation headers experiment, we reserve the end of the chunk for
+ // a pointer/scalar bitmap. We also reserve space for a dummy _type that
+ // refers to the bitmap. The PtrBytes field of the dummy _type indicates how
+ // many of those bits are valid.
+ return userArenaChunkBytes/goarch.PtrSize/8 + unsafe.Sizeof(_type{})
+ }
+ return 0
+}
+
type userArena struct {
// full is a list of full chunks that have not enough free memory left, and
// that we'll free once this user arena is freed.
// Set up heap bitmap and do extra accounting.
if typ.PtrBytes != 0 {
if cap >= 0 {
- userArenaHeapBitsSetSliceType(typ, cap, ptr, s.base())
+ userArenaHeapBitsSetSliceType(typ, cap, ptr, s)
} else {
- userArenaHeapBitsSetType(typ, ptr, s.base())
+ userArenaHeapBitsSetType(typ, ptr, s)
}
c := getMCache(mp)
if c == nil {
// userArenaHeapBitsSetSliceType is the equivalent of heapBitsSetType but for
// Go slice backing store values allocated in a user arena chunk. It sets up the
// heap bitmap for n consecutive values with type typ allocated at address ptr.
-func userArenaHeapBitsSetSliceType(typ *_type, n int, ptr unsafe.Pointer, base uintptr) {
+func userArenaHeapBitsSetSliceType(typ *_type, n int, ptr unsafe.Pointer, s *mspan) {
mem, overflow := math.MulUintptr(typ.Size_, uintptr(n))
if overflow || n < 0 || mem > maxAlloc {
panic(plainError("runtime: allocation size out of range"))
}
for i := 0; i < n; i++ {
- userArenaHeapBitsSetType(typ, add(ptr, uintptr(i)*typ.Size_), base)
+ userArenaHeapBitsSetType(typ, add(ptr, uintptr(i)*typ.Size_), s)
}
}
// TODO(mknyszek): Track individual objects.
rzSize := computeRZlog(span.elemsize)
span.elemsize -= rzSize
- span.limit -= rzSize
- span.userArenaChunkFree = makeAddrRange(span.base(), span.limit)
- asanpoison(unsafe.Pointer(span.limit), span.npages*pageSize-span.elemsize)
+ if goexperiment.AllocHeaders {
+ span.largeType.Size_ = span.elemsize
+ }
+ rzStart := span.base() + span.elemsize
+ span.userArenaChunkFree = makeAddrRange(span.base(), rzStart)
+ asanpoison(unsafe.Pointer(rzStart), span.limit-rzStart)
asanunpoison(unsafe.Pointer(span.base()), span.elemsize)
}
// the span gets off the quarantine list. The main reason is so that the
// amount of bytes allocated doesn't exceed how much is counted as
// "mapped ready," which could cause a deadlock in the pacer.
- gcController.totalFree.Add(int64(s.npages * pageSize))
+ gcController.totalFree.Add(int64(s.elemsize))
// Update consistent stats to match.
//
atomic.Xaddint64(&stats.committed, -int64(s.npages*pageSize))
atomic.Xaddint64(&stats.inHeap, -int64(s.npages*pageSize))
atomic.Xadd64(&stats.largeFreeCount, 1)
- atomic.Xadd64(&stats.largeFree, int64(s.npages*pageSize))
+ atomic.Xadd64(&stats.largeFree, int64(s.elemsize))
memstats.heapStats.release()
// This counts as a free, so update heapLive.
- gcController.update(-int64(s.npages*pageSize), 0)
+ gcController.update(-int64(s.elemsize), 0)
// Mark it as free for the race detector.
if raceenabled {
spc := makeSpanClass(0, false)
h.initSpan(s, spanAllocHeap, spc, base, userArenaChunkPages)
s.isUserArenaChunk = true
+ s.elemsize -= userArenaChunkReserveBytes()
+ s.limit = s.base() + s.elemsize
+ s.freeindex = 1
+ s.allocCount = 1
// Account for this new arena chunk memory.
gcController.heapInUse.add(int64(userArenaChunkBytes))
atomic.Xaddint64(&stats.committed, int64(userArenaChunkBytes))
// Model the arena as a single large malloc.
- atomic.Xadd64(&stats.largeAlloc, int64(userArenaChunkBytes))
+ atomic.Xadd64(&stats.largeAlloc, int64(s.elemsize))
atomic.Xadd64(&stats.largeAllocCount, 1)
memstats.heapStats.release()
// Count the alloc in inconsistent, internal stats.
- gcController.totalAlloc.Add(int64(userArenaChunkBytes))
+ gcController.totalAlloc.Add(int64(s.elemsize))
// Update heapLive.
- gcController.update(int64(userArenaChunkBytes), 0)
-
- // Put the large span in the mcentral swept list so that it's
- // visible to the background sweeper.
- h.central[spc].mcentral.fullSwept(h.sweepgen).push(s)
- s.limit = s.base() + userArenaChunkBytes
- s.freeindex = 1
- s.allocCount = 1
+ gcController.update(int64(s.elemsize), 0)
// This must clear the entire heap bitmap so that it's safe
// to allocate noscan data without writing anything out.
s.freeIndexForScan = 1
// Set up the range for allocation.
- s.userArenaChunkFree = makeAddrRange(base, s.limit)
+ s.userArenaChunkFree = makeAddrRange(base, base+s.elemsize)
+
+ // Put the large span in the mcentral swept list so that it's
+ // visible to the background sweeper.
+ h.central[spc].mcentral.fullSwept(h.sweepgen).push(s)
+
+ if goexperiment.AllocHeaders {
+ // Set up an allocation header. Avoid write barriers here because this type
+ // is not a real type, and it exists in an invalid location.
+ *(*uintptr)(unsafe.Pointer(&s.largeType)) = uintptr(unsafe.Pointer(s.limit))
+ *(*uintptr)(unsafe.Pointer(&s.largeType.GCData)) = s.limit + unsafe.Sizeof(_type{})
+ s.largeType.PtrBytes = 0
+ s.largeType.Size_ = s.elemsize
+ }
return s
}