//
// Stack, data, and bss bitmaps
//
-// Stack frames and global variables in the data and bss sections are described
-// by 1-bit bitmaps in which 0 means uninteresting and 1 means live pointer
-// to be visited during GC. The bits in each byte are consumed starting with
-// the low bit: 1<<0, 1<<1, and so on.
+// Stack frames and global variables in the data and bss sections are
+// described by bitmaps with 1 bit per pointer-sized word. A "1" bit
+// means the word is a live pointer to be visited by the GC (referred to
+// as "pointer"). A "0" bit means the word should be ignored by GC
+// (referred to as "scalar", though it could be a dead pointer value).
//
// Heap bitmap
//
// through start+3*ptrSize, ha.bitmap[1] holds the entries for
// start+4*ptrSize through start+7*ptrSize, and so on.
//
-// In each 2-bit entry, the lower bit holds the same information as in the 1-bit
-// bitmaps: 0 means uninteresting and 1 means live pointer to be visited during GC.
-// The meaning of the high bit depends on the position of the word being described
-// in its allocated object. In all words *except* the second word, the
-// high bit indicates that the object is still being described. In
-// these words, if a bit pair with a high bit 0 is encountered, the
-// low bit can also be assumed to be 0, and the object description is
-// over. This 00 is called the ``dead'' encoding: it signals that the
-// rest of the words in the object are uninteresting to the garbage
-// collector.
-//
-// In the second word, the high bit is the GC ``checkmarked'' bit (see below).
+// In each 2-bit entry, the lower bit is a pointer/scalar bit, just
+// like in the stack/data bitmaps described above. The upper bit
+// indicates scan/dead: a "1" value ("scan") indicates that there may
+// be pointers in later words of the allocation, and a "0" value
+// ("dead") indicates there are no more pointers in the allocation. If
+// the upper bit is 0, the lower bit must also be 0, and this
+// indicates scanning can ignore the rest of the allocation.
//
// The 2-bit entries are split when written into the byte, so that the top half
// of the byte contains 4 high bits and the bottom half contains 4 low (pointer)
// This form allows a copy from the 1-bit to the 4-bit form to keep the
// pointer bits contiguous, instead of having to space them out.
//
-// The code makes use of the fact that the zero value for a heap bitmap
-// has no live pointer bit set and is (depending on position), not used,
-// not checkmarked, and is the dead encoding.
-// These properties must be preserved when modifying the encoding.
+// The code makes use of the fact that the zero value for a heap
+// bitmap means scalar/dead. This property must be preserved when
+// modifying the encoding.
//
// The bitmap for noscan spans is not maintained. Code must ensure
// that an object is scannable before consulting its bitmap by
// checking either the noscan bit in the span or by consulting its
// type's information.
-//
-// Checkmarks
-//
-// In a concurrent garbage collector, one worries about failing to mark
-// a live object due to mutations without write barriers or bugs in the
-// collector implementation. As a sanity check, the GC has a 'checkmark'
-// mode that retraverses the object graph with the world stopped, to make
-// sure that everything that should be marked is marked.
-// In checkmark mode, in the heap bitmap, the high bit of the 2-bit entry
-// for the second word of the object holds the checkmark bit.
-// When not in checkmark mode, this bit is set to 1.
-//
-// The smallest possible allocation is 8 bytes. On a 32-bit machine, that
-// means every allocated object has two words, so there is room for the
-// checkmark bit. On a 64-bit machine, however, the 8-byte allocation is
-// just one word, so the second bit pair is not available for encoding the
-// checkmark. However, because non-pointer allocations are combined
-// into larger 16-byte (maxTinySize) allocations, a plain 8-byte allocation
-// must be a pointer, so the type bit in the first word is not actually needed.
-// It is still used in general, except in checkmark the type bit is repurposed
-// as the checkmark bit and then reinitialized (to 1) as the type bit when
-// finished.
-//
package runtime
return h.bits()&bitPointer != 0
}
-// isCheckmarked reports whether the heap bits have the checkmarked bit set.
-// It must be told how large the object at h is, because the encoding of the
-// checkmark bit varies by size.
-// h must describe the initial word of the object.
-func (h heapBits) isCheckmarked(size uintptr) bool {
- if size == sys.PtrSize {
- return (*h.bitp>>h.shift)&bitPointer != 0
- }
- // All multiword objects are 2-word aligned,
- // so we know that the initial word's 2-bit pair
- // and the second word's 2-bit pair are in the
- // same heap bitmap byte, *h.bitp.
- return (*h.bitp>>(heapBitsShift+h.shift))&bitScan != 0
-}
-
-// setCheckmarked sets the checkmarked bit.
-// It must be told how large the object at h is, because the encoding of the
-// checkmark bit varies by size.
-// h must describe the initial word of the object.
-func (h heapBits) setCheckmarked(size uintptr) {
- if size == sys.PtrSize {
- atomic.Or8(h.bitp, bitPointer<<h.shift)
- return
- }
- atomic.Or8(h.bitp, bitScan<<(heapBitsShift+h.shift))
-}
-
// bulkBarrierPreWrite executes a write barrier
// for every pointer slot in the memory range [src, src+size),
// using pointer/scalar information from [dst, dst+size).
// TODO(rsc): Perhaps introduce a different heapBitsSpan type.
// initSpan initializes the heap bitmap for a span.
-// It clears all checkmark bits.
// If this is a span of pointer-sized objects, it initializes all
// words to pointer/scan.
// Otherwise, it initializes all words to scalar/dead.
}
}
-// initCheckmarkSpan initializes a span for being checkmarked.
-// It clears the checkmark bits, which are set to 1 in normal operation.
-func (h heapBits) initCheckmarkSpan(size, n, total uintptr) {
- // The ptrSize == 8 is a compile-time constant false on 32-bit and eliminates this code entirely.
- if sys.PtrSize == 8 && size == sys.PtrSize {
- // Checkmark bit is type bit, bottom bit of every 2-bit entry.
- // Only possible on 64-bit system, since minimum size is 8.
- // Must clear type bit (checkmark bit) of every word.
- // The type bit is the lower of every two-bit pair.
- for i := uintptr(0); i < n; i += wordsPerBitmapByte {
- *h.bitp &^= bitPointerAll
- h = h.forward(wordsPerBitmapByte)
- }
- return
- }
- for i := uintptr(0); i < n; i++ {
- *h.bitp &^= bitScan << (heapBitsShift + h.shift)
- h = h.forward(size / sys.PtrSize)
- }
-}
-
-// clearCheckmarkSpan undoes all the checkmarking in a span.
-// The actual checkmark bits are ignored, so the only work to do
-// is to fix the pointer bits. (Pointer bits are ignored by scanobject
-// but consulted by typedmemmove.)
-func (h heapBits) clearCheckmarkSpan(size, n, total uintptr) {
- // The ptrSize == 8 is a compile-time constant false on 32-bit and eliminates this code entirely.
- if sys.PtrSize == 8 && size == sys.PtrSize {
- // Checkmark bit is type bit, bottom bit of every 2-bit entry.
- // Only possible on 64-bit system, since minimum size is 8.
- // Must clear type bit (checkmark bit) of every word.
- // The type bit is the lower of every two-bit pair.
- for i := uintptr(0); i < n; i += wordsPerBitmapByte {
- *h.bitp |= bitPointerAll
- h = h.forward(wordsPerBitmapByte)
- }
- }
-}
-
// countAlloc returns the number of objects allocated in span s by
// scanning the allocation bitmap.
func (s *mspan) countAlloc() int {
if sys.PtrSize == 4 && dataSize == sys.PtrSize {
// 1 pointer object. On 32-bit machines clear the bit for the
// unused second word.
- *h.bitp &^= (bitPointer | bitScan | ((bitPointer | bitScan) << heapBitsShift)) << h.shift
+ *h.bitp &^= (bitPointer | bitScan | (bitPointer|bitScan)<<heapBitsShift) << h.shift
*h.bitp |= (bitPointer | bitScan) << h.shift
} else {
// 2-element slice of pointer.
- *h.bitp |= (bitPointer | bitScan | bitPointer<<heapBitsShift) << h.shift
+ *h.bitp |= (bitPointer | bitScan | (bitPointer|bitScan)<<heapBitsShift) << h.shift
}
return
}
}
}
b := uint32(*ptrmask)
- hb := (b & 3) | bitScan
- // bitPointer == 1, bitScan is 1 << 4, heapBitsShift is 1.
- // 110011 is shifted h.shift and complemented.
- // This clears out the bits that are about to be
- // ored into *h.hbitp in the next instructions.
+ hb := b & 3
+ hb |= bitScanAll & ((bitScan << (typ.ptrdata / sys.PtrSize)) - 1)
+ // Clear the bits for this object so we can set the
+ // appropriate ones.
*h.bitp &^= (bitPointer | bitScan | ((bitPointer | bitScan) << heapBitsShift)) << h.shift
*h.bitp |= uint8(hb << h.shift)
return
throw("heapBitsSetType: called with non-pointer type")
return
}
- if nw < 2 {
- // Must write at least 2 words, because the "no scan"
- // encoding doesn't take effect until the third word.
- nw = 2
- }
// Phase 1: Special case for leading byte (shift==0) or half-byte (shift==2).
// The leading byte is special because it contains the bits for word 1,
case h.shift == 0:
// Ptrmask and heap bitmap are aligned.
- // Handle first byte of bitmap specially.
+ //
+ // This is a fast path for small objects.
//
// The first byte we write out covers the first four
// words of the object. The scan/dead bit on the first
// word must be set to scan since there are pointers
- // somewhere in the object. The scan/dead bit on the
- // second word is the checkmark, so we don't set it.
+ // somewhere in the object.
// In all following words, we set the scan/dead
// appropriately to indicate that the object contains
// to the next 2-bit entry in the bitmap.
//
- // TODO: It doesn't matter if we set the checkmark, so
- // maybe this case isn't needed any more.
+ // We set four bits at a time here, but if the object
+ // is fewer than four words, phase 3 will clear
+ // unnecessary bits.
hb = b & bitPointerAll
- hb |= bitScan | bitScan<<(2*heapBitsShift) | bitScan<<(3*heapBitsShift)
+ hb |= bitScanAll
if w += 4; w >= nw {
goto Phase3
}
// We took care of 1-word and 2-word objects above,
// so this is at least a 6-word object.
hb = (b & (bitPointer | bitPointer<<heapBitsShift)) << (2 * heapBitsShift)
- // This is not noscan, so set the scan bit in the
- // first word.
hb |= bitScan << (2 * heapBitsShift)
+ if nw > 1 {
+ hb |= bitScan << (3 * heapBitsShift)
+ }
b >>= 2
nb -= 2
- // Note: no bitScan for second word because that's
- // the checkmark.
- *hbitp &^= uint8((bitPointer | bitScan | (bitPointer << heapBitsShift)) << (2 * heapBitsShift))
+ *hbitp &^= uint8((bitPointer | bitScan | ((bitPointer | bitScan) << heapBitsShift)) << (2 * heapBitsShift))
*hbitp |= uint8(hb)
hbitp = add1(hbitp)
if w += 2; w >= nw {
if j < nptr && (*addb(ptrmask, j/8)>>(j%8))&1 != 0 {
want |= bitPointer
}
- if i != 1 {
- want |= bitScan
- } else {
- have &^= bitScan
- }
+ want |= bitScan
}
if have != want {
println("mismatch writing bits for", typ.string(), "x", dataSize/typ.size)
if hbits.isPointer() {
mask[i/sys.PtrSize] = 1
}
- if i != 1*sys.PtrSize && !hbits.morePointers() {
+ if !hbits.morePointers() {
mask = mask[:i/sys.PtrSize]
break
}
--- /dev/null
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// GC checkmarks
+//
+// In a concurrent garbage collector, one worries about failing to mark
+// a live object due to mutations without write barriers or bugs in the
+// collector implementation. As a sanity check, the GC has a 'checkmark'
+// mode that retraverses the object graph with the world stopped, to make
+// sure that everything that should be marked is marked.
+
+package runtime
+
+import (
+ "runtime/internal/atomic"
+ "runtime/internal/sys"
+ "unsafe"
+)
+
+// A checkmarksMap stores the GC marks in "checkmarks" mode. It is a
+// per-arena bitmap with a bit for every word in the arena. The mark
+// is stored on the bit corresponding to the first word of the marked
+// allocation.
+//
+//go:notinheap
+type checkmarksMap [heapArenaBytes / sys.PtrSize / 8]uint8
+
+// If useCheckmark is true, marking of an object uses the checkmark
+// bits instead of the standard mark bits.
+var useCheckmark = false
+
+// startCheckmarks prepares for the checkmarks phase.
+//
+// The world must be stopped.
+func startCheckmarks() {
+ // Clear all checkmarks.
+ for _, ai := range mheap_.allArenas {
+ arena := mheap_.arenas[ai.l1()][ai.l2()]
+ bitmap := arena.checkmarks
+
+ if bitmap == nil {
+ // Allocate bitmap on first use.
+ bitmap = (*checkmarksMap)(persistentalloc(unsafe.Sizeof(*bitmap), 0, &memstats.gc_sys))
+ if bitmap == nil {
+ throw("out of memory allocating checkmarks bitmap")
+ }
+ arena.checkmarks = bitmap
+ } else {
+ // Otherwise clear the existing bitmap.
+ for i := range bitmap {
+ bitmap[i] = 0
+ }
+ }
+ }
+ // Enable checkmarking.
+ useCheckmark = true
+}
+
+// endCheckmarks ends the checkmarks phase.
+func endCheckmarks() {
+ if gcMarkWorkAvailable(nil) {
+ throw("GC work not flushed")
+ }
+ useCheckmark = false
+}
+
+// setCheckmark throws if marking object is a checkmarks violation,
+// and otherwise sets obj's checkmark. It returns true if obj was
+// already checkmarked.
+func setCheckmark(obj, base, off uintptr, mbits markBits) bool {
+ if !mbits.isMarked() {
+ printlock()
+ print("runtime: checkmarks found unexpected unmarked object obj=", hex(obj), "\n")
+ print("runtime: found obj at *(", hex(base), "+", hex(off), ")\n")
+
+ // Dump the source (base) object
+ gcDumpObject("base", base, off)
+
+ // Dump the object
+ gcDumpObject("obj", obj, ^uintptr(0))
+
+ getg().m.traceback = 2
+ throw("checkmark found unmarked object")
+ }
+
+ ai := arenaIndex(obj)
+ arena := mheap_.arenas[ai.l1()][ai.l2()]
+ arenaWord := (obj / heapArenaBytes / 8) % uintptr(len(arena.checkmarks))
+ mask := byte(1 << ((obj / heapArenaBytes) % 8))
+ bytep := &arena.checkmarks[arenaWord]
+
+ if atomic.Load8(bytep)&mask != 0 {
+ // Already checkmarked.
+ return true
+ }
+
+ atomic.Or8(bytep, mask)
+ return false
+}
}
// Load bits once. See CL 22712 and issue 16973 for discussion.
bits := hbits.bits()
- // During checkmarking, 1-word objects store the checkmark
- // in the type bit for the one word. The only one-word objects
- // are pointers, or else they'd be merged with other non-pointer
- // data into larger allocations.
- if i != 1*sys.PtrSize && bits&bitScan == 0 {
+ if bits&bitScan == 0 {
break // no more pointers in this object
}
if bits&bitPointer == 0 {
mbits := span.markBitsForIndex(objIndex)
if useCheckmark {
- if !mbits.isMarked() {
- printlock()
- print("runtime:greyobject: checkmarks finds unexpected unmarked object obj=", hex(obj), "\n")
- print("runtime: found obj at *(", hex(base), "+", hex(off), ")\n")
-
- // Dump the source (base) object
- gcDumpObject("base", base, off)
-
- // Dump the object
- gcDumpObject("obj", obj, ^uintptr(0))
-
- getg().m.traceback = 2
- throw("checkmark found unmarked object")
- }
- hbits := heapBitsForAddr(obj)
- if hbits.isCheckmarked(span.elemsize) {
+ if setCheckmark(obj, base, off, mbits) {
+ // Already marked.
return
}
- hbits.setCheckmarked(span.elemsize)
- if !hbits.isCheckmarked(span.elemsize) {
- throw("setCheckmarked and isCheckmarked disagree")
- }
} else {
if debug.gccheckmark > 0 && span.isFree(objIndex) {
print("runtime: marking free object ", hex(obj), " found at *(", hex(base), "+", hex(off), ")\n")
greyobject(c.tiny, 0, 0, span, gcw, objIndex)
}
}
-
-// Checkmarking
-
-// To help debug the concurrent GC we remark with the world
-// stopped ensuring that any object encountered has their normal
-// mark bit set. To do this we use an orthogonal bit
-// pattern to indicate the object is marked. The following pattern
-// uses the upper two bits in the object's boundary nibble.
-// 01: scalar not marked
-// 10: pointer not marked
-// 11: pointer marked
-// 00: scalar marked
-// Xoring with 01 will flip the pattern from marked to unmarked and vica versa.
-// The higher bit is 1 for pointers and 0 for scalars, whether the object
-// is marked or not.
-// The first nibble no longer holds the typeDead pattern indicating that the
-// there are no more pointers in the object. This information is held
-// in the second nibble.
-
-// If useCheckmark is true, marking of an object uses the
-// checkmark bits (encoding above) instead of the standard
-// mark bits.
-var useCheckmark = false
-
-//go:nowritebarrier
-func initCheckmarks() {
- useCheckmark = true
- for _, s := range mheap_.allspans {
- if s.state.get() == mSpanInUse {
- heapBitsForAddr(s.base()).initCheckmarkSpan(s.layout())
- }
- }
-}
-
-func clearCheckmarks() {
- useCheckmark = false
- for _, s := range mheap_.allspans {
- if s.state.get() == mSpanInUse {
- heapBitsForAddr(s.base()).clearCheckmarkSpan(s.layout())
- }
- }
-}