}
s := spanOfUnchecked(uintptr(src))
- if s.state == mSpanManual {
+ if s.state.get() == mSpanManual {
// There are no heap bits for value stored on the stack.
// For a channel receive src might be on the stack of some
// other goroutine, so we can't unwind the stack even if
pagesInUse = uintptr(mheap_.pagesInUse)
for _, s := range mheap_.allspans {
- if s.state == mSpanInUse {
+ if s.state.get() == mSpanInUse {
counted += s.npages
}
}
// Add up current allocations in spans.
for _, s := range mheap_.allspans {
- if s.state != mSpanInUse {
+ if s.state.get() != mSpanInUse {
continue
}
if sizeclass := s.spanclass.sizeclass(); sizeclass == 0 {
lock(&mheap_.lock)
base = mheap_.free.unscavHugePages
for _, s := range mheap_.allspans {
- if s.state == mSpanFree && !s.scavenged {
+ if s.state.get() == mSpanFree && !s.scavenged {
slow += s.hugePages()
}
}
// mspan.types
for _, s := range mheap_.allspans {
- if s.state == mSpanInUse {
+ if s.state.get() == mSpanInUse {
// Finalizers
for sp := s.specials; sp != nil; sp = sp.next {
if sp.kind != _KindSpecialFinalizer {
func dumpobjs() {
for _, s := range mheap_.allspans {
- if s.state != mSpanInUse {
+ if s.state.get() != mSpanInUse {
continue
}
p := s.base()
func dumpmemprof() {
iterate_memprof(dumpmemprof_callback)
for _, s := range mheap_.allspans {
- if s.state != mSpanInUse {
+ if s.state.get() != mSpanInUse {
continue
}
for sp := s.specials; sp != nil; sp = sp.next {
func mdump() {
// make sure we're done sweeping
for _, s := range mheap_.allspans {
- if s.state == mSpanInUse {
+ if s.state.get() == mSpanInUse {
s.ensureSwept()
}
}
}
// isFree reports whether the index'th object in s is unallocated.
+//
+// The caller must ensure s.state is mSpanInUse, and there must have
+// been no preemption points since ensuring this (which could allow a
+// GC transition, which would allow the state to change).
func (s *mspan) isFree(index uintptr) bool {
if index < s.freeindex {
return false
// in allocated spans.
printlock()
print("runtime: pointer ", hex(p))
- if s.state != mSpanInUse {
+ state := s.state.get()
+ if state != mSpanInUse {
print(" to unallocated span")
} else {
print(" to unused region of span")
}
- print(" span.base()=", hex(s.base()), " span.limit=", hex(s.limit), " span.state=", s.state, "\n")
+ print(" span.base()=", hex(s.base()), " span.limit=", hex(s.limit), " span.state=", state, "\n")
if refBase != 0 {
print("runtime: found in object at *(", hex(refBase), "+", hex(refOff), ")\n")
gcDumpObject("object", refBase, refOff)
return
}
// If p is a bad pointer, it may not be in s's bounds.
- if p < s.base() || p >= s.limit || s.state != mSpanInUse {
+ //
+ // Check s.state to synchronize with span initialization
+ // before checking other fields. See also spanOfHeap.
+ if state := s.state.get(); state != mSpanInUse || p < s.base() || p >= s.limit {
// Pointers into stacks are also ok, the runtime manages these explicitly.
- if s.state == mSpanManual {
+ if state == mSpanManual {
return
}
// The following ensures that we are rigorous about what data
}
}
return
- } else if s.state != mSpanInUse || dst < s.base() || s.limit <= dst {
+ } else if s.state.get() != mSpanInUse || dst < s.base() || s.limit <= dst {
// dst was heap memory at some point, but isn't now.
// It can't be a global. It must be either our stack,
// or in the case of direct channel sends, it could be
// entered the scan phase, so addfinalizer will have ensured
// the above invariants for them.
for _, s := range spans {
- if s.state != mSpanInUse {
+ // This is racing with spans being initialized, so
+ // check the state carefully.
+ if s.state.get() != mSpanInUse {
continue
}
// Check that this span was swept (it may be cached or uncached).
return
}
print(" s.base()=", hex(s.base()), " s.limit=", hex(s.limit), " s.spanclass=", s.spanclass, " s.elemsize=", s.elemsize, " s.state=")
- if 0 <= s.state && int(s.state) < len(mSpanStateNames) {
- print(mSpanStateNames[s.state], "\n")
+ if state := s.state.get(); 0 <= state && int(state) < len(mSpanStateNames) {
+ print(mSpanStateNames[state], "\n")
} else {
- print("unknown(", s.state, ")\n")
+ print("unknown(", state, ")\n")
}
skipped := false
size := s.elemsize
- if s.state == mSpanManual && size == 0 {
+ if s.state.get() == mSpanManual && size == 0 {
// We're printing something from a stack frame. We
// don't know how big it is, so just show up to an
// including off.
func initCheckmarks() {
useCheckmark = true
for _, s := range mheap_.allspans {
- if s.state == mSpanInUse {
+ if s.state.get() == mSpanInUse {
heapBitsForAddr(s.base()).initCheckmarkSpan(s.layout())
}
}
func clearCheckmarks() {
useCheckmark = false
for _, s := range mheap_.allspans {
- if s.state == mSpanInUse {
+ if s.state.get() == mSpanInUse {
heapBitsForAddr(s.base()).clearCheckmarkSpan(s.layout())
}
}
atomic.Store(&mheap_.sweepdone, 1)
break
}
- if s.state != mSpanInUse {
+ if state := s.state.get(); state != mSpanInUse {
// This can happen if direct sweeping already
// swept this span, but in that case the sweep
// generation should always be up-to-date.
if !(s.sweepgen == sg || s.sweepgen == sg+3) {
- print("runtime: bad span s.state=", s.state, " s.sweepgen=", s.sweepgen, " sweepgen=", sg, "\n")
+ print("runtime: bad span s.state=", state, " s.sweepgen=", s.sweepgen, " sweepgen=", sg, "\n")
throw("non in-use span in unswept list")
}
continue
throw("mspan.sweep: m is not locked")
}
sweepgen := mheap_.sweepgen
- if s.state != mSpanInUse || s.sweepgen != sweepgen-1 {
- print("mspan.sweep: state=", s.state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n")
+ if state := s.state.get(); state != mSpanInUse || s.sweepgen != sweepgen-1 {
+ print("mspan.sweep: state=", state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n")
throw("mspan.sweep: bad span state")
}
if freeToHeap || nfreed == 0 {
// The span must be in our exclusive ownership until we update sweepgen,
// check for potential races.
- if s.state != mSpanInUse || s.sweepgen != sweepgen-1 {
- print("mspan.sweep: state=", s.state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n")
+ if state := s.state.get(); state != mSpanInUse || s.sweepgen != sweepgen-1 {
+ print("mspan.sweep: state=", state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n")
throw("mspan.sweep: bad span state after sweep")
}
// Serialization point.
// * During GC (gcphase != _GCoff), a span *must not* transition from
// manual or in-use to free. Because concurrent GC may read a pointer
// and then look up its span, the span state must be monotonic.
+//
+// Setting mspan.state to mSpanInUse or mSpanManual must be done
+// atomically and only after all other span fields are valid.
+// Likewise, if inspecting a span is contingent on it being
+// mSpanInUse, the state should be loaded atomically and checked
+// before depending on other fields. This allows the garbage collector
+// to safely deal with potentially invalid pointers, since resolving
+// such pointers may race with a span being allocated.
type mSpanState uint8
const (
"mSpanFree",
}
+// mSpanStateBox holds an mSpanState and provides atomic operations on
+// it. This is a separate type to disallow accidental comparison or
+// assignment with mSpanState.
+type mSpanStateBox struct {
+ s mSpanState
+}
+
+func (b *mSpanStateBox) set(s mSpanState) {
+ atomic.Store8((*uint8)(&b.s), uint8(s))
+}
+
+func (b *mSpanStateBox) get() mSpanState {
+ return mSpanState(atomic.Load8((*uint8)(&b.s)))
+}
+
// mSpanList heads a linked list of spans.
//
//go:notinheap
// h->sweepgen is incremented by 2 after every GC
sweepgen uint32
- divMul uint16 // for divide by elemsize - divMagic.mul
- baseMask uint16 // if non-0, elemsize is a power of 2, & this will get object allocation base
- allocCount uint16 // number of allocated objects
- spanclass spanClass // size class and noscan (uint8)
- state mSpanState // mspaninuse etc
- needzero uint8 // needs to be zeroed before allocation
- divShift uint8 // for divide by elemsize - divMagic.shift
- divShift2 uint8 // for divide by elemsize - divMagic.shift2
- scavenged bool // whether this span has had its pages released to the OS
- elemsize uintptr // computed from sizeclass or from npages
- limit uintptr // end of data in span
- speciallock mutex // guards specials list
- specials *special // linked list of special records sorted by offset.
+ divMul uint16 // for divide by elemsize - divMagic.mul
+ baseMask uint16 // if non-0, elemsize is a power of 2, & this will get object allocation base
+ allocCount uint16 // number of allocated objects
+ spanclass spanClass // size class and noscan (uint8)
+ state mSpanStateBox // mSpanInUse etc; accessed atomically (get/set methods)
+ needzero uint8 // needs to be zeroed before allocation
+ divShift uint8 // for divide by elemsize - divMagic.shift
+ divShift2 uint8 // for divide by elemsize - divMagic.shift2
+ scavenged bool // whether this span has had its pages released to the OS
+ elemsize uintptr // computed from sizeclass or from npages
+ limit uintptr // end of data in span
+ speciallock mutex // guards specials list
+ specials *special // linked list of special records sorted by offset.
}
func (s *mspan) base() uintptr {
// The size is potentially changing so the treap needs to delete adjacent nodes and
// insert back as a combined node.
h.free.removeSpan(other)
- other.state = mSpanDead
+ other.state.set(mSpanDead)
h.spanalloc.free(unsafe.Pointer(other))
}
// Coalesce with earlier, later spans.
var hpBefore uintptr
- if before := spanOf(s.base() - 1); before != nil && before.state == mSpanFree {
+ if before := spanOf(s.base() - 1); before != nil && before.state.get() == mSpanFree {
if s.scavenged == before.scavenged {
hpBefore = before.hugePages()
merge(before, s, before)
// Now check to see if next (greater addresses) span is free and can be coalesced.
var hpAfter uintptr
- if after := spanOf(s.base() + s.npages*pageSize); after != nil && after.state == mSpanFree {
+ if after := spanOf(s.base() + s.npages*pageSize); after != nil && after.state.get() == mSpanFree {
if s.scavenged == after.scavenged {
hpAfter = after.hugePages()
merge(s, after, after)
if s == nil || b < s.base() {
return false
}
- switch s.state {
+ switch s.state.get() {
case mSpanInUse, mSpanManual:
return b < s.limit
default:
//go:nosplit
func spanOfHeap(p uintptr) *mspan {
s := spanOf(p)
- // If p is not allocated, it may point to a stale span, so we
- // have to check the span's bounds and state.
- if s == nil || p < s.base() || p >= s.limit || s.state != mSpanInUse {
+ // s is nil if it's never been allocated. Otherwise, we check
+ // its state first because we don't trust this pointer, so we
+ // have to synchronize with span initialization. Then, it's
+ // still possible we picked up a stale span pointer, so we
+ // have to check the span's bounds.
+ if s == nil || s.state.get() != mSpanInUse || p < s.base() || p >= s.limit {
return nil
}
return s
// able to map interior pointer to containing span.
atomic.Store(&s.sweepgen, h.sweepgen)
h.sweepSpans[h.sweepgen/2%2].push(s) // Add to swept in-use list.
- s.state = mSpanInUse
s.allocCount = 0
s.spanclass = spanclass
s.elemsize = elemSize
s.gcmarkBits = gcmarkBits
s.allocBits = allocBits
+ // Now that the span is filled in, set its state. This
+ // is a publication barrier for the other fields in
+ // the span. While valid pointers into this span
+ // should never be visible until the span is returned,
+ // if the garbage collector finds an invalid pointer,
+ // access to the span may race with initialization of
+ // the span. We resolve this race by atomically
+ // setting the state after the span is fully
+ // initialized, and atomically checking the state in
+ // any situation where a pointer is suspect.
+ s.state.set(mSpanInUse)
+
// Mark in-use span in arena page bitmap.
arena, pageIdx, pageMask := pageIndexOf(s.base())
arena.pageInUse[pageIdx] |= pageMask
lock(&h.lock)
s := h.allocSpanLocked(npage, stat)
if s != nil {
- s.state = mSpanManual
s.manualFreeList = 0
s.allocCount = 0
s.spanclass = 0
s.nelems = 0
s.elemsize = 0
s.limit = s.base() + s.npages<<_PageShift
+ s.state.set(mSpanManual) // Publish the span
// Manually managed memory doesn't count toward heap_sys.
memstats.heap_sys -= uint64(s.npages << _PageShift)
}
HaveSpan:
s := t.span()
- if s.state != mSpanFree {
+ if s.state.get() != mSpanFree {
throw("candidate mspan for allocation is not free")
}
s := (*mspan)(h.spanalloc.alloc())
s.init(uintptr(v), size/pageSize)
h.setSpans(s.base(), s.npages, s)
- s.state = mSpanFree
+ s.state.set(mSpanFree)
// [v, v+size) is always in the Prepared state. The new span
// must be marked scavenged so the allocator transitions it to
// Ready when allocating from it.
}
func (h *mheap) freeSpanLocked(s *mspan, acctinuse, acctidle bool) {
- switch s.state {
+ switch s.state.get() {
case mSpanManual:
if s.allocCount != 0 {
throw("mheap.freeSpanLocked - invalid stack free")
if acctidle {
memstats.heap_idle += uint64(s.npages << _PageShift)
}
- s.state = mSpanFree
+ s.state.set(mSpanFree)
// Coalesce span with neighbors.
h.coalesce(s)
h.setSpan(n.base(), n)
h.setSpan(n.base()+nbytes-1, n)
n.needzero = s.needzero
- n.state = s.state
+ n.state.set(s.state.get())
})
return n
}
span.allocCount = 0
span.spanclass = 0
span.elemsize = 0
- span.state = mSpanDead
span.scavenged = false
span.speciallock.key = 0
span.specials = nil
span.freeindex = 0
span.allocBits = nil
span.gcmarkBits = nil
+ span.state.set(mSpanDead)
}
func (span *mspan) inList() bool {
// work.
sp := getcallersp()
s := spanOf(sp)
- if s != nil && s.state == mSpanManual && s.base() < sp && sp < s.limit {
+ if s != nil && s.state.get() == mSpanManual && s.base() < sp && sp < s.limit {
gp := *(**g)(unsafe.Pointer(s.base()))
return gp
}
// Adds stack x to the free pool. Must be called with stackpool[order].item.mu held.
func stackpoolfree(x gclinkptr, order uint8) {
s := spanOfUnchecked(uintptr(x))
- if s.state != mSpanManual {
+ if s.state.get() != mSpanManual {
throw("freeing stack not in a stack span")
}
if s.manualFreeList.ptr() == nil {
}
} else {
s := spanOfUnchecked(uintptr(v))
- if s.state != mSpanManual {
+ if s.state.get() != mSpanManual {
println(hex(s.base()), v)
throw("bad span state")
}