1 // Copyright 2017 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // This implements the write barrier buffer. The write barrier itself
6 // is gcWriteBarrier and is implemented in assembly.
8 // See mbarrier.go for algorithmic details on the write barrier. This
9 // file deals only with the buffer.
11 // The write barrier has a fast path and a slow path. The fast path
12 // simply enqueues to a per-P write barrier buffer. It's written in
13 // assembly and doesn't clobber any general purpose registers, so it
14 // doesn't have the usual overheads of a Go call.
16 // When the buffer fills up, the write barrier invokes the slow path
17 // (wbBufFlush) to flush the buffer to the GC work queues. In this
18 // path, since the compiler didn't spill registers, we spill *all*
19 // registers and disallow any GC safe points that could observe the
20 // stack frame (since we don't know the types of the spilled
27 "runtime/internal/atomic"
31 // testSmallBuf forces a small write barrier buffer to stress write
33 const testSmallBuf = false
35 // wbBuf is a per-P buffer of pointers queued by the write barrier.
36 // This buffer is flushed to the GC workbufs when it fills up and on
37 // various GC transitions.
39 // This is closely related to a "sequential store buffer" (SSB),
40 // except that SSBs are usually used for maintaining remembered sets,
41 // while this is used for marking.
43 // next points to the next slot in buf. It must not be a
44 // pointer type because it can point past the end of buf and
45 // must be updated without write barriers.
47 // This is a pointer rather than an index to optimize the
48 // write barrier assembly.
51 // end points to just past the end of buf. It must not be a
52 // pointer type because it points past the end of buf and must
53 // be updated without write barriers.
56 // buf stores a series of pointers to execute write barriers on.
57 buf [wbBufEntries]uintptr
61 // wbBufEntries is the maximum number of pointers that can be
62 // stored in the write barrier buffer.
64 // This trades latency for throughput amortization. Higher
65 // values amortize flushing overhead more, but increase the
66 // latency of flushing. Higher values also increase the cache
67 // footprint of the buffer.
69 // TODO: What is the latency cost of this? Tune this value.
72 // Maximum number of entries that we need to ask from the
73 // buffer in a single call.
74 wbMaxEntriesPerCall = 8
77 // reset empties b by resetting its next and end pointers.
78 func (b *wbBuf) reset() {
79 start := uintptr(unsafe.Pointer(&b.buf[0]))
82 // For testing, make the buffer smaller but more than
83 // 1 write barrier's worth, so it tests both the
84 // immediate flush and delayed flush cases.
85 b.end = uintptr(unsafe.Pointer(&b.buf[wbMaxEntriesPerCall+1]))
87 b.end = start + uintptr(len(b.buf))*unsafe.Sizeof(b.buf[0])
90 if (b.end-b.next)%unsafe.Sizeof(b.buf[0]) != 0 {
91 throw("bad write barrier buffer bounds")
95 // discard resets b's next pointer, but not its end pointer.
97 // This must be nosplit because it's called by wbBufFlush.
100 func (b *wbBuf) discard() {
101 b.next = uintptr(unsafe.Pointer(&b.buf[0]))
104 // empty reports whether b contains no pointers.
105 func (b *wbBuf) empty() bool {
106 return b.next == uintptr(unsafe.Pointer(&b.buf[0]))
109 // getX returns space in the write barrier buffer to store X pointers.
110 // getX will flush the buffer if necessary. Callers should use this as:
112 // buf := &getg().m.p.ptr().wbBuf
114 // p[0], p[1] = old, new
115 // ... actual memory write ...
117 // The caller must ensure there are no preemption points during the
118 // above sequence. There must be no preemption points while buf is in
119 // use because it is a per-P resource. There must be no preemption
120 // points between the buffer put and the write to memory because this
121 // could allow a GC phase change, which could result in missed write
124 // getX must be nowritebarrierrec to because write barriers here would
125 // corrupt the write barrier buffer. It (and everything it calls, if
126 // it called anything) has to be nosplit to avoid scheduling on to a
127 // different P and a different buffer.
129 //go:nowritebarrierrec
131 func (b *wbBuf) get1() *[1]uintptr {
132 if b.next+goarch.PtrSize > b.end {
135 p := (*[1]uintptr)(unsafe.Pointer(b.next))
136 b.next += goarch.PtrSize
140 //go:nowritebarrierrec
142 func (b *wbBuf) get2() *[2]uintptr {
143 if b.next+2*goarch.PtrSize > b.end {
146 p := (*[2]uintptr)(unsafe.Pointer(b.next))
147 b.next += 2 * goarch.PtrSize
151 // wbBufFlush flushes the current P's write barrier buffer to the GC
154 // This must not have write barriers because it is part of the write
155 // barrier implementation.
157 // This and everything it calls must be nosplit because 1) the stack
158 // contains untyped slots from gcWriteBarrier and 2) there must not be
159 // a GC safe point between the write barrier test in the caller and
160 // flushing the buffer.
162 // TODO: A "go:nosplitrec" annotation would be perfect for this.
164 //go:nowritebarrierrec
167 // Note: Every possible return from this function must reset
168 // the buffer's next pointer to prevent buffer overflow.
170 if getg().m.dying > 0 {
171 // We're going down. Not much point in write barriers
172 // and this way we can allow write barriers in the
174 getg().m.p.ptr().wbBuf.discard()
178 // Switch to the system stack so we don't have to worry about
181 wbBufFlush1(getg().m.p.ptr())
185 // wbBufFlush1 flushes p's write barrier buffer to the GC work queue.
187 // This must not have write barriers because it is part of the write
188 // barrier implementation, so this may lead to infinite loops or
189 // buffer corruption.
191 // This must be non-preemptible because it uses the P's workbuf.
193 //go:nowritebarrierrec
195 func wbBufFlush1(pp *p) {
196 // Get the buffered pointers.
197 start := uintptr(unsafe.Pointer(&pp.wbBuf.buf[0]))
198 n := (pp.wbBuf.next - start) / unsafe.Sizeof(pp.wbBuf.buf[0])
199 ptrs := pp.wbBuf.buf[:n]
201 // Poison the buffer to make extra sure nothing is enqueued
202 // while we're processing the buffer.
206 // Slow path for checkmark mode.
207 for _, ptr := range ptrs {
214 // Mark all of the pointers in the buffer and record only the
215 // pointers we greyed. We use the buffer itself to temporarily
216 // record greyed pointers.
218 // TODO: Should scanobject/scanblock just stuff pointers into
219 // the wbBuf? Then this would become the sole greying path.
221 // TODO: We could avoid shading any of the "new" pointers in
222 // the buffer if the stack has been shaded, or even avoid
223 // putting them in the buffer at all (which would double its
224 // capacity). This is slightly complicated with the buffer; we
225 // could track whether any un-shaded goroutine has used the
226 // buffer, or just track globally whether there are any
227 // un-shaded stacks and flush after each stack scan.
230 for _, ptr := range ptrs {
231 if ptr < minLegalPointer {
232 // nil pointers are very common, especially
233 // for the "old" values. Filter out these and
234 // other "obvious" non-heap pointers ASAP.
236 // TODO: Should we filter out nils in the fast
237 // path to reduce the rate of flushes?
240 obj, span, objIndex := findObject(ptr, 0, 0)
244 // TODO: Consider making two passes where the first
245 // just prefetches the mark bits.
246 mbits := span.markBitsForIndex(objIndex)
247 if mbits.isMarked() {
253 arena, pageIdx, pageMask := pageIndexOf(span.base())
254 if arena.pageMarks[pageIdx]&pageMask == 0 {
255 atomic.Or8(&arena.pageMarks[pageIdx], pageMask)
258 if span.spanclass.noscan() {
259 gcw.bytesMarked += uint64(span.elemsize)
266 // Enqueue the greyed objects.
267 gcw.putBatch(ptrs[:pos])