1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 // TODO(rsc): The code having to do with the heap bitmap needs very serious cleanup.
6 // It has gotten completely out of control.
8 // Garbage collector (GC).
10 // The GC runs concurrently with mutator threads, is type accurate (aka precise), allows multiple GC
11 // thread to run in parallel. It is a concurrent mark and sweep that uses a write barrier. It is
12 // non-generational and non-compacting. Allocation is done using size segregated per P allocation
13 // areas to minimize fragmentation while eliminating locks in the common case.
15 // The algorithm decomposes into several steps.
16 // This is a high level description of the algorithm being used. For an overview of GC a good
17 // place to start is Richard Jones' gchandbook.org.
19 // The algorithm's intellectual heritage includes Dijkstra's on-the-fly algorithm, see
20 // Edsger W. Dijkstra, Leslie Lamport, A. J. Martin, C. S. Scholten, and E. F. M. Steffens. 1978.
21 // On-the-fly garbage collection: an exercise in cooperation. Commun. ACM 21, 11 (November 1978), 966-975.
22 // For journal quality proofs that these steps are complete, correct, and terminate see
23 // Hudson, R., and Moss, J.E.B. Copying Garbage Collection without stopping the world.
24 // Concurrency and Computation: Practice and Experience 15(3-5), 2003.
26 // 0. Set phase = GCscan from GCoff.
27 // 1. Wait for all P's to acknowledge phase change.
28 // At this point all goroutines have passed through a GC safepoint and
29 // know we are in the GCscan phase.
30 // 2. GC scans all goroutine stacks, mark and enqueues all encountered pointers
31 // (marking avoids most duplicate enqueuing but races may produce duplication which is benign).
32 // Preempted goroutines are scanned before P schedules next goroutine.
33 // 3. Set phase = GCmark.
34 // 4. Wait for all P's to acknowledge phase change.
35 // 5. Now write barrier marks and enqueues black, grey, or white to white pointers.
36 // Malloc still allocates white (non-marked) objects.
37 // 6. Meanwhile GC transitively walks the heap marking reachable objects.
38 // 7. When GC finishes marking heap, it preempts P's one-by-one and
39 // retakes partial wbufs (filled by write barrier or during a stack scan of the goroutine
40 // currently scheduled on the P).
41 // 8. Once the GC has exhausted all available marking work it sets phase = marktermination.
42 // 9. Wait for all P's to acknowledge phase change.
43 // 10. Malloc now allocates black objects, so number of unmarked reachable objects
44 // monotonically decreases.
45 // 11. GC preempts P's one-by-one taking partial wbufs and marks all unmarked yet reachable objects.
46 // 12. When GC completes a full cycle over P's and discovers no new grey
47 // objects, (which means all reachable objects are marked) set phase = GCsweep.
48 // 13. Wait for all P's to acknowledge phase change.
49 // 14. Now malloc allocates white (but sweeps spans before use).
50 // Write barrier becomes nop.
51 // 15. GC does background sweeping, see description below.
52 // 16. When sweeping is complete set phase to GCoff.
53 // 17. When sufficient allocation has taken place replay the sequence starting at 0 above,
54 // see discussion of GC rate below.
57 // Phases are changed by setting the gcphase to the next phase and possibly calling ackgcphase.
58 // All phase action must be benign in the presence of a change.
59 // Starting with GCoff
61 // GSscan scans stacks and globals greying them and never marks an object black.
62 // Once all the P's are aware of the new phase they will scan gs on preemption.
63 // This means that the scanning of preempted gs can't start until all the Ps
66 // GCMark turns on the write barrier which also only greys objects. No scanning
67 // of objects (making them black) can happen until all the Ps have acknowledged
69 // GCmark to GCmarktermination
70 // The only change here is that we start allocating black so the Ps must acknowledge
71 // the change before we begin the termination algorithm
72 // GCmarktermination to GSsweep
73 // Object currently on the freelist must be marked black for this to work.
74 // Are things on the free lists black or white? How does the sweep phase work?
77 // The sweep phase proceeds concurrently with normal program execution.
78 // The heap is swept span-by-span both lazily (when a goroutine needs another span)
79 // and concurrently in a background goroutine (this helps programs that are not CPU bound).
80 // However, at the end of the stop-the-world GC phase we don't know the size of the live heap,
81 // and so next_gc calculation is tricky and happens as follows.
82 // At the end of the stop-the-world phase next_gc is conservatively set based on total
83 // heap size; all spans are marked as "needs sweeping".
84 // Whenever a span is swept, next_gc is decremented by GOGC*newly_freed_memory.
85 // The background sweeper goroutine simply sweeps spans one-by-one bringing next_gc
86 // closer to the target value. However, this is not enough to avoid over-allocating memory.
87 // Consider that a goroutine wants to allocate a new span for a large object and
88 // there are no free swept spans, but there are small-object unswept spans.
89 // If the goroutine naively allocates a new span, it can surpass the yet-unknown
90 // target next_gc value. In order to prevent such cases (1) when a goroutine needs
91 // to allocate a new small-object span, it sweeps small-object spans for the same
92 // object size until it frees at least one object; (2) when a goroutine needs to
93 // allocate large-object span from heap, it sweeps spans until it frees at least
94 // that many pages into heap. Together these two measures ensure that we don't surpass
95 // target next_gc value by a large margin. There is an exception: if a goroutine sweeps
96 // and frees two nonadjacent one-page spans to the heap, it will allocate a new two-page span,
97 // but there can still be other one-page unswept spans which could be combined into a two-page span.
98 // It's critical to ensure that no operations proceed on unswept spans (that would corrupt
99 // mark bits in GC bitmap). During GC all mcaches are flushed into the central cache,
100 // so they are empty. When a goroutine grabs a new span into mcache, it sweeps it.
101 // When a goroutine explicitly frees an object or sets a finalizer, it ensures that
102 // the span is swept (either by sweeping it, or by waiting for the concurrent sweep to finish).
103 // The finalizer goroutine is kicked off only when all spans are swept.
104 // When the next GC starts, it sweeps all not-yet-swept spans (if any).
107 // Next GC is after we've allocated an extra amount of memory proportional to
108 // the amount already in use. The proportion is controlled by GOGC environment variable
109 // (100 by default). If GOGC=100 and we're using 4M, we'll GC again when we get to 8M
110 // (this mark is tracked in next_gc variable). This keeps the GC cost in linear
111 // proportion to the allocation cost. Adjusting GOGC just changes the linear constant
112 // (and also the amount of extra memory used).
120 _DebugGCPtrs = false // if true, print trace of every pointer load during GC
121 _ConcurrentSweep = true
123 _WorkbufSize = 4 * 1024
124 _FinBlockSize = 4 * 1024
133 // ptrmask for an allocation containing a single pointer.
134 var oneptr = [...]uint8{bitsPointer}
136 // Initialized from $GOGC. GOGC=off means no GC.
139 // Holding worldsema grants an M the right to try to stop the world.
142 // semacquire(&worldsema);
149 // semrelease(&worldsema);
152 var worldsema uint32 = 1
154 // It is a bug if bits does not have bitBoundary set but
155 // there are still some cases where this happens related
157 type markbits struct {
158 bitp *byte // pointer to the byte holding xbits
159 shift uintptr // bits xbits needs to be shifted to get bits
160 xbits byte // byte holding all the bits from *bitp
161 bits byte // mark and boundary bits relevant to corresponding slot.
162 tbits byte // pointer||scalar bits relevant to corresponding slot.
165 type workbuf struct {
166 node lfnode // must be first
168 obj [(_WorkbufSize - unsafe.Sizeof(lfnode{}) - ptrSize) / ptrSize]uintptr
171 var data, edata, bss, ebss, gcdata, gcbss struct{}
173 var finlock mutex // protects the following variables
174 var fing *g // goroutine that runs finalizers
175 var finq *finblock // list of finalizers that are to be executed
176 var finc *finblock // cache of free blocks
177 var finptrmask [_FinBlockSize / ptrSize / pointersPerByte]byte
180 var allfin *finblock // list of all blocks
182 var gcdatamask bitvector
183 var gcbssmask bitvector
187 var badblock [1024]uintptr
190 type workdata struct {
191 full uint64 // lock-free list of full blocks
192 empty uint64 // lock-free list of empty blocks
193 partial uint64 // lock-free list of partially filled blocks
194 pad0 [_CacheLineSize]uint8 // prevents false-sharing between full/empty and nproc/nwait
202 // Copy of mheap.allspans for marker or sweeper.
208 //go:linkname weak_cgo_allocate go.weak.runtime._cgo_allocate_internal
209 var weak_cgo_allocate byte
211 // Is _cgo_allocate linked into the binary?
212 func have_cgo_allocate() bool {
213 return &weak_cgo_allocate != nil
216 // To help debug the concurrent GC we remark with the world
217 // stopped ensuring that any object encountered has their normal
218 // mark bit set. To do this we use an orthogonal bit
219 // pattern to indicate the object is marked. The following pattern
220 // uses the upper two bits in the object's bounday nibble.
221 // 01: scalar not marked
222 // 10: pointer not marked
223 // 11: pointer marked
225 // Xoring with 01 will flip the pattern from marked to unmarked and vica versa.
226 // The higher bit is 1 for pointers and 0 for scalars, whether the object
228 // The first nibble no longer holds the bitsDead pattern indicating that the
229 // there are no more pointers in the object. This information is held
230 // in the second nibble.
232 // When marking an object if the bool checkmark is true one uses the above
233 // encoding, otherwise one uses the bitMarked bit in the lower two bits
237 gccheckmarkenable = true
240 // Is address b in the known heap. If it doesn't have a valid gcmap
241 // returns false. For example pointers into stacks will return false.
242 func inheap(b uintptr) bool {
243 if b == 0 || b < mheap_.arena_start || b >= mheap_.arena_used {
246 // Not a beginning of a block, consult span table to find the block beginning.
249 x -= mheap_.arena_start >> _PageShift
251 if s == nil || pageID(k) < s.start || b >= s.limit || s.state != mSpanInUse {
257 // Given an address in the heap return the relevant byte from the gcmap. This routine
258 // can be used on addresses to the start of an object or to the interior of the an object.
259 func slottombits(obj uintptr, mbits *markbits) {
260 off := (obj&^(ptrSize-1) - mheap_.arena_start) / ptrSize
261 mbits.bitp = (*byte)(unsafe.Pointer(mheap_.arena_start - off/wordsPerBitmapByte - 1))
262 mbits.shift = off % wordsPerBitmapByte * gcBits
263 mbits.xbits = *mbits.bitp
264 mbits.bits = (mbits.xbits >> mbits.shift) & bitMask
265 mbits.tbits = ((mbits.xbits >> mbits.shift) & bitPtrMask) >> 2
268 // b is a pointer into the heap.
269 // Find the start of the object refered to by b.
270 // Set mbits to the associated bits from the bit map.
271 // If b is not a valid heap object return nil and
272 // undefined values in mbits.
273 func objectstart(b uintptr, mbits *markbits) uintptr {
274 obj := b &^ (ptrSize - 1)
276 slottombits(obj, mbits)
277 if mbits.bits&bitBoundary == bitBoundary {
281 // Not a beginning of a block, consult span table to find the block beginning.
284 x -= mheap_.arena_start >> _PageShift
286 if s == nil || pageID(k) < s.start || b >= s.limit || s.state != mSpanInUse {
287 if s != nil && s.state == _MSpanStack {
288 return 0 // This is legit.
291 // The following ensures that we are rigorous about what data
292 // structures hold valid pointers
294 // Still happens sometimes. We don't know why.
296 print("runtime:objectstart Span weird: obj=", hex(obj), " k=", hex(k))
300 print(" s.start=", hex(s.start<<_PageShift), " s.limit=", hex(s.limit), " s.state=", s.state, "\n")
303 gothrow("objectstart: bad pointer in unexpected span")
308 p := uintptr(s.start) << _PageShift
309 if s.sizeclass != 0 {
311 idx := (obj - p) / size
315 print("runtime: failed to find block beginning for ", hex(p), " s=", hex(s.start*_PageSize), " s.limit=", s.limit, "\n")
316 gothrow("failed to find block beginning")
321 // if size(obj.firstfield) < PtrSize, the &obj.secondfield could map to the boundary bit
322 // Clear any low bits to get to the start of the object.
323 // greyobject depends on this.
327 // Slow for now as we serialize this, since this is on a debug path
328 // speed is not critical at this point.
331 func atomicand8(src *byte, val byte) {
337 // Mark using the checkmark scheme.
338 func docheckmark(mbits *markbits) {
339 // xor 01 moves 01(scalar unmarked) to 00(scalar marked)
340 // and 10(pointer unmarked) to 11(pointer marked)
341 if mbits.tbits == _BitsScalar {
342 atomicand8(mbits.bitp, ^byte(_BitsCheckMarkXor<<mbits.shift<<2))
343 } else if mbits.tbits == _BitsPointer {
344 atomicor8(mbits.bitp, byte(_BitsCheckMarkXor<<mbits.shift<<2))
347 // reload bits for ischeckmarked
348 mbits.xbits = *mbits.bitp
349 mbits.bits = (mbits.xbits >> mbits.shift) & bitMask
350 mbits.tbits = ((mbits.xbits >> mbits.shift) & bitPtrMask) >> 2
353 // In the default scheme does mbits refer to a marked object.
354 func ismarked(mbits *markbits) bool {
355 if mbits.bits&bitBoundary != bitBoundary {
356 gothrow("ismarked: bits should have boundary bit set")
358 return mbits.bits&bitMarked == bitMarked
361 // In the checkmark scheme does mbits refer to a marked object.
362 func ischeckmarked(mbits *markbits) bool {
363 if mbits.bits&bitBoundary != bitBoundary {
364 gothrow("ischeckmarked: bits should have boundary bit set")
366 return mbits.tbits == _BitsScalarMarked || mbits.tbits == _BitsPointerMarked
369 // When in GCmarkterminate phase we allocate black.
370 func gcmarknewobject_m(obj uintptr) {
371 if gcphase != _GCmarktermination {
372 gothrow("marking new object while not in mark termination phase")
374 if checkmark { // The world should be stopped so this should not happen.
375 gothrow("gcmarknewobject called while doing checkmark")
379 slottombits(obj, &mbits)
380 if mbits.bits&bitMarked != 0 {
384 // Each byte of GC bitmap holds info for two words.
385 // If the current object is larger than two words, or if the object is one word
386 // but the object it shares the byte with is already marked,
387 // then all the possible concurrent updates are trying to set the same bit,
388 // so we can use a non-atomic update.
389 if mbits.xbits&(bitMask|(bitMask<<gcBits)) != bitBoundary|bitBoundary<<gcBits || work.nproc == 1 {
390 *mbits.bitp = mbits.xbits | bitMarked<<mbits.shift
392 atomicor8(mbits.bitp, bitMarked<<mbits.shift)
396 // obj is the start of an object with mark mbits.
397 // If it isn't already marked, mark it and enqueue into workbuf.
398 // Return possibly new workbuf to use.
399 func greyobject(obj uintptr, mbits *markbits, wbuf *workbuf) *workbuf {
400 // obj should be start of allocation, and so must be at least pointer-aligned.
401 if obj&(ptrSize-1) != 0 {
402 gothrow("greyobject: obj not pointer-aligned")
406 if !ismarked(mbits) {
407 print("runtime:greyobject: checkmarks finds unexpected unmarked object obj=", hex(obj), ", mbits->bits=", hex(mbits.bits), " *mbits->bitp=", hex(*mbits.bitp), "\n")
409 k := obj >> _PageShift
411 x -= mheap_.arena_start >> _PageShift
414 print("runtime:greyobject Span: obj=", hex(obj), " k=", hex(k))
418 print(" s.start=", hex(s.start*_PageSize), " s.limit=", hex(s.limit), " s.sizeclass=", s.sizeclass, " s.elemsize=", s.elemsize, "\n")
419 // NOTE(rsc): This code is using s.sizeclass as an approximation of the
420 // number of pointer-sized words in an object. Perhaps not what was intended.
421 for i := 0; i < int(s.sizeclass); i++ {
422 print(" *(obj+", i*ptrSize, ") = ", hex(*(*uintptr)(unsafe.Pointer(obj + uintptr(i)*ptrSize))), "\n")
425 gothrow("checkmark found unmarked object")
427 if ischeckmarked(mbits) {
431 if !ischeckmarked(mbits) {
432 print("mbits xbits=", hex(mbits.xbits), " bits=", hex(mbits.bits), " tbits=", hex(mbits.tbits), " shift=", mbits.shift, "\n")
433 gothrow("docheckmark and ischeckmarked disagree")
436 // If marked we have nothing to do.
437 if mbits.bits&bitMarked != 0 {
441 // Each byte of GC bitmap holds info for two words.
442 // If the current object is larger than two words, or if the object is one word
443 // but the object it shares the byte with is already marked,
444 // then all the possible concurrent updates are trying to set the same bit,
445 // so we can use a non-atomic update.
446 if mbits.xbits&(bitMask|bitMask<<gcBits) != bitBoundary|bitBoundary<<gcBits || work.nproc == 1 {
447 *mbits.bitp = mbits.xbits | bitMarked<<mbits.shift
449 atomicor8(mbits.bitp, bitMarked<<mbits.shift)
453 if !checkmark && (mbits.xbits>>(mbits.shift+2))&_BitsMask == _BitsDead {
454 return wbuf // noscan object
457 // Queue the obj for scanning. The PREFETCH(obj) logic has been removed but
458 // seems like a nice optimization that can be added back in.
459 // There needs to be time between the PREFETCH and the use.
460 // Previously we put the obj in an 8 element buffer that is drained at a rate
461 // to give the PREFETCH time to do its work.
462 // Use of PREFETCHNTA might be more appropriate than PREFETCH
464 // If workbuf is full, obtain an empty one.
465 if wbuf.nobj >= uintptr(len(wbuf.obj)) {
466 wbuf = getempty(wbuf)
469 wbuf.obj[wbuf.nobj] = obj
474 // Scan the object b of size n, adding pointers to wbuf.
475 // Return possibly new wbuf to use.
476 // If ptrmask != nil, it specifies where pointers are in b.
477 // If ptrmask == nil, the GC bitmap should be consulted.
478 // In this case, n may be an overestimate of the size; the GC bitmap
479 // must also be used to make sure the scan stops at the end of b.
480 func scanobject(b, n uintptr, ptrmask *uint8, wbuf *workbuf) *workbuf {
481 arena_start := mheap_.arena_start
482 arena_used := mheap_.arena_used
484 // Find bits of the beginning of the object.
485 var ptrbitp unsafe.Pointer
488 b = objectstart(b, &mbits)
492 ptrbitp = unsafe.Pointer(mbits.bitp)
494 for i := uintptr(0); i < n; i += ptrSize {
495 // Find bits for this word.
498 // dense mask (stack or data)
499 bits = (uintptr(*(*byte)(add(unsafe.Pointer(ptrmask), (i/ptrSize)/4))) >> (((i / ptrSize) % 4) * bitsPerPointer)) & bitsMask
501 // Check if we have reached end of span.
502 // n is an overestimate of the size of the object.
503 if (b+i)%_PageSize == 0 && h_spans[(b-arena_start)>>_PageShift] != h_spans[(b+i-arena_start)>>_PageShift] {
507 // Consult GC bitmap.
508 bits = uintptr(*(*byte)(ptrbitp))
509 if wordsPerBitmapByte != 2 {
510 gothrow("alg doesn't work for wordsPerBitmapByte != 2")
512 j := (uintptr(b) + i) / ptrSize & 1 // j indicates upper nibble or lower nibble
517 ptrbitp = add(ptrbitp, -j)
519 if bits&bitBoundary != 0 && i != 0 {
520 break // reached beginning of the next object
522 bits = (bits & bitPtrMask) >> 2 // bits refer to the type bits.
524 if i != 0 && bits == bitsDead { // BitsDead in first nibble not valid during checkmark
525 break // reached no-scan part of the object
529 if bits <= _BitsScalar { // _BitsScalar, _BitsDead, _BitsScalarMarked
533 if bits&_BitsPointer != _BitsPointer {
534 print("gc checkmark=", checkmark, " b=", hex(b), " ptrmask=", ptrmask, " mbits.bitp=", mbits.bitp, " mbits.xbits=", hex(mbits.xbits), " bits=", hex(bits), "\n")
535 gothrow("unexpected garbage collection bits")
538 obj := *(*uintptr)(unsafe.Pointer(b + i))
540 // At this point we have extracted the next potential pointer.
541 // Check if it points into heap.
542 if obj == 0 || obj < arena_start || obj >= arena_used {
546 // Mark the object. return some important bits.
547 // We we combine the following two rotines we don't have to pass mbits or obj around.
549 obj = objectstart(obj, &mbits)
553 wbuf = greyobject(obj, &mbits, wbuf)
558 // scanblock starts by scanning b as scanobject would.
559 // If the gcphase is GCscan, that's all scanblock does.
560 // Otherwise it traverses some fraction of the pointers it found in b, recursively.
561 // As a special case, scanblock(nil, 0, nil) means to scan previously queued work,
562 // stopping only when no work is left in the system.
563 func scanblock(b, n uintptr, ptrmask *uint8) {
564 wbuf := getpartialorempty()
566 wbuf = scanobject(b, n, ptrmask, wbuf)
567 if gcphase == _GCscan {
568 if inheap(b) && ptrmask == nil {
569 // b is in heap, we are in GCscan so there should be a ptrmask.
570 gothrow("scanblock: In GCscan phase and inheap is true.")
572 // GCscan only goes one level deep since mark wb not turned on.
577 if gcphase == _GCscan {
578 gothrow("scanblock: In GCscan phase but no b passed in.")
581 keepworking := b == 0
583 // ptrmask can have 2 possible values:
584 // 1. nil - obtain pointer mask from GC bitmap.
585 // 2. pointer to a compact mask (for stacks and data).
592 // Refill workbuf from global queue.
594 if wbuf == nil { // nil means out of work barrier reached
599 gothrow("runtime:scanblock getfull returns empty buffer")
603 // If another proc wants a pointer, give it some.
604 if work.nwait > 0 && wbuf.nobj > 4 && work.full == 0 {
608 // This might be a good place to add prefetch code...
609 // if(wbuf->nobj > 4) {
610 // PREFETCH(wbuf->obj[wbuf->nobj - 3];
613 b = wbuf.obj[wbuf.nobj]
614 wbuf = scanobject(b, mheap_.arena_used-b, nil, wbuf)
618 func markroot(desc *parfor, i uint32) {
619 // Note: if you add a case here, please also update heapdump.c:dumproots.
622 scanblock(uintptr(unsafe.Pointer(&data)), uintptr(unsafe.Pointer(&edata))-uintptr(unsafe.Pointer(&data)), gcdatamask.bytedata)
625 scanblock(uintptr(unsafe.Pointer(&bss)), uintptr(unsafe.Pointer(&ebss))-uintptr(unsafe.Pointer(&bss)), gcbssmask.bytedata)
627 case _RootFinalizers:
628 for fb := allfin; fb != nil; fb = fb.alllink {
629 scanblock(uintptr(unsafe.Pointer(&fb.fin[0])), uintptr(fb.cnt)*unsafe.Sizeof(fb.fin[0]), &finptrmask[0])
633 // mark MSpan.specials
634 sg := mheap_.sweepgen
635 for spanidx := uint32(0); spanidx < uint32(len(work.spans)); spanidx++ {
636 s := work.spans[spanidx]
637 if s.state != mSpanInUse {
640 if !checkmark && s.sweepgen != sg {
641 // sweepgen was updated (+2) during non-checkmark GC pass
642 print("sweep ", s.sweepgen, " ", sg, "\n")
643 gothrow("gc: unswept span")
645 for sp := s.specials; sp != nil; sp = sp.next {
646 if sp.kind != _KindSpecialFinalizer {
649 // don't mark finalized object, but scan it so we
650 // retain everything it points to.
651 spf := (*specialfinalizer)(unsafe.Pointer(sp))
652 // A finalizer can be set for an inner byte of an object, find object beginning.
653 p := uintptr(s.start<<_PageShift) + uintptr(spf.special.offset)/s.elemsize*s.elemsize
654 if gcphase != _GCscan {
655 scanblock(p, s.elemsize, nil) // scanned during mark phase
657 scanblock(uintptr(unsafe.Pointer(&spf.fn)), ptrSize, &oneptr[0])
661 case _RootFlushCaches:
662 if gcphase != _GCscan { // Do not flush mcaches during GCscan phase.
667 // the rest is scanning goroutine stacks
668 if uintptr(i-_RootCount) >= allglen {
669 gothrow("markroot: bad index")
671 gp := allgs[i-_RootCount]
673 // remember when we've first observed the G blocked
674 // needed only to output in traceback
675 status := readgstatus(gp) // We are not in a scan state
676 if (status == _Gwaiting || status == _Gsyscall) && gp.waitsince == 0 {
677 gp.waitsince = work.tstart
680 // Shrink a stack if not much of it is being used but not in the scan phase.
681 if gcphase != _GCscan { // Do not shrink during GCscan phase.
684 if readgstatus(gp) == _Gdead {
687 gp.gcworkdone = false
691 // goroutine will scan its own stack when it stops running.
692 // Wait until it has.
693 for readgstatus(gp) == _Grunning && !gp.gcworkdone {
696 // scanstack(gp) is done as part of gcphasework
697 // But to make sure we finished we need to make sure that
698 // the stack traps have all responded so drop into
699 // this while loop until they respond.
701 status = readgstatus(gp)
702 if status == _Gdead {
703 gp.gcworkdone = true // scan is a noop
706 if status == _Gwaiting || status == _Grunnable {
716 // Get an empty work buffer off the work.empty list,
717 // allocating new buffers as needed.
718 func getempty(b *workbuf) *workbuf {
724 b = (*workbuf)(lfstackpop(&work.empty))
726 if b != nil && b.nobj != 0 {
728 print("m", _g_.m.id, ": getempty: popped b=", b, " with non-zero b.nobj=", b.nobj, "\n")
729 gothrow("getempty: workbuffer not empty, b->nobj not 0")
732 b = (*workbuf)(persistentalloc(unsafe.Sizeof(*b), _CacheLineSize, &memstats.gc_sys))
738 func putempty(b *workbuf) {
740 gothrow("putempty: b->nobj not 0")
742 lfstackpush(&work.empty, &b.node)
745 func putfull(b *workbuf) {
747 gothrow("putfull: b->nobj <= 0")
749 lfstackpush(&work.full, &b.node)
752 // Get an partially empty work buffer
753 // if none are available get an empty one.
754 func getpartialorempty() *workbuf {
755 b := (*workbuf)(lfstackpop(&work.partial))
762 func putpartial(b *workbuf) {
764 lfstackpush(&work.empty, &b.node)
765 } else if b.nobj < uintptr(len(b.obj)) {
766 lfstackpush(&work.partial, &b.node)
767 } else if b.nobj == uintptr(len(b.obj)) {
768 lfstackpush(&work.full, &b.node)
770 print("b=", b, " b.nobj=", b.nobj, " len(b.obj)=", len(b.obj), "\n")
771 gothrow("putpartial: bad Workbuf b.nobj")
775 // Get a full work buffer off the work.full or a partially
776 // filled one off the work.partial list. If nothing is available
777 // wait until all the other gc helpers have finished and then
779 // getfull acts as a barrier for work.nproc helpers. As long as one
780 // gchelper is actively marking objects it
781 // may create a workbuffer that the other helpers can work on.
782 // The for loop either exits when a work buffer is found
783 // or when _all_ of the work.nproc GC helpers are in the loop
784 // looking for work and thus not capable of creating new work.
785 // This is in fact the termination condition for the STW mark
787 func getfull(b *workbuf) *workbuf {
792 b = (*workbuf)(lfstackpop(&work.full))
794 b = (*workbuf)(lfstackpop(&work.partial))
796 if b != nil || work.nproc == 1 {
800 xadd(&work.nwait, +1)
803 xadd(&work.nwait, -1)
804 b = (*workbuf)(lfstackpop(&work.full))
806 b = (*workbuf)(lfstackpop(&work.partial))
811 xadd(&work.nwait, +1)
813 if work.nwait == work.nproc {
818 _g_.m.gcstats.nprocyield++
821 _g_.m.gcstats.nosyield++
824 _g_.m.gcstats.nsleep++
830 func handoff(b *workbuf) *workbuf {
831 // Make new buffer with half of b's pointers.
836 memmove(unsafe.Pointer(&b1.obj[0]), unsafe.Pointer(&b.obj[b.nobj]), n*unsafe.Sizeof(b1.obj[0]))
838 _g_.m.gcstats.nhandoff++
839 _g_.m.gcstats.nhandoffcnt += uint64(n)
841 // Put b on full list - let first half of b get stolen.
842 lfstackpush(&work.full, &b.node)
846 func stackmapdata(stkmap *stackmap, n int32) bitvector {
847 if n < 0 || n >= stkmap.n {
848 gothrow("stackmapdata: index out of range")
850 return bitvector{stkmap.nbit, (*byte)(add(unsafe.Pointer(&stkmap.bytedata), uintptr(n*((stkmap.nbit+31)/32*4))))}
853 // Scan a stack frame: local variables and function arguments/results.
854 func scanframe(frame *stkframe, unused unsafe.Pointer) bool {
857 targetpc := frame.continpc
863 print("scanframe ", gofuncname(f), "\n")
865 if targetpc != f.entry {
868 pcdata := pcdatavalue(f, _PCDATA_StackMapIndex, targetpc)
870 // We do not have a valid pcdata value but there might be a
871 // stackmap for this function. It is likely that we are looking
872 // at the function prologue, assume so and hope for the best.
876 // Scan local variables if stack frame has been allocated.
877 size := frame.varp - frame.sp
879 if thechar != '6' && thechar != '8' {
885 stkmap := (*stackmap)(funcdata(f, _FUNCDATA_LocalsPointerMaps))
886 if stkmap == nil || stkmap.n <= 0 {
887 print("runtime: frame ", gofuncname(f), " untyped locals ", hex(frame.varp-size), "+", hex(size), "\n")
888 gothrow("missing stackmap")
891 // Locals bitmap information, scan just the pointers in locals.
892 if pcdata < 0 || pcdata >= stkmap.n {
893 // don't know where we are
894 print("runtime: pcdata is ", pcdata, " and ", stkmap.n, " locals stack map entries for ", gofuncname(f), " (targetpc=", targetpc, ")\n")
895 gothrow("scanframe: bad symbol table")
897 bv := stackmapdata(stkmap, pcdata)
898 size = (uintptr(bv.n) * ptrSize) / bitsPerPointer
899 scanblock(frame.varp-size, uintptr(bv.n)/bitsPerPointer*ptrSize, bv.bytedata)
903 if frame.arglen > 0 {
905 if frame.argmap != nil {
908 stkmap := (*stackmap)(funcdata(f, _FUNCDATA_ArgsPointerMaps))
909 if stkmap == nil || stkmap.n <= 0 {
910 print("runtime: frame ", gofuncname(f), " untyped args ", hex(frame.argp), "+", hex(frame.arglen), "\n")
911 gothrow("missing stackmap")
913 if pcdata < 0 || pcdata >= stkmap.n {
914 // don't know where we are
915 print("runtime: pcdata is ", pcdata, " and ", stkmap.n, " args stack map entries for ", gofuncname(f), " (targetpc=", targetpc, ")\n")
916 gothrow("scanframe: bad symbol table")
918 bv = stackmapdata(stkmap, pcdata)
920 scanblock(frame.argp, uintptr(bv.n)/bitsPerPointer*ptrSize, bv.bytedata)
925 func scanstack(gp *g) {
926 // TODO(rsc): Due to a precedence error, this was never checked in the original C version.
927 // If you enable the check, the gothrow happens.
929 if readgstatus(gp)&_Gscan == 0 {
930 print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n")
931 gothrow("mark - bad status")
935 switch readgstatus(gp) &^ _Gscan {
937 print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n")
938 gothrow("mark - bad status")
942 print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n")
943 gothrow("scanstack: goroutine not stopped")
944 case _Grunnable, _Gsyscall, _Gwaiting:
949 gothrow("can't scan our own stack")
952 if mp != nil && mp.helpgc != 0 {
953 gothrow("can't scan gchelper stack")
956 gentraceback(^uintptr(0), ^uintptr(0), 0, gp, 0, nil, 0x7fffffff, scanframe, nil, 0)
957 tracebackdefers(gp, scanframe, nil)
960 // If the slot is grey or black return true, if white return false.
961 // If the slot is not in the known heap and thus does not have a valid GC bitmap then
962 // it is considered grey. Globals and stacks can hold such slots.
963 // The slot is grey if its mark bit is set and it is enqueued to be scanned.
964 // The slot is black if it has already been scanned.
965 // It is white if it has a valid mark bit and the bit is not set.
966 func shaded(slot uintptr) bool {
967 if !inheap(slot) { // non-heap slots considered grey
972 valid := objectstart(slot, &mbits)
978 return ischeckmarked(&mbits)
981 return mbits.bits&bitMarked != 0
984 // Shade the object if it isn't already.
985 // The object is not nil and known to be in the heap.
986 func shade(b uintptr) {
988 gothrow("shade: passed an address not in the heap")
991 wbuf := getpartialorempty()
992 // Mark the object, return some important bits.
993 // If we combine the following two rotines we don't have to pass mbits or obj around.
995 obj := objectstart(b, &mbits)
997 wbuf = greyobject(obj, &mbits, wbuf) // augments the wbuf
1002 // This is the Dijkstra barrier coarsened to always shade the ptr (dst) object.
1003 // The original Dijkstra barrier only shaded ptrs being placed in black slots.
1005 // Shade indicates that it has seen a white pointer by adding the referent
1006 // to wbuf as well as marking it.
1008 // slot is the destination (dst) in go code
1009 // ptr is the value that goes into the slot (src) in the go code
1011 // Dijkstra pointed out that maintaining the no black to white
1012 // pointers means that white to white pointers not need
1013 // to be noted by the write barrier. Furthermore if either
1014 // white object dies before it is reached by the
1015 // GC then the object can be collected during this GC cycle
1016 // instead of waiting for the next cycle. Unfortunately the cost of
1017 // ensure that the object holding the slot doesn't concurrently
1018 // change to black without the mutator noticing seems prohibitive.
1020 // Consider the following example where the mutator writes into
1021 // a slot and then loads the slot's mark bit while the GC thread
1022 // writes to the slot's mark bit and then as part of scanning reads
1025 // Initially both [slot] and [slotmark] are 0 (nil)
1026 // Mutator thread GC thread
1027 // st [slot], ptr st [slotmark], 1
1029 // ld r1, [slotmark] ld r2, [slot]
1031 // This is a classic example of independent reads of independent writes,
1032 // aka IRIW. The question is if r1==r2==0 is allowed and for most HW the
1033 // answer is yes without inserting a memory barriers between the st and the ld.
1034 // These barriers are expensive so we have decided that we will
1035 // always grey the ptr object regardless of the slot's color.
1036 func gcmarkwb_m(slot *uintptr, ptr uintptr) {
1039 gothrow("gcphasework in bad gcphase")
1041 case _GCoff, _GCquiesce, _GCstw, _GCsweep, _GCscan:
1044 case _GCmark, _GCmarktermination:
1045 if ptr != 0 && inheap(ptr) {
1051 // The gp has been moved to a GC safepoint. GC phase specific
1052 // work is done here.
1053 func gcphasework(gp *g) {
1056 gothrow("gcphasework in bad gcphase")
1057 case _GCoff, _GCquiesce, _GCstw, _GCsweep:
1060 // scan the stack, mark the objects, put pointers in work buffers
1061 // hanging off the P where this is being run.
1065 case _GCmarktermination:
1067 // All available mark work will be emptied before returning.
1069 gp.gcworkdone = true
1072 var finalizer1 = [...]byte{
1073 // Each Finalizer is 5 words, ptr ptr uintptr ptr ptr.
1074 // Each byte describes 4 words.
1075 // Need 4 Finalizers described by 5 bytes before pattern repeats:
1076 // ptr ptr uintptr ptr ptr
1077 // ptr ptr uintptr ptr ptr
1078 // ptr ptr uintptr ptr ptr
1079 // ptr ptr uintptr ptr ptr
1081 // ptr ptr uintptr ptr
1082 // ptr ptr ptr uintptr
1084 // uintptr ptr ptr ptr
1085 // ptr uintptr ptr ptr
1086 // Assumptions about Finalizer layout checked below.
1087 bitsPointer | bitsPointer<<2 | bitsScalar<<4 | bitsPointer<<6,
1088 bitsPointer | bitsPointer<<2 | bitsPointer<<4 | bitsScalar<<6,
1089 bitsPointer | bitsPointer<<2 | bitsPointer<<4 | bitsPointer<<6,
1090 bitsScalar | bitsPointer<<2 | bitsPointer<<4 | bitsPointer<<6,
1091 bitsPointer | bitsScalar<<2 | bitsPointer<<4 | bitsPointer<<6,
1094 func queuefinalizer(p unsafe.Pointer, fn *funcval, nret uintptr, fint *_type, ot *ptrtype) {
1096 if finq == nil || finq.cnt == finq.cap {
1098 finc = (*finblock)(persistentalloc(_FinBlockSize, 0, &memstats.gc_sys))
1099 finc.cap = int32((_FinBlockSize-unsafe.Sizeof(finblock{}))/unsafe.Sizeof(finalizer{}) + 1)
1100 finc.alllink = allfin
1102 if finptrmask[0] == 0 {
1103 // Build pointer mask for Finalizer array in block.
1104 // Check assumptions made in finalizer1 array above.
1105 if (unsafe.Sizeof(finalizer{}) != 5*ptrSize ||
1106 unsafe.Offsetof(finalizer{}.fn) != 0 ||
1107 unsafe.Offsetof(finalizer{}.arg) != ptrSize ||
1108 unsafe.Offsetof(finalizer{}.nret) != 2*ptrSize ||
1109 unsafe.Offsetof(finalizer{}.fint) != 3*ptrSize ||
1110 unsafe.Offsetof(finalizer{}.ot) != 4*ptrSize ||
1111 bitsPerPointer != 2) {
1112 gothrow("finalizer out of sync")
1114 for i := range finptrmask {
1115 finptrmask[i] = finalizer1[i%len(finalizer1)]
1124 f := (*finalizer)(add(unsafe.Pointer(&finq.fin[0]), uintptr(finq.cnt)*unsafe.Sizeof(finq.fin[0])))
1135 func iterate_finq(callback func(*funcval, unsafe.Pointer, uintptr, *_type, *ptrtype)) {
1136 for fb := allfin; fb != nil; fb = fb.alllink {
1137 for i := int32(0); i < fb.cnt; i++ {
1139 callback(f.fn, f.arg, f.nret, f.fint, f.ot)
1144 // Returns only when span s has been swept.
1145 func mSpan_EnsureSwept(s *mspan) {
1146 // Caller must disable preemption.
1147 // Otherwise when this function returns the span can become unswept again
1148 // (if GC is triggered on another goroutine).
1150 if _g_.m.locks == 0 && _g_.m.mallocing == 0 && _g_ != _g_.m.g0 {
1151 gothrow("MSpan_EnsureSwept: m is not locked")
1154 sg := mheap_.sweepgen
1155 if atomicload(&s.sweepgen) == sg {
1158 // The caller must be sure that the span is a MSpanInUse span.
1159 if cas(&s.sweepgen, sg-2, sg-1) {
1160 mSpan_Sweep(s, false)
1163 // unfortunate condition, and we don't have efficient means to wait
1164 for atomicload(&s.sweepgen) != sg {
1169 // Sweep frees or collects finalizers for blocks not marked in the mark phase.
1170 // It clears the mark bits in preparation for the next GC round.
1171 // Returns true if the span was returned to heap.
1172 // If preserve=true, don't return it to heap nor relink in MCentral lists;
1173 // caller takes care of it.
1174 func mSpan_Sweep(s *mspan, preserve bool) bool {
1176 gothrow("MSpan_Sweep: checkmark only runs in STW and after the sweep")
1179 // It's critical that we enter this function with preemption disabled,
1180 // GC must not start while we are in the middle of this function.
1182 if _g_.m.locks == 0 && _g_.m.mallocing == 0 && _g_ != _g_.m.g0 {
1183 gothrow("MSpan_Sweep: m is not locked")
1185 sweepgen := mheap_.sweepgen
1186 if s.state != mSpanInUse || s.sweepgen != sweepgen-1 {
1187 print("MSpan_Sweep: state=", s.state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n")
1188 gothrow("MSpan_Sweep: bad span state")
1190 arena_start := mheap_.arena_start
1198 // Chunk full of small blocks.
1199 npages = class_to_allocnpages[cl]
1200 n = (npages << _PageShift) / int32(size)
1207 sweepgenset := false
1209 // Mark any free objects in this span so we don't collect them.
1210 for link := s.freelist; link != nil; link = link.next {
1211 off := (uintptr(unsafe.Pointer(link)) - arena_start) / ptrSize
1212 bitp := arena_start - off/wordsPerBitmapByte - 1
1213 shift := (off % wordsPerBitmapByte) * gcBits
1214 *(*byte)(unsafe.Pointer(bitp)) |= bitMarked << shift
1217 // Unlink & free special records for any objects we're about to free.
1218 specialp := &s.specials
1219 special := *specialp
1220 for special != nil {
1221 // A finalizer can be set for an inner byte of an object, find object beginning.
1222 p := uintptr(s.start<<_PageShift) + uintptr(special.offset)/size*size
1223 off := (p - arena_start) / ptrSize
1224 bitp := arena_start - off/wordsPerBitmapByte - 1
1225 shift := (off % wordsPerBitmapByte) * gcBits
1226 bits := (*(*byte)(unsafe.Pointer(bitp)) >> shift) & bitMask
1227 if bits&bitMarked == 0 {
1228 // Find the exact byte for which the special was setup
1229 // (as opposed to object beginning).
1230 p := uintptr(s.start<<_PageShift) + uintptr(special.offset)
1231 // about to free object: splice out special record
1233 special = special.next
1235 if !freespecial(y, unsafe.Pointer(p), size, false) {
1236 // stop freeing of object if it has a finalizer
1237 *(*byte)(unsafe.Pointer(bitp)) |= bitMarked << shift
1240 // object is still live: keep special record
1241 specialp = &special.next
1246 // Sweep through n objects of given size starting at p.
1247 // This thread owns the span now, so it can manipulate
1248 // the block bitmap without atomic operations.
1249 p := uintptr(s.start << _PageShift)
1250 off := (p - arena_start) / ptrSize
1251 bitp := arena_start - off/wordsPerBitmapByte - 1
1253 step := size / (ptrSize * wordsPerBitmapByte)
1254 // Rewind to the previous quadruple as we move to the next
1255 // in the beginning of the loop.
1262 for ; n > 0; n, p = n-1, p+size {
1268 shift = gcBits - shift
1271 xbits := *(*byte)(unsafe.Pointer(bitp))
1272 bits := (xbits >> shift) & bitMask
1274 // Allocated and marked object, reset bits to allocated.
1275 if bits&bitMarked != 0 {
1276 *(*byte)(unsafe.Pointer(bitp)) &^= bitMarked << shift
1280 // At this point we know that we are looking at garbage object
1281 // that needs to be collected.
1282 if debug.allocfreetrace != 0 {
1283 tracefree(unsafe.Pointer(p), size)
1286 // Reset to allocated+noscan.
1287 *(*byte)(unsafe.Pointer(bitp)) = uint8(uintptr(xbits&^((bitMarked|bitsMask<<2)<<shift)) | uintptr(bitsDead)<<(shift+2))
1291 gothrow("can't preserve large span")
1293 unmarkspan(p, s.npages<<_PageShift)
1296 // important to set sweepgen before returning it to heap
1297 atomicstore(&s.sweepgen, sweepgen)
1300 // NOTE(rsc,dvyukov): The original implementation of efence
1301 // in CL 22060046 used SysFree instead of SysFault, so that
1302 // the operating system would eventually give the memory
1303 // back to us again, so that an efence program could run
1304 // longer without running out of memory. Unfortunately,
1305 // calling SysFree here without any kind of adjustment of the
1306 // heap data structures means that when the memory does
1307 // come back to us, we have the wrong metadata for it, either in
1308 // the MSpan structures or in the garbage collection bitmap.
1309 // Using SysFault here means that the program will run out of
1310 // memory fairly quickly in efence mode, but at least it won't
1311 // have mysterious crashes due to confused memory reuse.
1312 // It should be possible to switch back to SysFree if we also
1313 // implement and then call some kind of MHeap_DeleteSpan.
1314 if debug.efence > 0 {
1315 s.limit = 0 // prevent mlookup from finding this span
1316 sysFault(unsafe.Pointer(p), size)
1318 mHeap_Free(&mheap_, s, 1)
1320 c.local_nlargefree++
1321 c.local_largefree += size
1322 xadd64(&memstats.next_gc, -int64(size)*int64(gcpercent+100)/100)
1325 // Free small object.
1326 if size > 2*ptrSize {
1327 *(*uintptr)(unsafe.Pointer(p + ptrSize)) = uintptrMask & 0xdeaddeaddeaddead // mark as "needs to be zeroed"
1328 } else if size > ptrSize {
1329 *(*uintptr)(unsafe.Pointer(p + ptrSize)) = 0
1331 end.next = (*mlink)(unsafe.Pointer(p))
1337 // We need to set s.sweepgen = h.sweepgen only when all blocks are swept,
1338 // because of the potential for a concurrent free/SetFinalizer.
1339 // But we need to set it before we make the span available for allocation
1340 // (return it to heap or mcentral), because allocation code assumes that a
1341 // span is already swept if available for allocation.
1342 if !sweepgenset && nfree == 0 {
1343 // The span must be in our exclusive ownership until we update sweepgen,
1344 // check for potential races.
1345 if s.state != mSpanInUse || s.sweepgen != sweepgen-1 {
1346 print("MSpan_Sweep: state=", s.state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n")
1347 gothrow("MSpan_Sweep: bad span state after sweep")
1349 atomicstore(&s.sweepgen, sweepgen)
1352 c.local_nsmallfree[cl] += uintptr(nfree)
1353 c.local_cachealloc -= intptr(uintptr(nfree) * size)
1354 xadd64(&memstats.next_gc, -int64(nfree)*int64(size)*int64(gcpercent+100)/100)
1355 res = mCentral_FreeSpan(&mheap_.central[cl].mcentral, s, int32(nfree), head.next, end, preserve)
1356 // MCentral_FreeSpan updates sweepgen
1361 // State of background sweep.
1362 // Protected by gclock.
1363 type sweepdata struct {
1368 spanidx uint32 // background sweeper position
1377 // returns number of pages returned to heap, or ^uintptr(0) if there is nothing to sweep
1378 func sweepone() uintptr {
1381 // increment locks to ensure that the goroutine is not preempted
1382 // in the middle of sweep thus leaving the span in an inconsistent state for next GC
1384 sg := mheap_.sweepgen
1386 idx := xadd(&sweep.spanidx, 1) - 1
1387 if idx >= uint32(len(work.spans)) {
1388 mheap_.sweepdone = 1
1392 s := work.spans[idx]
1393 if s.state != mSpanInUse {
1397 if s.sweepgen != sg-2 || !cas(&s.sweepgen, sg-2, sg-1) {
1401 if !mSpan_Sweep(s, false) {
1409 func gosweepone() uintptr {
1411 systemstack(func() {
1417 func gosweepdone() bool {
1418 return mheap_.sweepdone != 0
1426 // parallel mark for over GC roots
1427 parfordo(work.markfor)
1428 if gcphase != _GCscan {
1429 scanblock(0, 0, nil) // blocks in getfull
1432 nproc := work.nproc // work.nproc can change right after we increment work.ndone
1433 if xadd(&work.ndone, +1) == nproc-1 {
1434 notewakeup(&work.alldone)
1453 func flushallmcaches() {
1463 mCache_ReleaseAll(c)
1468 func updatememstats(stats *gcstats) {
1472 for mp := allm; mp != nil; mp = mp.alllink {
1474 src := (*[unsafe.Sizeof(gcstats{}) / 8]uint64)(unsafe.Pointer(&mp.gcstats))
1475 dst := (*[unsafe.Sizeof(gcstats{}) / 8]uint64)(unsafe.Pointer(stats))
1476 for i, v := range src {
1479 mp.gcstats = gcstats{}
1483 memstats.mcache_inuse = uint64(mheap_.cachealloc.inuse)
1484 memstats.mspan_inuse = uint64(mheap_.spanalloc.inuse)
1485 memstats.sys = memstats.heap_sys + memstats.stacks_sys + memstats.mspan_sys +
1486 memstats.mcache_sys + memstats.buckhash_sys + memstats.gc_sys + memstats.other_sys
1488 // Calculate memory allocator stats.
1489 // During program execution we only count number of frees and amount of freed memory.
1490 // Current number of alive object in the heap and amount of alive heap memory
1491 // are calculated by scanning all spans.
1492 // Total number of mallocs is calculated as number of frees plus number of alive objects.
1493 // Similarly, total amount of allocated memory is calculated as amount of freed memory
1494 // plus amount of alive heap memory.
1496 memstats.total_alloc = 0
1497 memstats.nmalloc = 0
1499 for i := 0; i < len(memstats.by_size); i++ {
1500 memstats.by_size[i].nmalloc = 0
1501 memstats.by_size[i].nfree = 0
1504 // Flush MCache's to MCentral.
1505 systemstack(flushallmcaches)
1507 // Aggregate local stats.
1510 // Scan all spans and count number of alive objects.
1512 for i := uint32(0); i < mheap_.nspan; i++ {
1514 if s.state != mSpanInUse {
1517 if s.sizeclass == 0 {
1519 memstats.alloc += uint64(s.elemsize)
1521 memstats.nmalloc += uint64(s.ref)
1522 memstats.by_size[s.sizeclass].nmalloc += uint64(s.ref)
1523 memstats.alloc += uint64(s.ref) * uint64(s.elemsize)
1526 unlock(&mheap_.lock)
1528 // Aggregate by size class.
1529 smallfree := uint64(0)
1530 memstats.nfree = mheap_.nlargefree
1531 for i := 0; i < len(memstats.by_size); i++ {
1532 memstats.nfree += mheap_.nsmallfree[i]
1533 memstats.by_size[i].nfree = mheap_.nsmallfree[i]
1534 memstats.by_size[i].nmalloc += mheap_.nsmallfree[i]
1535 smallfree += uint64(mheap_.nsmallfree[i]) * uint64(class_to_size[i])
1537 memstats.nfree += memstats.tinyallocs
1538 memstats.nmalloc += memstats.nfree
1540 // Calculate derived stats.
1541 memstats.total_alloc = uint64(memstats.alloc) + uint64(mheap_.largefree) + smallfree
1542 memstats.heap_alloc = memstats.alloc
1543 memstats.heap_objects = memstats.nmalloc - memstats.nfree
1547 if unsafe.Sizeof(workbuf{}) != _WorkbufSize {
1548 gothrow("runtime: size of Workbuf is suboptimal")
1551 work.markfor = parforalloc(_MaxGcproc)
1552 gcpercent = readgogc()
1553 gcdatamask = unrollglobgcprog((*byte)(unsafe.Pointer(&gcdata)), uintptr(unsafe.Pointer(&edata))-uintptr(unsafe.Pointer(&data)))
1554 gcbssmask = unrollglobgcprog((*byte)(unsafe.Pointer(&gcbss)), uintptr(unsafe.Pointer(&ebss))-uintptr(unsafe.Pointer(&bss)))
1557 // Called from malloc.go using onM, stopping and starting the world handled in caller.
1558 func gc_m(start_time int64, eagersweep bool) {
1561 casgstatus(gp, _Grunning, _Gwaiting)
1562 gp.waitreason = "garbage collection"
1564 gc(start_time, eagersweep)
1565 casgstatus(gp, _Gwaiting, _Grunning)
1568 // Similar to clearcheckmarkbits but works on a single span.
1569 // It preforms two tasks.
1570 // 1. When used before the checkmark phase it converts BitsDead (00) to bitsScalar (01)
1571 // for nibbles with the BoundaryBit set.
1572 // 2. When used after the checkmark phase it converts BitsPointerMark (11) to BitsPointer 10 and
1573 // BitsScalarMark (00) to BitsScalar (01), thus clearing the checkmark mark encoding.
1574 // For the second case it is possible to restore the BitsDead pattern but since
1575 // clearmark is a debug tool performance has a lower priority than simplicity.
1576 // The span is MSpanInUse and the world is stopped.
1577 func clearcheckmarkbitsspan(s *mspan) {
1578 if s.state != _MSpanInUse {
1579 print("runtime:clearcheckmarkbitsspan: state=", s.state, "\n")
1580 gothrow("clearcheckmarkbitsspan: bad span state")
1583 arena_start := mheap_.arena_start
1590 // Chunk full of small blocks
1591 npages := class_to_allocnpages[cl]
1592 n = npages << _PageShift / int32(size)
1595 // MSpan_Sweep has similar code but instead of overloading and
1596 // complicating that routine we do a simpler walk here.
1597 // Sweep through n objects of given size starting at p.
1598 // This thread owns the span now, so it can manipulate
1599 // the block bitmap without atomic operations.
1600 p := uintptr(s.start) << _PageShift
1602 // Find bits for the beginning of the span.
1603 off := (p - arena_start) / ptrSize
1604 bitp := (*byte)(unsafe.Pointer(arena_start - off/wordsPerBitmapByte - 1))
1605 step := size / (ptrSize * wordsPerBitmapByte)
1607 // The type bit values are:
1608 // 00 - BitsDead, for us BitsScalarMarked
1611 // 11 - unused, for us BitsPointerMarked
1613 // When called to prepare for the checkmark phase (checkmark==1),
1614 // we change BitsDead to BitsScalar, so that there are no BitsScalarMarked
1615 // type bits anywhere.
1617 // The checkmark phase marks by changing BitsScalar to BitsScalarMarked
1618 // and BitsPointer to BitsPointerMarked.
1620 // When called to clean up after the checkmark phase (checkmark==0),
1621 // we unmark by changing BitsScalarMarked back to BitsScalar and
1622 // BitsPointerMarked back to BitsPointer.
1624 // There are two problems with the scheme as just described.
1625 // First, the setup rewrites BitsDead to BitsScalar, but the type bits
1626 // following a BitsDead are uninitialized and must not be used.
1627 // Second, objects that are free are expected to have their type
1628 // bits zeroed (BitsDead), so in the cleanup we need to restore
1629 // any BitsDeads that were there originally.
1631 // In a one-word object (8-byte allocation on 64-bit system),
1632 // there is no difference between BitsScalar and BitsDead, because
1633 // neither is a pointer and there are no more words in the object,
1634 // so using BitsScalar during the checkmark is safe and mapping
1635 // both back to BitsDead during cleanup is also safe.
1637 // In a larger object, we need to be more careful. During setup,
1638 // if the type of the first word is BitsDead, we change it to BitsScalar
1639 // (as we must) but also initialize the type of the second
1640 // word to BitsDead, so that a scan during the checkmark phase
1641 // will still stop before seeing the uninitialized type bits in the
1642 // rest of the object. The sequence 'BitsScalar BitsDead' never
1643 // happens in real type bitmaps - BitsDead is always as early
1644 // as possible, so immediately after the last BitsPointer.
1645 // During cleanup, if we see a BitsScalar, we can check to see if it
1646 // is followed by BitsDead. If so, it was originally BitsDead and
1647 // we can change it back.
1650 // updating top and bottom nibbles, all boundaries
1651 for i := int32(0); i < n/2; i, bitp = i+1, addb(bitp, uintptrMask&-1) {
1652 if *bitp&bitBoundary == 0 {
1653 gothrow("missing bitBoundary")
1655 b := (*bitp & bitPtrMask) >> 2
1656 if !checkmark && (b == _BitsScalar || b == _BitsScalarMarked) {
1657 *bitp &^= 0x0c // convert to _BitsDead
1658 } else if b == _BitsScalarMarked || b == _BitsPointerMarked {
1659 *bitp &^= _BitsCheckMarkXor << 2
1662 if (*bitp>>gcBits)&bitBoundary == 0 {
1663 gothrow("missing bitBoundary")
1665 b = ((*bitp >> gcBits) & bitPtrMask) >> 2
1666 if !checkmark && (b == _BitsScalar || b == _BitsScalarMarked) {
1667 *bitp &^= 0xc0 // convert to _BitsDead
1668 } else if b == _BitsScalarMarked || b == _BitsPointerMarked {
1669 *bitp &^= _BitsCheckMarkXor << (2 + gcBits)
1673 // updating bottom nibble for first word of each object
1674 for i := int32(0); i < n; i, bitp = i+1, addb(bitp, -step) {
1675 if *bitp&bitBoundary == 0 {
1676 gothrow("missing bitBoundary")
1678 b := (*bitp & bitPtrMask) >> 2
1680 if checkmark && b == _BitsDead {
1681 // move BitsDead into second word.
1682 // set bits to BitsScalar in preparation for checkmark phase.
1684 *bitp |= _BitsScalar << 2
1685 } else if !checkmark && (b == _BitsScalar || b == _BitsScalarMarked) && *bitp&0xc0 == 0 {
1686 // Cleaning up after checkmark phase.
1687 // First word is scalar or dead (we forgot)
1688 // and second word is dead.
1689 // First word might as well be dead too.
1691 } else if b == _BitsScalarMarked || b == _BitsPointerMarked {
1692 *bitp ^= _BitsCheckMarkXor << 2
1698 // clearcheckmarkbits preforms two tasks.
1699 // 1. When used before the checkmark phase it converts BitsDead (00) to bitsScalar (01)
1700 // for nibbles with the BoundaryBit set.
1701 // 2. When used after the checkmark phase it converts BitsPointerMark (11) to BitsPointer 10 and
1702 // BitsScalarMark (00) to BitsScalar (01), thus clearing the checkmark mark encoding.
1703 // This is a bit expensive but preserves the BitsDead encoding during the normal marking.
1704 // BitsDead remains valid for every nibble except the ones with BitsBoundary set.
1705 func clearcheckmarkbits() {
1706 for _, s := range work.spans {
1707 if s.state == _MSpanInUse {
1708 clearcheckmarkbitsspan(s)
1713 // Called from malloc.go using onM.
1714 // The world is stopped. Rerun the scan and mark phases
1715 // using the bitMarkedCheck bit instead of the
1716 // bitMarked bit. If the marking encounters an
1717 // bitMarked bit that is not set then we throw.
1718 func gccheckmark_m(startTime int64, eagersweep bool) {
1719 if !gccheckmarkenable {
1724 gothrow("gccheckmark_m, entered with checkmark already true")
1728 clearcheckmarkbits() // Converts BitsDead to BitsScalar.
1729 gc_m(startTime, eagersweep) // turns off checkmark
1730 // Work done, fixed up the GC bitmap to remove the checkmark bits.
1731 clearcheckmarkbits()
1734 func gccheckmarkenable_m() {
1735 gccheckmarkenable = true
1738 func gccheckmarkdisable_m() {
1739 gccheckmarkenable = false
1742 func finishsweep_m() {
1743 // The world is stopped so we should be able to complete the sweeps
1745 for sweepone() != ^uintptr(0) {
1749 // There may be some other spans being swept concurrently that
1750 // we need to wait for. If finishsweep_m is done with the world stopped
1751 // this code is not required.
1752 sg := mheap_.sweepgen
1753 for _, s := range work.spans {
1754 if s.sweepgen != sg && s.state == _MSpanInUse {
1755 mSpan_EnsureSwept(s)
1760 // Scan all of the stacks, greying (or graying if in America) the referents
1761 // but not blackening them since the mark write barrier isn't installed.
1765 // Grab the g that called us and potentially allow rescheduling.
1766 // This allows it to be scanned like other goroutines.
1767 mastergp := _g_.m.curg
1768 casgstatus(mastergp, _Grunning, _Gwaiting)
1769 mastergp.waitreason = "garbage collection scan"
1771 // Span sweeping has been done by finishsweep_m.
1772 // Long term we will want to make this goroutine runnable
1773 // by placing it onto a scanenqueue state and then calling
1774 // runtime·restartg(mastergp) to make it Grunnable.
1775 // At the bottom we will want to return this p back to the scheduler.
1778 // Prepare flag indicating that the scan has not been completed.
1780 local_allglen := allglen
1781 for i := uintptr(0); i < local_allglen; i++ {
1783 gp.gcworkdone = false // set to true in gcphasework
1789 work.nproc = 1 // For now do not do this in parallel.
1791 // ackgcphase is not needed since we are not scanning running goroutines.
1792 parforsetup(work.markfor, work.nproc, uint32(_RootCount+local_allglen), nil, false, markroot)
1793 parfordo(work.markfor)
1796 // Check that gc work is done.
1797 for i := uintptr(0); i < local_allglen; i++ {
1800 gothrow("scan missed a g")
1806 casgstatus(mastergp, _Gwaiting, _Grunning)
1807 // Let the g that called us continue to run.
1810 // Mark all objects that are known about.
1812 scanblock(0, 0, nil)
1815 // For now this must be bracketed with a stoptheworld and a starttheworld to ensure
1816 // all go routines see the new barrier.
1817 func gcinstallmarkwb_m() {
1821 // For now this must be bracketed with a stoptheworld and a starttheworld to ensure
1822 // all go routines see the new barrier.
1823 func gcinstalloffwb_m() {
1827 func gc(start_time int64, eagersweep bool) {
1832 if debug.allocfreetrace > 0 {
1839 work.tstart = start_time
1842 if debug.gctrace > 0 {
1847 finishsweep_m() // skip during checkmark debug phase.
1850 // Cache runtime.mheap_.allspans in work.spans to avoid conflicts with
1851 // resizing/freeing allspans.
1852 // New spans can be created while GC progresses, but they are not garbage for
1854 // - new stack spans can be created even while the world is stopped.
1855 // - new malloc spans can be created during the concurrent sweep
1857 // Even if this is stop-the-world, a concurrent exitsyscall can allocate a stack from heap.
1859 // Free the old cached sweep array if necessary.
1860 if work.spans != nil && &work.spans[0] != &h_allspans[0] {
1861 sysFree(unsafe.Pointer(&work.spans[0]), uintptr(len(work.spans))*unsafe.Sizeof(work.spans[0]), &memstats.other_sys)
1863 // Cache the current array for marking.
1864 mheap_.gcspans = mheap_.allspans
1865 work.spans = h_allspans
1866 unlock(&mheap_.lock)
1871 work.nproc = uint32(gcprocs())
1872 gcphase = _GCmarktermination
1874 // World is stopped so allglen will not change.
1875 for i := uintptr(0); i < allglen; i++ {
1877 gp.gcworkdone = false // set to true in gcphasework
1880 parforsetup(work.markfor, work.nproc, uint32(_RootCount+allglen), nil, false, markroot)
1882 noteclear(&work.alldone)
1883 helpgc(int32(work.nproc))
1887 if debug.gctrace > 0 {
1892 parfordo(work.markfor)
1893 scanblock(0, 0, nil)
1896 gothrow("work.full != 0")
1898 if work.partial != 0 {
1899 gothrow("work.partial != 0")
1904 if debug.gctrace > 0 {
1909 notesleep(&work.alldone)
1915 // next_gc calculation is tricky with concurrent sweep since we don't know size of live heap
1916 // estimate what was live heap size after previous GC (for printing only)
1917 heap0 := memstats.next_gc * 100 / (uint64(gcpercent) + 100)
1918 // conservatively set next_gc to high value assuming that everything is live
1919 // concurrent/lazy sweep will reduce this number while discovering new garbage
1920 memstats.next_gc = memstats.heap_alloc + memstats.heap_alloc*uint64(gcpercent)/100
1923 atomicstore64(&memstats.last_gc, uint64(unixnanotime())) // must be Unix time to make sense to user
1924 memstats.pause_ns[memstats.numgc%uint32(len(memstats.pause_ns))] = uint64(t4 - t0)
1925 memstats.pause_end[memstats.numgc%uint32(len(memstats.pause_end))] = uint64(t4)
1926 memstats.pause_total_ns += uint64(t4 - t0)
1928 if memstats.debuggc {
1929 print("pause ", t4-t0, "\n")
1932 if debug.gctrace > 0 {
1933 heap1 := memstats.heap_alloc
1935 updatememstats(&stats)
1936 if heap1 != memstats.heap_alloc {
1937 print("runtime: mstats skew: heap=", heap1, "/", memstats.heap_alloc, "\n")
1938 gothrow("mstats skew")
1940 obj := memstats.nmalloc - memstats.nfree
1942 stats.nprocyield += work.markfor.nprocyield
1943 stats.nosyield += work.markfor.nosyield
1944 stats.nsleep += work.markfor.nsleep
1946 print("gc", memstats.numgc, "(", work.nproc, "): ",
1947 (t1-t0)/1000, "+", (t2-t1)/1000, "+", (t3-t2)/1000, "+", (t4-t3)/1000, " us, ",
1948 heap0>>20, " -> ", heap1>>20, " MB, ",
1949 obj, " (", memstats.nmalloc, "-", memstats.nfree, ") objects, ",
1950 gcount(), " goroutines, ",
1951 len(work.spans), "/", sweep.nbgsweep, "/", sweep.npausesweep, " sweeps, ",
1952 stats.nhandoff, "(", stats.nhandoffcnt, ") handoff, ",
1953 work.markfor.nsteal, "(", work.markfor.nstealcnt, ") steal, ",
1954 stats.nprocyield, "/", stats.nosyield, "/", stats.nsleep, " yields\n")
1956 sweep.npausesweep = 0
1959 // See the comment in the beginning of this function as to why we need the following.
1960 // Even if this is still stop-the-world, a concurrent exitsyscall can allocate a stack from heap.
1962 // Free the old cached mark array if necessary.
1963 if work.spans != nil && &work.spans[0] != &h_allspans[0] {
1964 sysFree(unsafe.Pointer(&work.spans[0]), uintptr(len(work.spans))*unsafe.Sizeof(work.spans[0]), &memstats.other_sys)
1967 if gccheckmarkenable {
1969 // first half of two-pass; don't set up sweep
1970 unlock(&mheap_.lock)
1973 checkmark = false // done checking marks
1976 // Cache the current array for sweeping.
1977 mheap_.gcspans = mheap_.allspans
1978 mheap_.sweepgen += 2
1979 mheap_.sweepdone = 0
1980 work.spans = h_allspans
1982 unlock(&mheap_.lock)
1984 if _ConcurrentSweep && !eagersweep {
1988 sweep.started = true
1989 } else if sweep.parked {
1990 sweep.parked = false
1995 // Sweep all spans eagerly.
1996 for sweepone() != ^uintptr(0) {
1999 // Do an additional mProf_GC, because all 'free' events are now real as well.
2011 func readmemstats_m(stats *MemStats) {
2014 // Size of the trailing by_size array differs between Go and C,
2015 // NumSizeClasses was changed, but we can not change Go struct because of backward compatibility.
2016 memmove(unsafe.Pointer(stats), unsafe.Pointer(&memstats), sizeof_C_MStats)
2018 // Stack numbers are part of the heap numbers, separate those out for user consumption
2019 stats.StackSys = stats.StackInuse
2020 stats.HeapInuse -= stats.StackInuse
2021 stats.HeapSys -= stats.StackInuse
2024 //go:linkname readGCStats runtime/debug.readGCStats
2025 func readGCStats(pauses *[]uint64) {
2026 systemstack(func() {
2027 readGCStats_m(pauses)
2031 func readGCStats_m(pauses *[]uint64) {
2033 // Calling code in runtime/debug should make the slice large enough.
2034 if cap(p) < len(memstats.pause_ns)+3 {
2035 gothrow("runtime: short slice passed to readGCStats")
2038 // Pass back: pauses, pause ends, last gc (absolute time), number of gc, total pause ns.
2042 if n > uint32(len(memstats.pause_ns)) {
2043 n = uint32(len(memstats.pause_ns))
2046 // The pause buffer is circular. The most recent pause is at
2047 // pause_ns[(numgc-1)%len(pause_ns)], and then backward
2048 // from there to go back farther in time. We deliver the times
2049 // most recent first (in p[0]).
2051 for i := uint32(0); i < n; i++ {
2052 j := (memstats.numgc - 1 - i) % uint32(len(memstats.pause_ns))
2053 p[i] = memstats.pause_ns[j]
2054 p[n+i] = memstats.pause_end[j]
2057 p[n+n] = memstats.last_gc
2058 p[n+n+1] = uint64(memstats.numgc)
2059 p[n+n+2] = memstats.pause_total_ns
2060 unlock(&mheap_.lock)
2064 func setGCPercent(in int32) (out int32) {
2071 unlock(&mheap_.lock)
2075 func gchelperstart() {
2078 if _g_.m.helpgc < 0 || _g_.m.helpgc >= _MaxGcproc {
2079 gothrow("gchelperstart: bad m->helpgc")
2081 if _g_ != _g_.m.g0 {
2082 gothrow("gchelper not running on g0 stack")
2086 func wakefing() *g {
2089 if fingwait && fingwake {
2098 func addb(p *byte, n uintptr) *byte {
2099 return (*byte)(add(unsafe.Pointer(p), n))
2102 // Recursively unrolls GC program in prog.
2103 // mask is where to store the result.
2104 // ppos is a pointer to position in mask, in bits.
2105 // sparse says to generate 4-bits per word mask for heap (2-bits for data/bss otherwise).
2106 func unrollgcprog1(maskp *byte, prog *byte, ppos *uintptr, inplace, sparse bool) *byte {
2107 arena_start := mheap_.arena_start
2109 mask := (*[1 << 30]byte)(unsafe.Pointer(maskp))
2113 gothrow("unrollgcprog: unknown instruction")
2116 prog = addb(prog, 1)
2118 prog = addb(prog, 1)
2119 p := (*[1 << 30]byte)(unsafe.Pointer(prog))
2120 for i := 0; i < siz; i++ {
2121 v := p[i/_PointersPerByte]
2122 v >>= (uint(i) % _PointersPerByte) * _BitsPerPointer
2125 // Store directly into GC bitmap.
2126 off := (uintptr(unsafe.Pointer(&mask[pos])) - arena_start) / ptrSize
2127 bitp := (*byte)(unsafe.Pointer(arena_start - off/wordsPerBitmapByte - 1))
2128 shift := (off % wordsPerBitmapByte) * gcBits
2132 *bitp |= v << (shift + 2)
2143 pos += _BitsPerPointer
2146 prog = addb(prog, round(uintptr(siz)*_BitsPerPointer, 8)/8)
2149 prog = (*byte)(add(unsafe.Pointer(prog), 1))
2151 for i := uintptr(0); i < ptrSize; i++ {
2152 siz = (siz << 8) + uintptr(*(*byte)(add(unsafe.Pointer(prog), ptrSize-i-1)))
2154 prog = (*byte)(add(unsafe.Pointer(prog), ptrSize))
2156 for i := uintptr(0); i < siz; i++ {
2157 prog1 = unrollgcprog1(&mask[0], prog, &pos, inplace, sparse)
2159 if *prog1 != insArrayEnd {
2160 gothrow("unrollgcprog: array does not end with insArrayEnd")
2162 prog = (*byte)(add(unsafe.Pointer(prog1), 1))
2164 case insArrayEnd, insEnd:
2171 // Unrolls GC program prog for data/bss, returns dense GC mask.
2172 func unrollglobgcprog(prog *byte, size uintptr) bitvector {
2173 masksize := round(round(size, ptrSize)/ptrSize*bitsPerPointer, 8) / 8
2174 mask := (*[1 << 30]byte)(persistentalloc(masksize+1, 0, &memstats.gc_sys))
2175 mask[masksize] = 0xa1
2177 prog = unrollgcprog1(&mask[0], prog, &pos, false, false)
2178 if pos != size/ptrSize*bitsPerPointer {
2179 print("unrollglobgcprog: bad program size, got ", pos, ", expect ", size/ptrSize*bitsPerPointer, "\n")
2180 gothrow("unrollglobgcprog: bad program size")
2182 if *prog != insEnd {
2183 gothrow("unrollglobgcprog: program does not end with insEnd")
2185 if mask[masksize] != 0xa1 {
2186 gothrow("unrollglobgcprog: overflow")
2188 return bitvector{int32(masksize * 8), &mask[0]}
2191 func unrollgcproginplace_m(v unsafe.Pointer, typ *_type, size, size0 uintptr) {
2193 prog := (*byte)(unsafe.Pointer(uintptr(typ.gc[1])))
2195 unrollgcprog1((*byte)(v), prog, &pos, true, true)
2198 // Mark first word as bitAllocated.
2199 arena_start := mheap_.arena_start
2200 off := (uintptr(v) - arena_start) / ptrSize
2201 bitp := (*byte)(unsafe.Pointer(arena_start - off/wordsPerBitmapByte - 1))
2202 shift := (off % wordsPerBitmapByte) * gcBits
2203 *bitp |= bitBoundary << shift
2205 // Mark word after last as BitsDead.
2207 off := (uintptr(v) + size0 - arena_start) / ptrSize
2208 bitp := (*byte)(unsafe.Pointer(arena_start - off/wordsPerBitmapByte - 1))
2209 shift := (off % wordsPerBitmapByte) * gcBits
2210 *bitp &= uint8(^(bitPtrMask << shift) | uintptr(bitsDead)<<(shift+2))
2216 // Unrolls GC program in typ.gc[1] into typ.gc[0]
2217 func unrollgcprog_m(typ *_type) {
2219 mask := (*byte)(unsafe.Pointer(uintptr(typ.gc[0])))
2221 pos := uintptr(8) // skip the unroll flag
2222 prog := (*byte)(unsafe.Pointer(uintptr(typ.gc[1])))
2223 prog = unrollgcprog1(mask, prog, &pos, false, true)
2224 if *prog != insEnd {
2225 gothrow("unrollgcprog: program does not end with insEnd")
2227 if typ.size/ptrSize%2 != 0 {
2228 // repeat the program
2229 prog := (*byte)(unsafe.Pointer(uintptr(typ.gc[1])))
2230 unrollgcprog1(mask, prog, &pos, false, true)
2233 // atomic way to say mask[0] = 1
2239 // mark the span of memory at v as having n blocks of the given size.
2240 // if leftover is true, there is left over space at the end of the span.
2241 func markspan(v unsafe.Pointer, size uintptr, n uintptr, leftover bool) {
2242 if uintptr(v)+size*n > mheap_.arena_used || uintptr(v) < mheap_.arena_start {
2243 gothrow("markspan: bad pointer")
2246 // Find bits of the beginning of the span.
2247 off := (uintptr(v) - uintptr(mheap_.arena_start)) / ptrSize
2248 if off%wordsPerBitmapByte != 0 {
2249 gothrow("markspan: unaligned length")
2251 b := mheap_.arena_start - off/wordsPerBitmapByte - 1
2253 // Okay to use non-atomic ops here, because we control
2254 // the entire span, and each bitmap byte has bits for only
2255 // one span, so no other goroutines are changing these bitmap words.
2257 if size == ptrSize {
2258 // Possible only on 64-bits (minimal size class is 8 bytes).
2259 // Set memory to 0x11.
2260 if (bitBoundary|bitsDead)<<gcBits|bitBoundary|bitsDead != 0x11 {
2261 gothrow("markspan: bad bits")
2263 if n%(wordsPerBitmapByte*ptrSize) != 0 {
2264 gothrow("markspan: unaligned length")
2266 b = b - n/wordsPerBitmapByte + 1 // find first byte
2268 gothrow("markspan: unaligned pointer")
2270 for i := uintptr(0); i < n; i, b = i+wordsPerBitmapByte*ptrSize, b+ptrSize {
2271 *(*uintptr)(unsafe.Pointer(b)) = uintptrMask & 0x1111111111111111 // bitBoundary | bitsDead, repeated
2277 n++ // mark a boundary just past end of last block too
2279 step := size / (ptrSize * wordsPerBitmapByte)
2280 for i := uintptr(0); i < n; i, b = i+1, b-step {
2281 *(*byte)(unsafe.Pointer(b)) = bitBoundary | bitsDead<<2
2285 // unmark the span of memory at v of length n bytes.
2286 func unmarkspan(v, n uintptr) {
2287 if v+n > mheap_.arena_used || v < mheap_.arena_start {
2288 gothrow("markspan: bad pointer")
2291 off := (v - mheap_.arena_start) / ptrSize // word offset
2292 if off%(ptrSize*wordsPerBitmapByte) != 0 {
2293 gothrow("markspan: unaligned pointer")
2296 b := mheap_.arena_start - off/wordsPerBitmapByte - 1
2298 if n%(ptrSize*wordsPerBitmapByte) != 0 {
2299 gothrow("unmarkspan: unaligned length")
2302 // Okay to use non-atomic ops here, because we control
2303 // the entire span, and each bitmap word has bits for only
2304 // one span, so no other goroutines are changing these
2306 n /= wordsPerBitmapByte
2307 memclr(unsafe.Pointer(b-n+1), n)
2310 func mHeap_MapBits(h *mheap) {
2311 // Caller has added extra mappings to the arena.
2312 // Add extra mappings of bitmap words as needed.
2313 // We allocate extra bitmap pieces in chunks of bitmapChunk.
2314 const bitmapChunk = 8192
2316 n := (h.arena_used - h.arena_start) / (ptrSize * wordsPerBitmapByte)
2317 n = round(n, bitmapChunk)
2318 n = round(n, _PhysPageSize)
2319 if h.bitmap_mapped >= n {
2323 sysMap(unsafe.Pointer(h.arena_start-n), n-h.bitmap_mapped, h.arena_reserved, &memstats.gc_sys)
2327 func getgcmaskcb(frame *stkframe, ctxt unsafe.Pointer) bool {
2328 target := (*stkframe)(ctxt)
2329 if frame.sp <= target.sp && target.sp < frame.varp {
2336 // Returns GC type info for object p for testing.
2337 func getgcmask(p unsafe.Pointer, t *_type, mask **byte, len *uintptr) {
2342 if uintptr(unsafe.Pointer(&data)) <= uintptr(p) && uintptr(p) < uintptr(unsafe.Pointer(&edata)) {
2343 n := (*ptrtype)(unsafe.Pointer(t)).elem.size
2345 *mask = &make([]byte, *len)[0]
2346 for i := uintptr(0); i < n; i += ptrSize {
2347 off := (uintptr(p) + i - uintptr(unsafe.Pointer(&data))) / ptrSize
2348 bits := (*(*byte)(add(unsafe.Pointer(gcdatamask.bytedata), off/pointersPerByte)) >> ((off % pointersPerByte) * bitsPerPointer)) & bitsMask
2349 *(*byte)(add(unsafe.Pointer(*mask), i/ptrSize)) = bits
2355 if uintptr(unsafe.Pointer(&bss)) <= uintptr(p) && uintptr(p) < uintptr(unsafe.Pointer(&ebss)) {
2356 n := (*ptrtype)(unsafe.Pointer(t)).elem.size
2358 *mask = &make([]byte, *len)[0]
2359 for i := uintptr(0); i < n; i += ptrSize {
2360 off := (uintptr(p) + i - uintptr(unsafe.Pointer(&bss))) / ptrSize
2361 bits := (*(*byte)(add(unsafe.Pointer(gcbssmask.bytedata), off/pointersPerByte)) >> ((off % pointersPerByte) * bitsPerPointer)) & bitsMask
2362 *(*byte)(add(unsafe.Pointer(*mask), i/ptrSize)) = bits
2370 if mlookup(uintptr(p), &base, &n, nil) != 0 {
2372 *mask = &make([]byte, *len)[0]
2373 for i := uintptr(0); i < n; i += ptrSize {
2374 off := (uintptr(base) + i - mheap_.arena_start) / ptrSize
2375 b := mheap_.arena_start - off/wordsPerBitmapByte - 1
2376 shift := (off % wordsPerBitmapByte) * gcBits
2377 bits := (*(*byte)(unsafe.Pointer(b)) >> (shift + 2)) & bitsMask
2378 *(*byte)(add(unsafe.Pointer(*mask), i/ptrSize)) = bits
2385 frame.sp = uintptr(p)
2387 gentraceback(_g_.m.curg.sched.pc, _g_.m.curg.sched.sp, 0, _g_.m.curg, 0, nil, 1000, getgcmaskcb, noescape(unsafe.Pointer(&frame)), 0)
2388 if frame.fn != nil {
2390 targetpc := frame.continpc
2394 if targetpc != f.entry {
2397 pcdata := pcdatavalue(f, _PCDATA_StackMapIndex, targetpc)
2401 stkmap := (*stackmap)(funcdata(f, _FUNCDATA_LocalsPointerMaps))
2402 if stkmap == nil || stkmap.n <= 0 {
2405 bv := stackmapdata(stkmap, pcdata)
2406 size := uintptr(bv.n) / bitsPerPointer * ptrSize
2407 n := (*ptrtype)(unsafe.Pointer(t)).elem.size
2409 *mask = &make([]byte, *len)[0]
2410 for i := uintptr(0); i < n; i += ptrSize {
2411 off := (uintptr(p) + i - frame.varp + size) / ptrSize
2412 bits := ((*(*byte)(add(unsafe.Pointer(bv.bytedata), off*bitsPerPointer/8))) >> ((off * bitsPerPointer) % 8)) & bitsMask
2413 *(*byte)(add(unsafe.Pointer(*mask), i/ptrSize)) = bits
2418 func unixnanotime() int64 {
2420 gc_unixnanotime(&now)