runtime: refactor runtime->tracer API to appear more like a lock

[gostls13.git] / src / runtime / mgcsweep.go

diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go
index 728a5bad7ed9d3b7dfa61a4147feac56ec054d52..2d84c0d07ceba82f5c753c0ed784555eb5ec1f08 100644 (file)
--- a/src/runtime/mgcsweep.go
+++ b/src/runtime/mgcsweep.go
@@ -25,6 +25,7 @@
 package runtime
 
 import (
+       "internal/goexperiment"
        "runtime/internal/atomic"
        "unsafe"
 )
@@ -37,9 +38,6 @@ type sweepdata struct {
        g      *g
        parked bool
 
-       nbgsweep    uint32
-       npausesweep uint32
-
        // active tracks outstanding sweepers and the sweep
        // termination condition.
        active activeSweep
@@ -237,7 +235,6 @@ func finishsweep_m() {
        // instantly. If GC was forced before the concurrent sweep
        // finished, there may be spans to sweep.
        for sweepone() != ^uintptr(0) {
-               sweep.npausesweep++
        }
 
        // Make sure there aren't any outstanding sweepers left.
@@ -299,7 +296,6 @@ func bgsweep(c chan int) {
                const sweepBatchSize = 10
                nSwept := 0
                for sweepone() != ^uintptr(0) {
-                       sweep.nbgsweep++
                        nSwept++
                        if nSwept%sweepBatchSize == 0 {
                                goschedIfBusy()
@@ -520,8 +516,10 @@ func (sl *sweepLocked) sweep(preserve bool) bool {
                throw("mspan.sweep: bad span state")
        }
 
-       if traceEnabled() {
-               traceGCSweepSpan(s.npages * _PageSize)
+       trace := traceAcquire()
+       if trace.ok() {
+               trace.GCSweepSpan(s.npages * _PageSize)
+               traceRelease(trace)
        }
 
        mheap_.pagesSwept.Add(int64(s.npages))
@@ -602,8 +600,8 @@ func (sl *sweepLocked) sweep(preserve bool) bool {
                // efficient; allocfreetrace has massive overhead.
                mbits := s.markBitsForBase()
                abits := s.allocBitsForIndex(0)
-               for i := uintptr(0); i < s.nelems; i++ {
-                       if !mbits.isMarked() && (abits.index < s.freeindex || abits.isMarked()) {
+               for i := uintptr(0); i < uintptr(s.nelems); i++ {
+                       if !mbits.isMarked() && (abits.index < uintptr(s.freeindex) || abits.isMarked()) {
                                x := s.base() + i*s.elemsize
                                if debug.allocfreetrace != 0 {
                                        tracefree(unsafe.Pointer(x), size)
@@ -634,12 +632,12 @@ func (sl *sweepLocked) sweep(preserve bool) bool {
                //
                // Check the first bitmap byte, where we have to be
                // careful with freeindex.
-               obj := s.freeindex
+               obj := uintptr(s.freeindex)
                if (*s.gcmarkBits.bytep(obj / 8)&^*s.allocBits.bytep(obj / 8))>>(obj%8) != 0 {
                        s.reportZombies()
                }
                // Check remaining bytes.
-               for i := obj/8 + 1; i < divRoundUp(s.nelems, 8); i++ {
+               for i := obj/8 + 1; i < divRoundUp(uintptr(s.nelems), 8); i++ {
                        if *s.gcmarkBits.bytep(i)&^*s.allocBits.bytep(i) != 0 {
                                s.reportZombies()
                        }
@@ -666,7 +664,12 @@ func (sl *sweepLocked) sweep(preserve bool) bool {
        // gcmarkBits becomes the allocBits.
        // get a fresh cleared gcmarkBits in preparation for next GC
        s.allocBits = s.gcmarkBits
-       s.gcmarkBits = newMarkBits(s.nelems)
+       s.gcmarkBits = newMarkBits(uintptr(s.nelems))
+
+       // refresh pinnerBits if they exists
+       if s.pinnerBits != nil {
+               s.refreshPinnerBits()
+       }
 
        // Initialize alloc bits cache.
        s.refillAllocCache(0)
@@ -755,7 +758,7 @@ func (sl *sweepLocked) sweep(preserve bool) bool {
                                return true
                        }
                        // Return span back to the right mcentral list.
-                       if uintptr(nalloc) == s.nelems {
+                       if nalloc == s.nelems {
                                mheap_.central[spc].mcentral.fullSwept(sweepgen).push(s)
                        } else {
                                mheap_.central[spc].mcentral.partialSwept(sweepgen).push(s)
@@ -786,6 +789,15 @@ func (sl *sweepLocked) sweep(preserve bool) bool {
                        } else {
                                mheap_.freeSpan(s)
                        }
+                       if goexperiment.AllocHeaders && s.largeType != nil && s.largeType.Kind_&kindGCProg != 0 {
+                               // In the allocheaders experiment, the unrolled GCProg bitmap is allocated separately.
+                               // Free the space for the unrolled bitmap.
+                               systemstack(func() {
+                                       s := spanOf(uintptr(unsafe.Pointer(s.largeType)))
+                                       mheap_.freeManual(s, spanAllocPtrScalarBits)
+                               })
+                               s.largeType = nil
+                       }
 
                        // Count the free in the consistent, external stats.
                        stats := memstats.heapStats.acquire()
@@ -824,10 +836,10 @@ func (s *mspan) reportZombies() {
        print("runtime: marked free object in span ", s, ", elemsize=", s.elemsize, " freeindex=", s.freeindex, " (bad use of unsafe.Pointer? try -d=checkptr)\n")
        mbits := s.markBitsForBase()
        abits := s.allocBitsForIndex(0)
-       for i := uintptr(0); i < s.nelems; i++ {
+       for i := uintptr(0); i < uintptr(s.nelems); i++ {
                addr := s.base() + i*s.elemsize
                print(hex(addr))
-               alloc := i < s.freeindex || abits.isMarked()
+               alloc := i < uintptr(s.freeindex) || abits.isMarked()
                if alloc {
                        print(" alloc")
                } else {
@@ -879,8 +891,10 @@ func deductSweepCredit(spanBytes uintptr, callerSweepPages uintptr) {
                return
        }
 
-       if traceEnabled() {
-               traceGCSweepStart()
+       trace := traceAcquire()
+       if trace.ok() {
+               trace.GCSweepStart()
+               traceRelease(trace)
        }
 
        // Fix debt if necessary.
@@ -919,8 +933,10 @@ retry:
                }
        }
 
-       if traceEnabled() {
-               traceGCSweepDone()
+       trace = traceAcquire()
+       if trace.ok() {
+               trace.GCSweepDone()
+               traceRelease(trace)
        }
 }