runtime: refactor runtime->tracer API to appear more like a lock

[gostls13.git] / src / runtime / mgcsweep.go

diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go
index 986eb573ca42ab7205f45e222c0e6e8453f13c32..2d84c0d07ceba82f5c753c0ed784555eb5ec1f08 100644 (file)
--- a/src/runtime/mgcsweep.go
+++ b/src/runtime/mgcsweep.go
@@ -25,6 +25,7 @@
 package runtime
 
 import (
+       "internal/goexperiment"
        "runtime/internal/atomic"
        "unsafe"
 )
@@ -37,9 +38,6 @@ type sweepdata struct {
        g      *g
        parked bool
 
-       nbgsweep    uint32
-       npausesweep uint32
-
        // active tracks outstanding sweepers and the sweep
        // termination condition.
        active activeSweep
@@ -237,7 +235,6 @@ func finishsweep_m() {
        // instantly. If GC was forced before the concurrent sweep
        // finished, there may be spans to sweep.
        for sweepone() != ^uintptr(0) {
-               sweep.npausesweep++
        }
 
        // Make sure there aren't any outstanding sweepers left.
@@ -299,7 +296,6 @@ func bgsweep(c chan int) {
                const sweepBatchSize = 10
                nSwept := 0
                for sweepone() != ^uintptr(0) {
-                       sweep.nbgsweep++
                        nSwept++
                        if nSwept%sweepBatchSize == 0 {
                                goschedIfBusy()
@@ -520,8 +516,10 @@ func (sl *sweepLocked) sweep(preserve bool) bool {
                throw("mspan.sweep: bad span state")
        }
 
-       if traceEnabled() {
-               traceGCSweepSpan(s.npages * _PageSize)
+       trace := traceAcquire()
+       if trace.ok() {
+               trace.GCSweepSpan(s.npages * _PageSize)
+               traceRelease(trace)
        }
 
        mheap_.pagesSwept.Add(int64(s.npages))
@@ -791,6 +789,15 @@ func (sl *sweepLocked) sweep(preserve bool) bool {
                        } else {
                                mheap_.freeSpan(s)
                        }
+                       if goexperiment.AllocHeaders && s.largeType != nil && s.largeType.Kind_&kindGCProg != 0 {
+                               // In the allocheaders experiment, the unrolled GCProg bitmap is allocated separately.
+                               // Free the space for the unrolled bitmap.
+                               systemstack(func() {
+                                       s := spanOf(uintptr(unsafe.Pointer(s.largeType)))
+                                       mheap_.freeManual(s, spanAllocPtrScalarBits)
+                               })
+                               s.largeType = nil
+                       }
 
                        // Count the free in the consistent, external stats.
                        stats := memstats.heapStats.acquire()
@@ -884,8 +891,10 @@ func deductSweepCredit(spanBytes uintptr, callerSweepPages uintptr) {
                return
        }
 
-       if traceEnabled() {
-               traceGCSweepStart()
+       trace := traceAcquire()
+       if trace.ok() {
+               trace.GCSweepStart()
+               traceRelease(trace)
        }
 
        // Fix debt if necessary.
@@ -924,8 +933,10 @@ retry:
                }
        }
 
-       if traceEnabled() {
-               traceGCSweepDone()
+       trace = traceAcquire()
+       if trace.ok() {
+               trace.GCSweepDone()
+               traceRelease(trace)
        }
 }