runtime: retype mheap.pagesSwept as atomic.Uint64

[git-generate]
cd src/runtime
mv export_test.go export.go
GOROOT=$(dirname $(dirname $PWD)) rf '
  add mheap.pagesSwept pagesSwept_ atomic.Uint64 // pages swept this cycle
  ex {
    import "runtime/internal/atomic"

    var t mheap
    var v, w uint64
    var d int64

    t.pagesSwept -> t.pagesSwept_.Load()
    t.pagesSwept = v -> t.pagesSwept_.Store(v)
    atomic.Load64(&t.pagesSwept) -> t.pagesSwept_.Load()
    atomic.LoadAcq64(&t.pagesSwept) -> t.pagesSwept_.LoadAcquire()
    atomic.Store64(&t.pagesSwept, v) -> t.pagesSwept_.Store(v)
    atomic.StoreRel64(&t.pagesSwept, v) -> t.pagesSwept_.StoreRelease(v)
    atomic.Cas64(&t.pagesSwept, v, w) -> t.pagesSwept_.CompareAndSwap(v, w)
    atomic.Xchg64(&t.pagesSwept, v) -> t.pagesSwept_.Swap(v)
    atomic.Xadd64(&t.pagesSwept, d) -> t.pagesSwept_.Add(d)
  }
  rm mheap.pagesSwept
  mv mheap.pagesSwept_ mheap.pagesSwept
'
mv export.go export_test.go

Change-Id: Ife99893d90a339655f604bc3a64ee3decec645ea
Reviewed-on: https://go-review.googlesource.com/c/go/+/356709
Trust: Michael Knyszek <mknyszek@google.com>
Reviewed-by: Austin Clements <austin@google.com>

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index 34b5b482a355b634d2f2b9b85d120ca4981afd9e..654fa4118a068896e89b48ff251f669ad1a19f2c 100644 (file)
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -1457,7 +1457,7 @@ func gcSweep(mode gcMode) {
        lock(&mheap_.lock)
        mheap_.sweepgen += 2
        mheap_.sweepDrained = 0
-       mheap_.pagesSwept = 0
+       mheap_.pagesSwept.Store(0)
        mheap_.sweepArenas = mheap_.allArenas
        mheap_.reclaimIndex = 0
        mheap_.reclaimCredit = 0

diff --git a/src/runtime/mgcpacer.go b/src/runtime/mgcpacer.go
index 3cd4ea2fdebe4d056594fda7ee74b49ada0c62dd..f858ab08d0896f79deadd7f7ec5927a3c9e7b25d 100644 (file)
--- a/src/runtime/mgcpacer.go
+++ b/src/runtime/mgcpacer.go
@@ -751,7 +751,7 @@ func (c *gcControllerState) commit(triggerRatio float64) {
                        // Avoid setting the sweep ratio extremely high
                        heapDistance = _PageSize
                }
-               pagesSwept := atomic.Load64(&mheap_.pagesSwept)
+               pagesSwept := mheap_.pagesSwept.Load()
                pagesInUse := mheap_.pagesInUse.Load()
                sweepDistancePages := int64(pagesInUse) - int64(pagesSwept)
                if sweepDistancePages <= 0 {

diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go
index 1812644623b2f2eb2ffb09217491c969a117deaf..78d1f339250e977f1c405ab14e59f41943d3a5fa 100644 (file)
--- a/src/runtime/mgcsweep.go
+++ b/src/runtime/mgcsweep.go
@@ -245,7 +245,7 @@ func (l *sweepLocker) dispose() {
 
 func (l *sweepLocker) sweepIsDone() {
        if debug.gcpacertrace > 0 {
-               print("pacer: sweep done at heap size ", gcController.heapLive>>20, "MB; allocated ", (gcController.heapLive-mheap_.sweepHeapLiveBasis)>>20, "MB during sweep; swept ", mheap_.pagesSwept, " pages at ", mheap_.sweepPagesPerByte, " pages/byte\n")
+               print("pacer: sweep done at heap size ", gcController.heapLive>>20, "MB; allocated ", (gcController.heapLive-mheap_.sweepHeapLiveBasis)>>20, "MB during sweep; swept ", mheap_.pagesSwept.Load(), " pages at ", mheap_.sweepPagesPerByte, " pages/byte\n")
        }
 }
 
@@ -408,7 +408,7 @@ func (sl *sweepLocked) sweep(preserve bool) bool {
                traceGCSweepSpan(s.npages * _PageSize)
        }
 
-       atomic.Xadd64(&mheap_.pagesSwept, int64(s.npages))
+       mheap_.pagesSwept.Add(int64(s.npages))
 
        spc := s.spanclass
        size := s.elemsize
@@ -724,7 +724,7 @@ retry:
        // Fix debt if necessary.
        newHeapLive := uintptr(atomic.Load64(&gcController.heapLive)-mheap_.sweepHeapLiveBasis) + spanBytes
        pagesTarget := int64(mheap_.sweepPagesPerByte*float64(newHeapLive)) - int64(callerSweepPages)
-       for pagesTarget > int64(atomic.Load64(&mheap_.pagesSwept)-sweptBasis) {
+       for pagesTarget > int64(mheap_.pagesSwept.Load()-sweptBasis) {
                if sweepone() == ^uintptr(0) {
                        mheap_.sweepPagesPerByte = 0
                        break

diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go
index a787f67460924fd8c257c82b4308476d10faf0b0..27f60771eb658a6b460eaafc23cfe0587d89fe4e 100644 (file)
--- a/src/runtime/mheap.go
+++ b/src/runtime/mheap.go
@@ -103,7 +103,7 @@ type mheap struct {
        // the slope, it would create a discontinuity in debt if any
        // progress has already been made.
        pagesInUse         atomic.Uint64 // pages of spans in stats mSpanInUse
-       pagesSwept         uint64        // pages swept this cycle; updated atomically
+       pagesSwept         atomic.Uint64 // pages swept this cycle
        pagesSweptBasis    uint64        // pagesSwept to use as the origin of the sweep ratio; updated atomically
        sweepHeapLiveBasis uint64        // value of gcController.heapLive to use as the origin of sweep ratio; written with lock, read without
        sweepPagesPerByte  float64       // proportional sweep ratio; written with lock, read without