runtime: add eager scavenging details to GODEBUG=scavtrace=1

Also, clean up atomics on released-per-cycle while we're here.

For #57069.

Change-Id: I14026e8281f01dea1e8c8de6aa8944712b7b24d9
Reviewed-on: https://go-review.googlesource.com/c/go/+/495916
Reviewed-by: Michael Pratt <mpratt@google.com>
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>

diff --git a/src/runtime/extern.go b/src/runtime/extern.go
index 189b4d4bb90b4e731d8ede4757072da970885301..9ad9fb7f3d1da4cea9bed8de6520cd1aa061a5f3 100644 (file)
--- a/src/runtime/extern.go
+++ b/src/runtime/extern.go
@@ -158,11 +158,13 @@ It is a comma-separated list of name=val pairs setting these named variables:
        scavenger as well as the total amount of memory returned to the operating system
        and an estimate of physical memory utilization. The format of this line is subject
        to change, but currently it is:
-               scav # KiB work, # KiB total, #% util
+               scav # KiB work (bg), # KiB work (eager), # KiB total, #% util
        where the fields are as follows:
-               # KiB work   the amount of memory returned to the OS since the last line
-               # KiB total  the total amount of memory returned to the OS
-               #% util      the fraction of all unscavenged memory which is in-use
+               # KiB work (bg)    the amount of memory returned to the OS in the background since
+                                  the last line
+               # KiB work (eager) the amount of memory returned to the OS eagerly since the last line
+               # KiB now          the amount of address space currently returned to the OS
+               #% util            the fraction of all unscavenged heap memory which is in-use
        If the line ends with "(forced)", then scavenging was forced by a
        debug.FreeOSMemory() call.
 

diff --git a/src/runtime/mgcscavenge.go b/src/runtime/mgcscavenge.go
index 782a2e696e2abb13844f35aaeb28cff5f3f1d73f..3f95bb04655f2bd7093fbcf587ad464b6697ae8c 100644 (file)
--- a/src/runtime/mgcscavenge.go
+++ b/src/runtime/mgcscavenge.go
@@ -658,7 +658,7 @@ func bgscavenge(c chan int) {
                        scavenger.park()
                        continue
                }
-               atomic.Xadduintptr(&mheap_.pages.scav.released, released)
+               mheap_.pages.scav.releasedBg.Add(released)
                scavenger.sleep(workTime)
        }
 }
@@ -696,13 +696,14 @@ func (p *pageAlloc) scavenge(nbytes uintptr, shouldStop func() bool, force bool)
 // application.
 //
 // scavenger.lock must be held.
-func printScavTrace(released uintptr, forced bool) {
+func printScavTrace(releasedBg, releasedEager uintptr, forced bool) {
        assertLockHeld(&scavenger.lock)
 
        printlock()
        print("scav ",
-               released>>10, " KiB work, ",
-               gcController.heapReleased.load()>>10, " KiB total, ",
+               releasedBg>>10, " KiB work (bg), ",
+               releasedEager>>10, " KiB work (eager), ",
+               gcController.heapReleased.load()>>10, " KiB now, ",
                (gcController.heapInUse.load()*100)/heapRetained(), "% util",
        )
        if forced {

diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go
index 4b0d655a9d2008f2028f91c996a0e6e8c28da319..e0e5bf0aefa276ecf2da54745bb3a6adbe345497 100644 (file)
--- a/src/runtime/mgcsweep.go
+++ b/src/runtime/mgcsweep.go
@@ -425,9 +425,17 @@ func sweepone() uintptr {
                if debug.scavtrace > 0 {
                        systemstack(func() {
                                lock(&mheap_.lock)
-                               released := atomic.Loaduintptr(&mheap_.pages.scav.released)
-                               printScavTrace(released, false)
-                               atomic.Storeuintptr(&mheap_.pages.scav.released, 0)
+
+                               // Get released stats.
+                               releasedBg := mheap_.pages.scav.releasedBg.Load()
+                               releasedEager := mheap_.pages.scav.releasedEager.Load()
+
+                               // Print the line.
+                               printScavTrace(releasedBg, releasedEager, false)
+
+                               // Update the stats.
+                               mheap_.pages.scav.releasedBg.Add(-releasedBg)
+                               mheap_.pages.scav.releasedEager.Add(-releasedEager)
                                unlock(&mheap_.lock)
                        })
                }

diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go
index fd6a8a715a979301b8a14aeab4d359093ba6acd0..d69822b143fc8471e9a08ceecb765a4a4d92f378 100644 (file)
--- a/src/runtime/mheap.go
+++ b/src/runtime/mheap.go
@@ -1323,10 +1323,12 @@ HaveSpan:
                track := pp.limiterEvent.start(limiterEventScavengeAssist, start)
 
                // Scavenge, but back out if the limiter turns on.
-               h.pages.scavenge(bytesToScavenge, func() bool {
+               released := h.pages.scavenge(bytesToScavenge, func() bool {
                        return gcCPULimiter.limiting()
                }, forceScavenge)
 
+               mheap_.pages.scav.releasedEager.Add(released)
+
                // Finish up accounting.
                now = nanotime()
                if track {
@@ -1658,7 +1660,7 @@ func (h *mheap) scavengeAll() {
        gp.m.mallocing--
 
        if debug.scavtrace > 0 {
-               printScavTrace(released, true)
+               printScavTrace(0, released, true)
        }
 }
 

diff --git a/src/runtime/mpagealloc.go b/src/runtime/mpagealloc.go
index 12ae474a4dad70699d7fefe173d919aa422d9761..ed53a5672b8d6177b97b049439abec39219d54bb 100644 (file)
--- a/src/runtime/mpagealloc.go
+++ b/src/runtime/mpagealloc.go
@@ -48,6 +48,7 @@
 package runtime
 
 import (
+       "runtime/internal/atomic"
        "unsafe"
 )
 
@@ -270,10 +271,13 @@ type pageAlloc struct {
                // scavenge.
                index scavengeIndex
 
-               // released is the amount of memory released this scavenge cycle.
-               //
-               // Updated atomically.
-               released uintptr
+               // releasedBg is the amount of memory released in the background this
+               // scavenge cycle.
+               releasedBg atomic.Uintptr
+
+               // releasedEager is the amount of memory released eagerly this scavenge
+               // cycle.
+               releasedEager atomic.Uintptr
        }
 
        // mheap_.lock. This level of indirection makes it possible