runtime: refactor runtime->tracer API to appear more like a lock

[gostls13.git] / src / runtime / mgcmark.go

diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go
index adf1d4fa2817b9f6106ac2f969ad9af5906b8aca..17412bf7235a7df843a8d7a9c1f8af307814d473 100644 (file)
--- a/src/runtime/mgcmark.go
+++ b/src/runtime/mgcmark.go
@@ -9,6 +9,7 @@ package runtime
 import (
        "internal/abi"
        "internal/goarch"
+       "internal/goexperiment"
        "runtime/internal/atomic"
        "runtime/internal/sys"
        "unsafe"
@@ -419,7 +420,11 @@ retry:
                // If the CPU limiter is enabled, intentionally don't
                // assist to reduce the amount of CPU time spent in the GC.
                if traced {
-                       traceGCMarkAssistDone()
+                       trace := traceAcquire()
+                       if trace.ok() {
+                               trace.GCMarkAssistDone()
+                               traceRelease(trace)
+                       }
                }
                return
        }
@@ -460,15 +465,22 @@ retry:
                        // We were able to steal all of the credit we
                        // needed.
                        if traced {
-                               traceGCMarkAssistDone()
+                               trace := traceAcquire()
+                               if trace.ok() {
+                                       trace.GCMarkAssistDone()
+                                       traceRelease(trace)
+                               }
                        }
                        return
                }
        }
-
        if traceEnabled() && !traced {
-               traced = true
-               traceGCMarkAssistStart()
+               trace := traceAcquire()
+               if trace.ok() {
+                       traced = true
+                       trace.GCMarkAssistStart()
+                       traceRelease(trace)
+               }
        }
 
        // Perform assist work
@@ -514,7 +526,11 @@ retry:
                // this G's assist debt, or the GC cycle is over.
        }
        if traced {
-               traceGCMarkAssistDone()
+               trace := traceAcquire()
+               if trace.ok() {
+                       trace.GCMarkAssistDone()
+                       traceRelease(trace)
+               }
        }
 }
 
@@ -1306,6 +1322,7 @@ func scanobject(b uintptr, gcw *gcWork) {
                throw("scanobject of a noscan object")
        }
 
+       var tp typePointers
        if n > maxObletBytes {
                // Large object. Break into oblets for better
                // parallelism and lower latency.
@@ -1327,15 +1344,34 @@ func scanobject(b uintptr, gcw *gcWork) {
                // of the object.
                n = s.base() + s.elemsize - b
                n = min(n, maxObletBytes)
+               if goexperiment.AllocHeaders {
+                       tp = s.typePointersOfUnchecked(s.base())
+                       tp = tp.fastForward(b-tp.addr, b+n)
+               }
+       } else {
+               if goexperiment.AllocHeaders {
+                       tp = s.typePointersOfUnchecked(b)
+               }
        }
 
-       hbits := heapBitsForAddr(b, n)
+       var hbits heapBits
+       if !goexperiment.AllocHeaders {
+               hbits = heapBitsForAddr(b, n)
+       }
        var scanSize uintptr
        for {
                var addr uintptr
-               if hbits, addr = hbits.nextFast(); addr == 0 {
-                       if hbits, addr = hbits.next(); addr == 0 {
-                               break
+               if goexperiment.AllocHeaders {
+                       if tp, addr = tp.nextFast(); addr == 0 {
+                               if tp, addr = tp.next(b + n); addr == 0 {
+                                       break
+                               }
+                       }
+               } else {
+                       if hbits, addr = hbits.nextFast(); addr == 0 {
+                               if hbits, addr = hbits.next(); addr == 0 {
+                                       break
+                               }
                        }
                }