[dev.garbage] Merge remote-tracking branch 'origin/master' into HEAD

[gostls13.git] / src / runtime / mgcmark.go

diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go
index b5a9ff9b5681188864b1871e92eefc9db7dc45e0..3704164527b69d363dd2e22268dc1a7167787367 100644 (file)
--- a/src/runtime/mgcmark.go
+++ b/src/runtime/mgcmark.go
@@ -360,7 +360,7 @@ func markrootSpans(gcw *gcWork, shard int) {
                        // retain everything it points to.
                        spf := (*specialfinalizer)(unsafe.Pointer(sp))
                        // A finalizer can be set for an inner byte of an object, find object beginning.
-                       p := uintptr(s.start<<_PageShift) + uintptr(spf.special.offset)/s.elemsize*s.elemsize
+                       p := s.base() + uintptr(spf.special.offset)/s.elemsize*s.elemsize
 
                        // Mark everything that can be reached from
                        // the object (but *not* the object itself or
@@ -962,7 +962,10 @@ func gcDrain(gcw *gcWork, flags gcDrainFlags) {
                if blocking {
                        b = gcw.get()
                } else {
-                       b = gcw.tryGet()
+                       b = gcw.tryGetFast()
+                       if b == 0 {
+                               b = gcw.tryGet()
+                       }
                }
                if b == 0 {
                        // work barrier reached or tryGet failed.
@@ -1025,7 +1028,11 @@ func gcDrainN(gcw *gcWork, scanWork int64) int64 {
                //         PREFETCH(wbuf->obj[wbuf.nobj - 3];
                //  }
                //
-               b := gcw.tryGet()
+               b := gcw.tryGetFast()
+               if b == 0 {
+                       b = gcw.tryGet()
+               }
+
                if b == 0 {
                        break
                }
@@ -1175,9 +1182,9 @@ func greyobject(obj, base, off uintptr, hbits heapBits, span *mspan, gcw *gcWork
        if obj&(sys.PtrSize-1) != 0 {
                throw("greyobject: obj not pointer-aligned")
        }
-
+       mbits := span.markBitsForAddr(obj)
        if useCheckmark {
-               if !hbits.isMarked() {
+               if !mbits.isMarked() {
                        printlock()
                        print("runtime:greyobject: checkmarks finds unexpected unmarked object obj=", hex(obj), "\n")
                        print("runtime: found obj at *(", hex(base), "+", hex(off), ")\n")
@@ -1199,10 +1206,10 @@ func greyobject(obj, base, off uintptr, hbits heapBits, span *mspan, gcw *gcWork
                }
        } else {
                // If marked we have nothing to do.
-               if hbits.isMarked() {
+               if mbits.isMarked() {
                        return
                }
-               hbits.setMarked()
+               mbits.setMarked()
 
                // If this is a noscan object, fast-track it to black
                // instead of greying it.
@@ -1218,8 +1225,9 @@ func greyobject(obj, base, off uintptr, hbits heapBits, span *mspan, gcw *gcWork
        // Previously we put the obj in an 8 element buffer that is drained at a rate
        // to give the PREFETCH time to do its work.
        // Use of PREFETCHNTA might be more appropriate than PREFETCH
-
-       gcw.put(obj)
+       if !gcw.putFast(obj) {
+               gcw.put(obj)
+       }
 }
 
 // gcDumpObject dumps the contents of obj for debugging and marks the
@@ -1274,7 +1282,7 @@ func gcmarknewobject(obj, size, scanSize uintptr) {
        if useCheckmark && !gcBlackenPromptly { // The world should be stopped so this should not happen.
                throw("gcmarknewobject called while doing checkmark")
        }
-       heapBitsForAddr(obj).setMarked()
+       markBitsForAddr(obj).setMarked()
        gcw := &getg().m.p.ptr().gcw
        gcw.bytesMarked += uint64(size)
        gcw.scanWork += int64(scanSize)