runtime: implement experiment to replace heap bitmap with alloc headers

[gostls13.git] / src / runtime / cgocheck.go

diff --git a/src/runtime/cgocheck.go b/src/runtime/cgocheck.go
index 2cfbdeebb5b50c870951c0bf20fd75c6fb23e2ff..3d6de4f855dc335ea699dc8dcf79ee157c78fcbd 100644 (file)
--- a/src/runtime/cgocheck.go
+++ b/src/runtime/cgocheck.go
@@ -9,13 +9,15 @@ package runtime
 
 import (
        "internal/goarch"
+       "internal/goexperiment"
        "unsafe"
 )
 
-const cgoWriteBarrierFail = "Go pointer stored into non-Go memory"
+const cgoWriteBarrierFail = "unpinned Go pointer stored into non-Go memory"
 
 // cgoCheckPtrWrite is called whenever a pointer is stored into memory.
-// It throws if the program is storing a Go pointer into non-Go memory.
+// It throws if the program is storing an unpinned Go pointer into non-Go
+// memory.
 //
 // This is called from generated code when GOEXPERIMENT=cgocheck2 is enabled.
 //
@@ -48,6 +50,12 @@ func cgoCheckPtrWrite(dst *unsafe.Pointer, src unsafe.Pointer) {
                return
        }
 
+       // If the object is pinned, it's safe to store it in C memory. The GC
+       // ensures it will not be moved or freed.
+       if isPinned(src) {
+               return
+       }
+
        // It's OK if writing to memory allocated by persistentalloc.
        // Do this check last because it is more expensive and rarely true.
        // If it is false the expense doesn't matter since we are crashing.
@@ -56,14 +64,14 @@ func cgoCheckPtrWrite(dst *unsafe.Pointer, src unsafe.Pointer) {
        }
 
        systemstack(func() {
-               println("write of Go pointer", hex(uintptr(src)), "to non-Go memory", hex(uintptr(unsafe.Pointer(dst))))
+               println("write of unpinned Go pointer", hex(uintptr(src)), "to non-Go memory", hex(uintptr(unsafe.Pointer(dst))))
                throw(cgoWriteBarrierFail)
        })
 }
 
 // cgoCheckMemmove is called when moving a block of memory.
-// It throws if the program is copying a block that contains a Go pointer
-// into non-Go memory.
+// It throws if the program is copying a block that contains an unpinned Go
+// pointer into non-Go memory.
 //
 // This is called from generated code when GOEXPERIMENT=cgocheck2 is enabled.
 //
@@ -76,8 +84,8 @@ func cgoCheckMemmove(typ *_type, dst, src unsafe.Pointer) {
 // cgoCheckMemmove2 is called when moving a block of memory.
 // dst and src point off bytes into the value to copy.
 // size is the number of bytes to copy.
-// It throws if the program is copying a block that contains a Go pointer
-// into non-Go memory.
+// It throws if the program is copying a block that contains an unpinned Go
+// pointer into non-Go memory.
 //
 //go:nosplit
 //go:nowritebarrier
@@ -97,8 +105,8 @@ func cgoCheckMemmove2(typ *_type, dst, src unsafe.Pointer, off, size uintptr) {
 // cgoCheckSliceCopy is called when copying n elements of a slice.
 // src and dst are pointers to the first element of the slice.
 // typ is the element type of the slice.
-// It throws if the program is copying slice elements that contain Go pointers
-// into non-Go memory.
+// It throws if the program is copying slice elements that contain unpinned Go
+// pointers into non-Go memory.
 //
 //go:nosplit
 //go:nowritebarrier
@@ -120,7 +128,7 @@ func cgoCheckSliceCopy(typ *_type, dst, src unsafe.Pointer, n int) {
 }
 
 // cgoCheckTypedBlock checks the block of memory at src, for up to size bytes,
-// and throws if it finds a Go pointer. The type of the memory is typ,
+// and throws if it finds an unpinned Go pointer. The type of the memory is typ,
 // and src is off bytes into that type.
 //
 //go:nosplit
@@ -169,22 +177,35 @@ func cgoCheckTypedBlock(typ *_type, src unsafe.Pointer, off, size uintptr) {
        }
 
        // src must be in the regular heap.
-
-       hbits := heapBitsForAddr(uintptr(src), size)
-       for {
-               var addr uintptr
-               if hbits, addr = hbits.next(); addr == 0 {
-                       break
+       if goexperiment.AllocHeaders {
+               tp := s.typePointersOf(uintptr(src), size)
+               for {
+                       var addr uintptr
+                       if tp, addr = tp.next(uintptr(src) + size); addr == 0 {
+                               break
+                       }
+                       v := *(*unsafe.Pointer)(unsafe.Pointer(addr))
+                       if cgoIsGoPointer(v) && !isPinned(v) {
+                               throw(cgoWriteBarrierFail)
+                       }
                }
-               v := *(*unsafe.Pointer)(unsafe.Pointer(addr))
-               if cgoIsGoPointer(v) {
-                       throw(cgoWriteBarrierFail)
+       } else {
+               hbits := heapBitsForAddr(uintptr(src), size)
+               for {
+                       var addr uintptr
+                       if hbits, addr = hbits.next(); addr == 0 {
+                               break
+                       }
+                       v := *(*unsafe.Pointer)(unsafe.Pointer(addr))
+                       if cgoIsGoPointer(v) && !isPinned(v) {
+                               throw(cgoWriteBarrierFail)
+                       }
                }
        }
 }
 
 // cgoCheckBits checks the block of memory at src, for up to size
-// bytes, and throws if it finds a Go pointer. The gcbits mark each
+// bytes, and throws if it finds an unpinned Go pointer. The gcbits mark each
 // pointer value. The src pointer is off bytes into the gcbits.
 //
 //go:nosplit
@@ -209,7 +230,7 @@ func cgoCheckBits(src unsafe.Pointer, gcbits *byte, off, size uintptr) {
                } else {
                        if bits&1 != 0 {
                                v := *(*unsafe.Pointer)(add(src, i))
-                               if cgoIsGoPointer(v) {
+                               if cgoIsGoPointer(v) && !isPinned(v) {
                                        throw(cgoWriteBarrierFail)
                                }
                        }
@@ -265,16 +286,16 @@ func cgoCheckUsingType(typ *_type, src unsafe.Pointer, off, size uintptr) {
                }
        case kindStruct:
                st := (*structtype)(unsafe.Pointer(typ))
-               for _, f := range st.fields {
-                       if off < f.typ.Size_ {
-                               cgoCheckUsingType(f.typ, src, off, size)
+               for _, f := range st.Fields {
+                       if off < f.Typ.Size_ {
+                               cgoCheckUsingType(f.Typ, src, off, size)
                        }
-                       src = add(src, f.typ.Size_)
+                       src = add(src, f.Typ.Size_)
                        skipped := off
-                       if skipped > f.typ.Size_ {
-                               skipped = f.typ.Size_
+                       if skipped > f.Typ.Size_ {
+                               skipped = f.Typ.Size_
                        }
-                       checked := f.typ.Size_ - skipped
+                       checked := f.Typ.Size_ - skipped
                        off -= skipped
                        if size <= checked {
                                return