runtime: implement experiment to replace heap bitmap with alloc headers

[gostls13.git] / src / runtime / stubs.go

diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go
index 929f8fadca81aa065ca81d2c9b406113512e23d0..9633d3d4a6f8888445906eb186bf87ce5bc369dc 100644 (file)
--- a/src/runtime/stubs.go
+++ b/src/runtime/stubs.go
@@ -59,13 +59,10 @@ func mcall(fn func(*g))
 //go:noescape
 func systemstack(fn func())
 
-var badsystemstackMsg = "fatal: systemstack called from unexpected goroutine"
-
 //go:nosplit
 //go:nowritebarrierrec
 func badsystemstack() {
-       sp := stringStructOf(&badsystemstackMsg)
-       write(2, sp.str, int32(sp.len))
+       writeErrStr("fatal: systemstack called from unexpected goroutine")
 }
 
 // memclrNoHeapPointers clears n bytes starting at ptr.
@@ -131,7 +128,7 @@ func fastrand() uint32 {
        // by the compiler should be in this list.
        if goarch.IsAmd64|goarch.IsArm64|goarch.IsPpc64|
                goarch.IsPpc64le|goarch.IsMips64|goarch.IsMips64le|
-               goarch.IsS390x|goarch.IsRiscv64 == 1 {
+               goarch.IsS390x|goarch.IsRiscv64|goarch.IsLoong64 == 1 {
                mp.fastrand += 0xa0761d6478bd642f
                hi, lo := math.Mul64(mp.fastrand, mp.fastrand^0xe7037ed1a0b428db)
                return uint32(hi ^ lo)
@@ -196,6 +193,12 @@ func fastrandu() uint {
        return uint(fastrand64())
 }
 
+//go:linkname rand_fastrand64 math/rand.fastrand64
+func rand_fastrand64() uint64 { return fastrand64() }
+
+//go:linkname rand2_fastrand64 math/rand/v2.fastrand64
+func rand2_fastrand64() uint64 { return fastrand64() }
+
 //go:linkname sync_fastrandn sync.fastrandn
 func sync_fastrandn(n uint32) uint32 { return fastrandn(n) }
 
@@ -222,12 +225,24 @@ func noescape(p unsafe.Pointer) unsafe.Pointer {
        return unsafe.Pointer(x ^ 0)
 }
 
+// noEscapePtr hides a pointer from escape analysis. See noescape.
+// USE CAREFULLY!
+//
+//go:nosplit
+func noEscapePtr[T any](p *T) *T {
+       x := uintptr(unsafe.Pointer(p))
+       return (*T)(unsafe.Pointer(x ^ 0))
+}
+
 // Not all cgocallback frames are actually cgocallback,
 // so not all have these arguments. Mark them uintptr so that the GC
 // does not misinterpret memory when the arguments are not present.
 // cgocallback is not called from Go, only from crosscall2.
 // This in turn calls cgocallbackg, which is where we'll find
 // pointer-declared arguments.
+//
+// When fn is nil (frame is saved g), call dropm instead,
+// this is used when the C thread is exiting.
 func cgocallback(fn, frame, ctxt uintptr)
 
 func gogo(buf *gobuf)
@@ -409,11 +424,15 @@ func call1073741824(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOf
 func systemstack_switch()
 
 // alignUp rounds n up to a multiple of a. a must be a power of 2.
+//
+//go:nosplit
 func alignUp(n, a uintptr) uintptr {
        return (n + a - 1) &^ (a - 1)
 }
 
 // alignDown rounds n down to a multiple of a. a must be a power of 2.
+//
+//go:nosplit
 func alignDown(n, a uintptr) uintptr {
        return n &^ (a - 1)
 }
@@ -434,7 +453,7 @@ func memequal_varlen(a, b unsafe.Pointer) bool
 func bool2int(x bool) int {
        // Avoid branches. In the SSA compiler, this compiles to
        // exactly what you would want it to.
-       return int(uint8(*(*uint8)(unsafe.Pointer(&x))))
+       return int(*(*uint8)(unsafe.Pointer(&x)))
 }
 
 // abort crashes the runtime in situations where even throw might not
@@ -445,7 +464,14 @@ func bool2int(x bool) int {
 func abort()
 
 // Called from compiled code; declared for vet; do NOT call from Go.
-func gcWriteBarrier()
+func gcWriteBarrier1()
+func gcWriteBarrier2()
+func gcWriteBarrier3()
+func gcWriteBarrier4()
+func gcWriteBarrier5()
+func gcWriteBarrier6()
+func gcWriteBarrier7()
+func gcWriteBarrier8()
 func duffzero()
 func duffcopy()