import (
"internal/abi"
"internal/goarch"
- "internal/goexperiment"
+ "runtime/internal/math"
"unsafe"
)
// Should be a built-in for unsafe.Pointer?
+//
//go:nosplit
func add(p unsafe.Pointer, x uintptr) unsafe.Pointer {
return unsafe.Pointer(uintptr(p) + x)
//go:noescape
func systemstack(fn func())
-var badsystemstackMsg = "fatal: systemstack called from unexpected goroutine"
-
//go:nosplit
//go:nowritebarrierrec
func badsystemstack() {
- sp := stringStructOf(&badsystemstackMsg)
- write(2, sp.str, int32(sp.len))
+ writeErrStr("fatal: systemstack called from unexpected goroutine")
}
// memclrNoHeapPointers clears n bytes starting at ptr.
func memmove(to, from unsafe.Pointer, n uintptr)
// Outside assembly calls memmove. Make sure it has ABI wrappers.
+//
//go:linkname memmove
//go:linkname reflect_memmove reflect.memmove
//go:nosplit
func fastrand() uint32 {
mp := getg().m
+ // Implement wyrand: https://github.com/wangyi-fudan/wyhash
+ // Only the platform that math.Mul64 can be lowered
+ // by the compiler should be in this list.
+ if goarch.IsAmd64|goarch.IsArm64|goarch.IsPpc64|
+ goarch.IsPpc64le|goarch.IsMips64|goarch.IsMips64le|
+ goarch.IsS390x|goarch.IsRiscv64|goarch.IsLoong64 == 1 {
+ mp.fastrand += 0xa0761d6478bd642f
+ hi, lo := math.Mul64(mp.fastrand, mp.fastrand^0xe7037ed1a0b428db)
+ return uint32(hi ^ lo)
+ }
+
// Implement xorshift64+: 2 32-bit xorshift sequences added together.
// Shift triplet [17,7,16] was calculated as indicated in Marsaglia's
// Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf
// This generator passes the SmallCrush suite, part of TestU01 framework:
// http://simul.iro.umontreal.ca/testu01/tu01.html
- s1, s0 := mp.fastrand[0], mp.fastrand[1]
+ t := (*[2]uint32)(unsafe.Pointer(&mp.fastrand))
+ s1, s0 := t[0], t[1]
s1 ^= s1 << 17
s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16
- mp.fastrand[0], mp.fastrand[1] = s0, s1
+ t[0], t[1] = s0, s1
return s0 + s1
}
return uint32(uint64(fastrand()) * uint64(n) >> 32)
}
-//go:linkname sync_fastrand sync.fastrand
-func sync_fastrand() uint32 { return fastrand() }
+func fastrand64() uint64 {
+ mp := getg().m
+ // Implement wyrand: https://github.com/wangyi-fudan/wyhash
+ // Only the platform that math.Mul64 can be lowered
+ // by the compiler should be in this list.
+ if goarch.IsAmd64|goarch.IsArm64|goarch.IsPpc64|
+ goarch.IsPpc64le|goarch.IsMips64|goarch.IsMips64le|
+ goarch.IsS390x|goarch.IsRiscv64 == 1 {
+ mp.fastrand += 0xa0761d6478bd642f
+ hi, lo := math.Mul64(mp.fastrand, mp.fastrand^0xe7037ed1a0b428db)
+ return hi ^ lo
+ }
+
+ // Implement xorshift64+: 2 32-bit xorshift sequences added together.
+ // Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf
+ // This generator passes the SmallCrush suite, part of TestU01 framework:
+ // http://simul.iro.umontreal.ca/testu01/tu01.html
+ t := (*[2]uint32)(unsafe.Pointer(&mp.fastrand))
+ s1, s0 := t[0], t[1]
+ s1 ^= s1 << 17
+ s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16
+ r := uint64(s0 + s1)
+
+ s0, s1 = s1, s0
+ s1 ^= s1 << 17
+ s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16
+ r += uint64(s0+s1) << 32
+
+ t[0], t[1] = s0, s1
+ return r
+}
-//go:linkname net_fastrand net.fastrand
-func net_fastrand() uint32 { return fastrand() }
+func fastrandu() uint {
+ if goarch.PtrSize == 4 {
+ return uint(fastrand())
+ }
+ return uint(fastrand64())
+}
+
+//go:linkname rand_fastrand64 math/rand.fastrand64
+func rand_fastrand64() uint64 { return fastrand64() }
+
+//go:linkname rand2_fastrand64 math/rand/v2.fastrand64
+func rand2_fastrand64() uint64 { return fastrand64() }
+
+//go:linkname sync_fastrandn sync.fastrandn
+func sync_fastrandn(n uint32) uint32 { return fastrandn(n) }
+
+//go:linkname net_fastrandu net.fastrandu
+func net_fastrandu() uint { return fastrandu() }
//go:linkname os_fastrand os.fastrand
func os_fastrand() uint32 { return fastrand() }
// in internal/bytealg/equal_*.s
+//
//go:noescape
func memequal(a, b unsafe.Pointer, size uintptr) bool
// output depends on the input. noescape is inlined and currently
// compiles down to zero instructions.
// USE CAREFULLY!
+//
//go:nosplit
func noescape(p unsafe.Pointer) unsafe.Pointer {
x := uintptr(p)
return unsafe.Pointer(x ^ 0)
}
+// noEscapePtr hides a pointer from escape analysis. See noescape.
+// USE CAREFULLY!
+//
+//go:nosplit
+func noEscapePtr[T any](p *T) *T {
+ x := uintptr(unsafe.Pointer(p))
+ return (*T)(unsafe.Pointer(x ^ 0))
+}
+
// Not all cgocallback frames are actually cgocallback,
// so not all have these arguments. Mark them uintptr so that the GC
// does not misinterpret memory when the arguments are not present.
// cgocallback is not called from Go, only from crosscall2.
// This in turn calls cgocallbackg, which is where we'll find
// pointer-declared arguments.
+//
+// When fn is nil (frame is saved g), call dropm instead,
+// this is used when the C thread is exiting.
func cgocallback(fn, frame, ctxt uintptr)
func gogo(buf *gobuf)
// Arguments passed through to reflectcall do not escape. The type is used
// only in a very limited callee of reflectcall, the stackArgs are copied, and
// regArgs is only used in the reflectcall frame.
+//
//go:noescape
func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
func systemstack_switch()
// alignUp rounds n up to a multiple of a. a must be a power of 2.
+//
+//go:nosplit
func alignUp(n, a uintptr) uintptr {
return (n + a - 1) &^ (a - 1)
}
// alignDown rounds n down to a multiple of a. a must be a power of 2.
+//
+//go:nosplit
func alignDown(n, a uintptr) uintptr {
return n &^ (a - 1)
}
func bool2int(x bool) int {
// Avoid branches. In the SSA compiler, this compiles to
// exactly what you would want it to.
- return int(uint8(*(*uint8)(unsafe.Pointer(&x))))
+ return int(*(*uint8)(unsafe.Pointer(&x)))
}
// abort crashes the runtime in situations where even throw might not
func abort()
// Called from compiled code; declared for vet; do NOT call from Go.
-func gcWriteBarrier()
+func gcWriteBarrier1()
+func gcWriteBarrier2()
+func gcWriteBarrier3()
+func gcWriteBarrier4()
+func gcWriteBarrier5()
+func gcWriteBarrier6()
+func gcWriteBarrier7()
+func gcWriteBarrier8()
func duffzero()
func duffcopy()
// registers the system supports.
//
// Protected by finlock.
-var intArgRegs = abi.IntArgRegs * (goexperiment.RegabiArgsInt | goarch.IsAmd64)
+var intArgRegs = abi.IntArgRegs