sync/atomic: reduce inlining cost for atomic types methods

The "&x.v" cost us 2 inline cost budget, causing extra inline cost for
the caller. By taking the fact that "v" is laid out in memory as the
first field of all atomic types, we can accessing it without addressing.

Discovering why attempting to convert sync.RWMutex.readerCount to atomic
type. RWMutex.RUnlock have the inline cost 75, with extra 7 inline cost
from Int32.Add causing it not inlinable anymore.

With this change, Int32.Add only has 5 inline cost budget, RWMutex can
use it while still be inlinable.

Change-Id: Iabe1d1bf53389b0b8b5f56b4611231b732fd9df5
Reviewed-on: https://go-review.googlesource.com/c/go/+/429766
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Cuong Manh Le <cuong.manhle.vn@gmail.com>
Reviewed-by: Bryan Mills <bcmills@google.com>
Reviewed-by: Cherry Mui <cherryyz@google.com>
Auto-Submit: Cuong Manh Le <cuong.manhle.vn@gmail.com>

diff --git a/src/sync/atomic/type.go b/src/sync/atomic/type.go
index 87c98b1e77e90780edcd38f8edc8856c43e3feff..be11e6109ef3bc30f227f7e32e762999315403d1 100644 (file)
--- a/src/sync/atomic/type.go
+++ b/src/sync/atomic/type.go
@@ -14,17 +14,19 @@ type Bool struct {
 }
 
 // Load atomically loads and returns the value stored in x.
-func (x *Bool) Load() bool { return LoadUint32(&x.v) != 0 }
+func (x *Bool) Load() bool { return LoadUint32((*uint32)(unsafe.Pointer(x))) != 0 }
 
 // Store atomically stores val into x.
-func (x *Bool) Store(val bool) { StoreUint32(&x.v, b32(val)) }
+func (x *Bool) Store(val bool) { StoreUint32((*uint32)(unsafe.Pointer(x)), b32(val)) }
 
 // Swap atomically stores new into x and returns the previous value.
-func (x *Bool) Swap(new bool) (old bool) { return SwapUint32(&x.v, b32(new)) != 0 }
+func (x *Bool) Swap(new bool) (old bool) {
+       return SwapUint32((*uint32)(unsafe.Pointer(x)), b32(new)) != 0
+}
 
 // CompareAndSwap executes the compare-and-swap operation for the boolean value x.
 func (x *Bool) CompareAndSwap(old, new bool) (swapped bool) {
-       return CompareAndSwapUint32(&x.v, b32(old), b32(new))
+       return CompareAndSwapUint32((*uint32)(unsafe.Pointer(x)), b32(old), b32(new))
 }
 
 // b32 returns a uint32 0 or 1 representing b.
@@ -46,17 +48,21 @@ type Pointer[T any] struct {
 }
 
 // Load atomically loads and returns the value stored in x.
-func (x *Pointer[T]) Load() *T { return (*T)(LoadPointer(&x.v)) }
+func (x *Pointer[T]) Load() *T { return (*T)(LoadPointer((*unsafe.Pointer)(unsafe.Pointer(x)))) }
 
 // Store atomically stores val into x.
-func (x *Pointer[T]) Store(val *T) { StorePointer(&x.v, unsafe.Pointer(val)) }
+func (x *Pointer[T]) Store(val *T) {
+       StorePointer((*unsafe.Pointer)(unsafe.Pointer(x)), unsafe.Pointer(val))
+}
 
 // Swap atomically stores new into x and returns the previous value.
-func (x *Pointer[T]) Swap(new *T) (old *T) { return (*T)(SwapPointer(&x.v, unsafe.Pointer(new))) }
+func (x *Pointer[T]) Swap(new *T) (old *T) {
+       return (*T)(SwapPointer((*unsafe.Pointer)(unsafe.Pointer(x)), unsafe.Pointer(new)))
+}
 
 // CompareAndSwap executes the compare-and-swap operation for x.
 func (x *Pointer[T]) CompareAndSwap(old, new *T) (swapped bool) {
-       return CompareAndSwapPointer(&x.v, unsafe.Pointer(old), unsafe.Pointer(new))
+       return CompareAndSwapPointer((*unsafe.Pointer)(unsafe.Pointer(x)), unsafe.Pointer(old), unsafe.Pointer(new))
 }
 
 // An Int32 is an atomic int32. The zero value is zero.
@@ -66,21 +72,21 @@ type Int32 struct {
 }
 
 // Load atomically loads and returns the value stored in x.
-func (x *Int32) Load() int32 { return LoadInt32(&x.v) }
+func (x *Int32) Load() int32 { return LoadInt32((*int32)(unsafe.Pointer(x))) }
 
 // Store atomically stores val into x.
-func (x *Int32) Store(val int32) { StoreInt32(&x.v, val) }
+func (x *Int32) Store(val int32) { StoreInt32((*int32)(unsafe.Pointer(x)), val) }
 
 // Swap atomically stores new into x and returns the previous value.
-func (x *Int32) Swap(new int32) (old int32) { return SwapInt32(&x.v, new) }
+func (x *Int32) Swap(new int32) (old int32) { return SwapInt32((*int32)(unsafe.Pointer(x)), new) }
 
 // CompareAndSwap executes the compare-and-swap operation for x.
 func (x *Int32) CompareAndSwap(old, new int32) (swapped bool) {
-       return CompareAndSwapInt32(&x.v, old, new)
+       return CompareAndSwapInt32((*int32)(unsafe.Pointer(x)), old, new)
 }
 
 // Add atomically adds delta to x and returns the new value.
-func (x *Int32) Add(delta int32) (new int32) { return AddInt32(&x.v, delta) }
+func (x *Int32) Add(delta int32) (new int32) { return AddInt32((*int32)(unsafe.Pointer(x)), delta) }
 
 // An Int64 is an atomic int64. The zero value is zero.
 type Int64 struct {
@@ -90,21 +96,21 @@ type Int64 struct {
 }
 
 // Load atomically loads and returns the value stored in x.
-func (x *Int64) Load() int64 { return LoadInt64(&x.v) }
+func (x *Int64) Load() int64 { return LoadInt64((*int64)(unsafe.Pointer(x))) }
 
 // Store atomically stores val into x.
-func (x *Int64) Store(val int64) { StoreInt64(&x.v, val) }
+func (x *Int64) Store(val int64) { StoreInt64((*int64)(unsafe.Pointer(x)), val) }
 
 // Swap atomically stores new into x and returns the previous value.
-func (x *Int64) Swap(new int64) (old int64) { return SwapInt64(&x.v, new) }
+func (x *Int64) Swap(new int64) (old int64) { return SwapInt64((*int64)(unsafe.Pointer(x)), new) }
 
 // CompareAndSwap executes the compare-and-swap operation for x.
 func (x *Int64) CompareAndSwap(old, new int64) (swapped bool) {
-       return CompareAndSwapInt64(&x.v, old, new)
+       return CompareAndSwapInt64((*int64)(unsafe.Pointer(x)), old, new)
 }
 
 // Add atomically adds delta to x and returns the new value.
-func (x *Int64) Add(delta int64) (new int64) { return AddInt64(&x.v, delta) }
+func (x *Int64) Add(delta int64) (new int64) { return AddInt64((*int64)(unsafe.Pointer(x)), delta) }
 
 // An Uint32 is an atomic uint32. The zero value is zero.
 type Uint32 struct {
@@ -113,21 +119,23 @@ type Uint32 struct {
 }
 
 // Load atomically loads and returns the value stored in x.
-func (x *Uint32) Load() uint32 { return LoadUint32(&x.v) }
+func (x *Uint32) Load() uint32 { return LoadUint32((*uint32)(unsafe.Pointer(x))) }
 
 // Store atomically stores val into x.
-func (x *Uint32) Store(val uint32) { StoreUint32(&x.v, val) }
+func (x *Uint32) Store(val uint32) { StoreUint32((*uint32)(unsafe.Pointer(x)), val) }
 
 // Swap atomically stores new into x and returns the previous value.
-func (x *Uint32) Swap(new uint32) (old uint32) { return SwapUint32(&x.v, new) }
+func (x *Uint32) Swap(new uint32) (old uint32) { return SwapUint32((*uint32)(unsafe.Pointer(x)), new) }
 
 // CompareAndSwap executes the compare-and-swap operation for x.
 func (x *Uint32) CompareAndSwap(old, new uint32) (swapped bool) {
-       return CompareAndSwapUint32(&x.v, old, new)
+       return CompareAndSwapUint32((*uint32)(unsafe.Pointer(x)), old, new)
 }
 
 // Add atomically adds delta to x and returns the new value.
-func (x *Uint32) Add(delta uint32) (new uint32) { return AddUint32(&x.v, delta) }
+func (x *Uint32) Add(delta uint32) (new uint32) {
+       return AddUint32((*uint32)(unsafe.Pointer(x)), delta)
+}
 
 // An Uint64 is an atomic uint64. The zero value is zero.
 type Uint64 struct {
@@ -137,21 +145,23 @@ type Uint64 struct {
 }
 
 // Load atomically loads and returns the value stored in x.
-func (x *Uint64) Load() uint64 { return LoadUint64(&x.v) }
+func (x *Uint64) Load() uint64 { return LoadUint64((*uint64)(unsafe.Pointer(x))) }
 
 // Store atomically stores val into x.
-func (x *Uint64) Store(val uint64) { StoreUint64(&x.v, val) }
+func (x *Uint64) Store(val uint64) { StoreUint64((*uint64)(unsafe.Pointer(x)), val) }
 
 // Swap atomically stores new into x and returns the previous value.
-func (x *Uint64) Swap(new uint64) (old uint64) { return SwapUint64(&x.v, new) }
+func (x *Uint64) Swap(new uint64) (old uint64) { return SwapUint64((*uint64)(unsafe.Pointer(x)), new) }
 
 // CompareAndSwap executes the compare-and-swap operation for x.
 func (x *Uint64) CompareAndSwap(old, new uint64) (swapped bool) {
-       return CompareAndSwapUint64(&x.v, old, new)
+       return CompareAndSwapUint64((*uint64)(unsafe.Pointer(x)), old, new)
 }
 
 // Add atomically adds delta to x and returns the new value.
-func (x *Uint64) Add(delta uint64) (new uint64) { return AddUint64(&x.v, delta) }
+func (x *Uint64) Add(delta uint64) (new uint64) {
+       return AddUint64((*uint64)(unsafe.Pointer(x)), delta)
+}
 
 // An Uintptr is an atomic uintptr. The zero value is zero.
 type Uintptr struct {
@@ -160,21 +170,25 @@ type Uintptr struct {
 }
 
 // Load atomically loads and returns the value stored in x.
-func (x *Uintptr) Load() uintptr { return LoadUintptr(&x.v) }
+func (x *Uintptr) Load() uintptr { return LoadUintptr((*uintptr)(unsafe.Pointer(x))) }
 
 // Store atomically stores val into x.
-func (x *Uintptr) Store(val uintptr) { StoreUintptr(&x.v, val) }
+func (x *Uintptr) Store(val uintptr) { StoreUintptr((*uintptr)(unsafe.Pointer(x)), val) }
 
 // Swap atomically stores new into x and returns the previous value.
-func (x *Uintptr) Swap(new uintptr) (old uintptr) { return SwapUintptr(&x.v, new) }
+func (x *Uintptr) Swap(new uintptr) (old uintptr) {
+       return SwapUintptr((*uintptr)(unsafe.Pointer(x)), new)
+}
 
 // CompareAndSwap executes the compare-and-swap operation for x.
 func (x *Uintptr) CompareAndSwap(old, new uintptr) (swapped bool) {
-       return CompareAndSwapUintptr(&x.v, old, new)
+       return CompareAndSwapUintptr((*uintptr)(unsafe.Pointer(x)), old, new)
 }
 
 // Add atomically adds delta to x and returns the new value.
-func (x *Uintptr) Add(delta uintptr) (new uintptr) { return AddUintptr(&x.v, delta) }
+func (x *Uintptr) Add(delta uintptr) (new uintptr) {
+       return AddUintptr((*uintptr)(unsafe.Pointer(x)), delta)
+}
 
 // noCopy may be added to structs which must not be copied
 // after the first use.