src/runtime/internal/atomic/atomic_arm64.s

   1 // Copyright 2014 The Go Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style
   3 // license that can be found in the LICENSE file.
   4
   5 #include "go_asm.h"
   6 #include "textflag.h"
   7
   8 TEXT ·Casint32(SB), NOSPLIT, $0-17
   9         B       ·Cas(SB)
  10
  11 TEXT ·Casint64(SB), NOSPLIT, $0-25
  12         B       ·Cas64(SB)
  13
  14 TEXT ·Casuintptr(SB), NOSPLIT, $0-25
  15         B       ·Cas64(SB)
  16
  17 TEXT ·CasRel(SB), NOSPLIT, $0-17
  18         B       ·Cas(SB)
  19
  20 TEXT ·Loadint32(SB), NOSPLIT, $0-12
  21         B       ·Load(SB)
  22
  23 TEXT ·Loadint64(SB), NOSPLIT, $0-16
  24         B       ·Load64(SB)
  25
  26 TEXT ·Loaduintptr(SB), NOSPLIT, $0-16
  27         B       ·Load64(SB)
  28
  29 TEXT ·Loaduint(SB), NOSPLIT, $0-16
  30         B       ·Load64(SB)
  31
  32 TEXT ·Storeint32(SB), NOSPLIT, $0-12
  33         B       ·Store(SB)
  34
  35 TEXT ·Storeint64(SB), NOSPLIT, $0-16
  36         B       ·Store64(SB)
  37
  38 TEXT ·Storeuintptr(SB), NOSPLIT, $0-16
  39         B       ·Store64(SB)
  40
  41 TEXT ·Xaddint32(SB), NOSPLIT, $0-20
  42         B       ·Xadd(SB)
  43
  44 TEXT ·Xaddint64(SB), NOSPLIT, $0-24
  45         B       ·Xadd64(SB)
  46
  47 TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
  48         B       ·Xadd64(SB)
  49
  50 TEXT ·Casp1(SB), NOSPLIT, $0-25
  51         B ·Cas64(SB)
  52
  53 // uint32 ·Load(uint32 volatile* addr)
  54 TEXT ·Load(SB),NOSPLIT,$0-12
  55         MOVD    ptr+0(FP), R0
  56         LDARW   (R0), R0
  57         MOVW    R0, ret+8(FP)
  58         RET
  59
  60 // uint8 ·Load8(uint8 volatile* addr)
  61 TEXT ·Load8(SB),NOSPLIT,$0-9
  62         MOVD    ptr+0(FP), R0
  63         LDARB   (R0), R0
  64         MOVB    R0, ret+8(FP)
  65         RET
  66
  67 // uint64 ·Load64(uint64 volatile* addr)
  68 TEXT ·Load64(SB),NOSPLIT,$0-16
  69         MOVD    ptr+0(FP), R0
  70         LDAR    (R0), R0
  71         MOVD    R0, ret+8(FP)
  72         RET
  73
  74 // void *·Loadp(void *volatile *addr)
  75 TEXT ·Loadp(SB),NOSPLIT,$0-16
  76         MOVD    ptr+0(FP), R0
  77         LDAR    (R0), R0
  78         MOVD    R0, ret+8(FP)
  79         RET
  80
  81 // uint32 ·LoadAcq(uint32 volatile* addr)
  82 TEXT ·LoadAcq(SB),NOSPLIT,$0-12
  83         B       ·Load(SB)
  84
  85 // uint64 ·LoadAcquintptr(uint64 volatile* addr)
  86 TEXT ·LoadAcq64(SB),NOSPLIT,$0-16
  87         B       ·Load64(SB)
  88
  89 // uintptr ·LoadAcq64(uintptr volatile* addr)
  90 TEXT ·LoadAcquintptr(SB),NOSPLIT,$0-16
  91         B       ·Load64(SB)
  92
  93 TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
  94         B       ·Store64(SB)
  95
  96 TEXT ·StoreRel(SB), NOSPLIT, $0-12
  97         B       ·Store(SB)
  98
  99 TEXT ·StoreRel64(SB), NOSPLIT, $0-16
 100         B       ·Store64(SB)
 101
 102 TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
 103         B       ·Store64(SB)
 104
 105 TEXT ·Store(SB), NOSPLIT, $0-12
 106         MOVD    ptr+0(FP), R0
 107         MOVW    val+8(FP), R1
 108         STLRW   R1, (R0)
 109         RET
 110
 111 TEXT ·Store8(SB), NOSPLIT, $0-9
 112         MOVD    ptr+0(FP), R0
 113         MOVB    val+8(FP), R1
 114         STLRB   R1, (R0)
 115         RET
 116
 117 TEXT ·Store64(SB), NOSPLIT, $0-16
 118         MOVD    ptr+0(FP), R0
 119         MOVD    val+8(FP), R1
 120         STLR    R1, (R0)
 121         RET
 122
 123 // uint32 Xchg(ptr *uint32, new uint32)
 124 // Atomically:
 125 //      old := *ptr;
 126 //      *ptr = new;
 127 //      return old;
 128 TEXT ·Xchg(SB), NOSPLIT, $0-20
 129         MOVD    ptr+0(FP), R0
 130         MOVW    new+8(FP), R1
 131         MOVBU   internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
 132         CBZ     R4, load_store_loop
 133         SWPALW  R1, (R0), R2
 134         MOVW    R2, ret+16(FP)
 135         RET
 136 load_store_loop:
 137         LDAXRW  (R0), R2
 138         STLXRW  R1, (R0), R3
 139         CBNZ    R3, load_store_loop
 140         MOVW    R2, ret+16(FP)
 141         RET
 142
 143 // uint64 Xchg64(ptr *uint64, new uint64)
 144 // Atomically:
 145 //      old := *ptr;
 146 //      *ptr = new;
 147 //      return old;
 148 TEXT ·Xchg64(SB), NOSPLIT, $0-24
 149         MOVD    ptr+0(FP), R0
 150         MOVD    new+8(FP), R1
 151         MOVBU   internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
 152         CBZ     R4, load_store_loop
 153         SWPALD  R1, (R0), R2
 154         MOVD    R2, ret+16(FP)
 155         RET
 156 load_store_loop:
 157         LDAXR   (R0), R2
 158         STLXR   R1, (R0), R3
 159         CBNZ    R3, load_store_loop
 160         MOVD    R2, ret+16(FP)
 161         RET
 162
 163 // bool Cas(uint32 *ptr, uint32 old, uint32 new)
 164 // Atomically:
 165 //      if(*val == old){
 166 //              *val = new;
 167 //              return 1;
 168 //      } else
 169 //              return 0;
 170 TEXT ·Cas(SB), NOSPLIT, $0-17
 171         MOVD    ptr+0(FP), R0
 172         MOVW    old+8(FP), R1
 173         MOVW    new+12(FP), R2
 174         MOVBU   internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
 175         CBZ     R4, load_store_loop
 176         MOVD    R1, R3
 177         CASALW  R3, (R0), R2
 178         CMP     R1, R3
 179         CSET    EQ, R0
 180         MOVB    R0, ret+16(FP)
 181         RET
 182 load_store_loop:
 183         LDAXRW  (R0), R3
 184         CMPW    R1, R3
 185         BNE     ok
 186         STLXRW  R2, (R0), R3
 187         CBNZ    R3, load_store_loop
 188 ok:
 189         CSET    EQ, R0
 190         MOVB    R0, ret+16(FP)
 191         RET
 192
 193 // bool ·Cas64(uint64 *ptr, uint64 old, uint64 new)
 194 // Atomically:
 195 //      if(*val == old){
 196 //              *val = new;
 197 //              return 1;
 198 //      } else {
 199 //              return 0;
 200 //      }
 201 TEXT ·Cas64(SB), NOSPLIT, $0-25
 202         MOVD    ptr+0(FP), R0
 203         MOVD    old+8(FP), R1
 204         MOVD    new+16(FP), R2
 205         MOVBU   internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
 206         CBZ     R4, load_store_loop
 207         MOVD    R1, R3
 208         CASALD  R3, (R0), R2
 209         CMP     R1, R3
 210         CSET    EQ, R0
 211         MOVB    R0, ret+24(FP)
 212         RET
 213 load_store_loop:
 214         LDAXR   (R0), R3
 215         CMP     R1, R3
 216         BNE     ok
 217         STLXR   R2, (R0), R3
 218         CBNZ    R3, load_store_loop
 219 ok:
 220         CSET    EQ, R0
 221         MOVB    R0, ret+24(FP)
 222         RET
 223
 224 // uint32 xadd(uint32 volatile *ptr, int32 delta)
 225 // Atomically:
 226 //      *val += delta;
 227 //      return *val;
 228 TEXT ·Xadd(SB), NOSPLIT, $0-20
 229         MOVD    ptr+0(FP), R0
 230         MOVW    delta+8(FP), R1
 231         MOVBU   internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
 232         CBZ     R4, load_store_loop
 233         LDADDALW        R1, (R0), R2
 234         ADD     R1, R2
 235         MOVW    R2, ret+16(FP)
 236         RET
 237 load_store_loop:
 238         LDAXRW  (R0), R2
 239         ADDW    R2, R1, R2
 240         STLXRW  R2, (R0), R3
 241         CBNZ    R3, load_store_loop
 242         MOVW    R2, ret+16(FP)
 243         RET
 244
 245 // uint64 Xadd64(uint64 volatile *ptr, int64 delta)
 246 // Atomically:
 247 //      *val += delta;
 248 //      return *val;
 249 TEXT ·Xadd64(SB), NOSPLIT, $0-24
 250         MOVD    ptr+0(FP), R0
 251         MOVD    delta+8(FP), R1
 252         MOVBU   internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
 253         CBZ     R4, load_store_loop
 254         LDADDALD        R1, (R0), R2
 255         ADD     R1, R2
 256         MOVD    R2, ret+16(FP)
 257         RET
 258 load_store_loop:
 259         LDAXR   (R0), R2
 260         ADD     R2, R1, R2
 261         STLXR   R2, (R0), R3
 262         CBNZ    R3, load_store_loop
 263         MOVD    R2, ret+16(FP)
 264         RET
 265
 266 TEXT ·Xchgint32(SB), NOSPLIT, $0-20
 267         B       ·Xchg(SB)
 268
 269 TEXT ·Xchgint64(SB), NOSPLIT, $0-24
 270         B       ·Xchg64(SB)
 271
 272 TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
 273         B       ·Xchg64(SB)
 274
 275 TEXT ·And8(SB), NOSPLIT, $0-9
 276         MOVD    ptr+0(FP), R0
 277         MOVB    val+8(FP), R1
 278         MOVBU   internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
 279         CBZ     R4, load_store_loop
 280         MVN     R1, R2
 281         LDCLRALB        R2, (R0), R3
 282         RET
 283 load_store_loop:
 284         LDAXRB  (R0), R2
 285         AND     R1, R2
 286         STLXRB  R2, (R0), R3
 287         CBNZ    R3, load_store_loop
 288         RET
 289
 290 TEXT ·Or8(SB), NOSPLIT, $0-9
 291         MOVD    ptr+0(FP), R0
 292         MOVB    val+8(FP), R1
 293         MOVBU   internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
 294         CBZ     R4, load_store_loop
 295         LDORALB R1, (R0), R2
 296         RET
 297 load_store_loop:
 298         LDAXRB  (R0), R2
 299         ORR     R1, R2
 300         STLXRB  R2, (R0), R3
 301         CBNZ    R3, load_store_loop
 302         RET
 303
 304 // func And(addr *uint32, v uint32)
 305 TEXT ·And(SB), NOSPLIT, $0-12
 306         MOVD    ptr+0(FP), R0
 307         MOVW    val+8(FP), R1
 308         MOVBU   internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
 309         CBZ     R4, load_store_loop
 310         MVN     R1, R2
 311         LDCLRALW        R2, (R0), R3
 312         RET
 313 load_store_loop:
 314         LDAXRW  (R0), R2
 315         AND     R1, R2
 316         STLXRW  R2, (R0), R3
 317         CBNZ    R3, load_store_loop
 318         RET
 319
 320 // func Or(addr *uint32, v uint32)
 321 TEXT ·Or(SB), NOSPLIT, $0-12
 322         MOVD    ptr+0(FP), R0
 323         MOVW    val+8(FP), R1
 324         MOVBU   internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
 325         CBZ     R4, load_store_loop
 326         LDORALW R1, (R0), R2
 327         RET
 328 load_store_loop:
 329         LDAXRW  (R0), R2
 330         ORR     R1, R2
 331         STLXRW  R2, (R0), R3
 332         CBNZ    R3, load_store_loop
 333         RET
 334
 335 // func Or32(addr *uint32, v uint32) old uint32
 336 TEXT ·Or32(SB), NOSPLIT, $0-20
 337         MOVD    ptr+0(FP), R0
 338         MOVW    val+8(FP), R1
 339         MOVBU   internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
 340         CBZ     R4, load_store_loop
 341         LDORALW R1, (R0), R2
 342         MOVD    R2, ret+16(FP)
 343         RET
 344 load_store_loop:
 345         LDAXRW  (R0), R2
 346         ORR     R1, R2, R3
 347         STLXRW  R3, (R0), R4
 348         CBNZ    R4, load_store_loop
 349         MOVD R2, ret+16(FP)
 350         RET
 351
 352 // func And32(addr *uint32, v uint32) old uint32
 353 TEXT ·And32(SB), NOSPLIT, $0-20
 354         MOVD    ptr+0(FP), R0
 355         MOVW    val+8(FP), R1
 356         MOVBU   internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
 357         CBZ     R4, load_store_loop
 358         MVN     R1, R2
 359         LDCLRALW        R2, (R0), R3
 360         MOVD    R3, ret+16(FP)
 361         RET
 362 load_store_loop:
 363         LDAXRW  (R0), R2
 364         AND     R1, R2, R3
 365         STLXRW  R3, (R0), R4
 366         CBNZ    R4, load_store_loop
 367         MOVD R2, ret+16(FP)
 368         RET
 369
 370 // func Or64(addr *uint64, v uint64) old uint64
 371 TEXT ·Or64(SB), NOSPLIT, $0-24
 372         MOVD    ptr+0(FP), R0
 373         MOVD    val+8(FP), R1
 374         MOVBU   internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
 375         CBZ     R4, load_store_loop
 376         LDORALD R1, (R0), R2
 377         MOVD    R2, ret+16(FP)
 378         RET
 379 load_store_loop:
 380         LDAXR   (R0), R2
 381         ORR     R1, R2, R3
 382         STLXR   R3, (R0), R4
 383         CBNZ    R4, load_store_loop
 384         MOVD    R2, ret+16(FP)
 385         RET
 386
 387 // func And64(addr *uint64, v uint64) old uint64
 388 TEXT ·And64(SB), NOSPLIT, $0-24
 389         MOVD    ptr+0(FP), R0
 390         MOVD    val+8(FP), R1
 391         MOVBU   internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
 392         CBZ     R4, load_store_loop
 393         MVN     R1, R2
 394         LDCLRALD        R2, (R0), R3
 395         MOVD    R3, ret+16(FP)
 396         RET
 397 load_store_loop:
 398         LDAXR   (R0), R2
 399         AND     R1, R2, R3
 400         STLXR   R3, (R0), R4
 401         CBNZ    R4, load_store_loop
 402         MOVD    R2, ret+16(FP)
 403         RET
 404
 405 // func Anduintptr(addr *uintptr, v uintptr) old uintptr
 406 TEXT ·Anduintptr(SB), NOSPLIT, $0-24
 407         B       ·And64(SB)
 408
 409 // func Oruintptr(addr *uintptr, v uintptr) old uintptr
 410 TEXT ·Oruintptr(SB), NOSPLIT, $0-24
 411         B       ·Or64(SB)