test/codegen/mathbits.go

   1 // asmcheck
   2
   3 // Copyright 2018 The Go Authors. All rights reserved.
   4 // Use of this source code is governed by a BSD-style
   5 // license that can be found in the LICENSE file.
   6
   7 package codegen
   8
   9 import "math/bits"
  10
  11 // ----------------------- //
  12 //    bits.LeadingZeros    //
  13 // ----------------------- //
  14
  15 func LeadingZeros(n uint) int {
  16         // amd64/v1,amd64/v2:"BSRQ"
  17         // amd64/v3:"LZCNTQ", -"BSRQ"
  18         // s390x:"FLOGR"
  19         // arm:"CLZ" arm64:"CLZ"
  20         // mips:"CLZ"
  21         // wasm:"I64Clz"
  22         // ppc64x:"CNTLZD"
  23         return bits.LeadingZeros(n)
  24 }
  25
  26 func LeadingZeros64(n uint64) int {
  27         // amd64/v1,amd64/v2:"BSRQ"
  28         // amd64/v3:"LZCNTQ", -"BSRQ"
  29         // s390x:"FLOGR"
  30         // arm:"CLZ" arm64:"CLZ"
  31         // mips:"CLZ"
  32         // wasm:"I64Clz"
  33         // ppc64x:"CNTLZD"
  34         return bits.LeadingZeros64(n)
  35 }
  36
  37 func LeadingZeros32(n uint32) int {
  38         // amd64/v1,amd64/v2:"BSRQ","LEAQ",-"CMOVQEQ"
  39         // amd64/v3: "LZCNTL",- "BSRL"
  40         // s390x:"FLOGR"
  41         // arm:"CLZ" arm64:"CLZW"
  42         // mips:"CLZ"
  43         // wasm:"I64Clz"
  44         // ppc64x:"CNTLZW"
  45         return bits.LeadingZeros32(n)
  46 }
  47
  48 func LeadingZeros16(n uint16) int {
  49         // amd64/v1,amd64/v2:"BSRL","LEAL",-"CMOVQEQ"
  50         // amd64/v3: "LZCNTL",- "BSRL"
  51         // s390x:"FLOGR"
  52         // arm:"CLZ" arm64:"CLZ"
  53         // mips:"CLZ"
  54         // wasm:"I64Clz"
  55         // ppc64x:"CNTLZD"
  56         return bits.LeadingZeros16(n)
  57 }
  58
  59 func LeadingZeros8(n uint8) int {
  60         // amd64/v1,amd64/v2:"BSRL","LEAL",-"CMOVQEQ"
  61         // amd64/v3: "LZCNTL",- "BSRL"
  62         // s390x:"FLOGR"
  63         // arm:"CLZ" arm64:"CLZ"
  64         // mips:"CLZ"
  65         // wasm:"I64Clz"
  66         // ppc64x:"CNTLZD"
  67         return bits.LeadingZeros8(n)
  68 }
  69
  70 // --------------- //
  71 //    bits.Len*    //
  72 // --------------- //
  73
  74 func Len(n uint) int {
  75         // amd64/v1,amd64/v2:"BSRQ"
  76         // amd64/v3: "LZCNTQ"
  77         // s390x:"FLOGR"
  78         // arm:"CLZ" arm64:"CLZ"
  79         // mips:"CLZ"
  80         // wasm:"I64Clz"
  81         // ppc64x:"SUBC","CNTLZD"
  82         return bits.Len(n)
  83 }
  84
  85 func Len64(n uint64) int {
  86         // amd64/v1,amd64/v2:"BSRQ"
  87         // amd64/v3: "LZCNTQ"
  88         // s390x:"FLOGR"
  89         // arm:"CLZ" arm64:"CLZ"
  90         // mips:"CLZ"
  91         // wasm:"I64Clz"
  92         // ppc64x:"SUBC","CNTLZD"
  93         return bits.Len64(n)
  94 }
  95
  96 func SubFromLen64(n uint64) int {
  97         // ppc64x:"CNTLZD",-"SUBC"
  98         return 64 - bits.Len64(n)
  99 }
 100
 101 func Len32(n uint32) int {
 102         // amd64/v1,amd64/v2:"BSRQ","LEAQ",-"CMOVQEQ"
 103         // amd64/v3: "LZCNTL"
 104         // s390x:"FLOGR"
 105         // arm:"CLZ" arm64:"CLZ"
 106         // mips:"CLZ"
 107         // wasm:"I64Clz"
 108         // ppc64x: "CNTLZW"
 109         return bits.Len32(n)
 110 }
 111
 112 func Len16(n uint16) int {
 113         // amd64/v1,amd64/v2:"BSRL","LEAL",-"CMOVQEQ"
 114         // amd64/v3: "LZCNTL"
 115         // s390x:"FLOGR"
 116         // arm:"CLZ" arm64:"CLZ"
 117         // mips:"CLZ"
 118         // wasm:"I64Clz"
 119         // ppc64x:"SUBC","CNTLZD"
 120         return bits.Len16(n)
 121 }
 122
 123 func Len8(n uint8) int {
 124         // amd64/v1,amd64/v2:"BSRL","LEAL",-"CMOVQEQ"
 125         // amd64/v3: "LZCNTL"
 126         // s390x:"FLOGR"
 127         // arm:"CLZ" arm64:"CLZ"
 128         // mips:"CLZ"
 129         // wasm:"I64Clz"
 130         // ppc64x:"SUBC","CNTLZD"
 131         return bits.Len8(n)
 132 }
 133
 134 // -------------------- //
 135 //    bits.OnesCount    //
 136 // -------------------- //
 137
 138 // TODO(register args) Restore a m d 6 4 / v 1 :.*x86HasPOPCNT when only one ABI is tested.
 139 func OnesCount(n uint) int {
 140         // amd64/v2:-".*x86HasPOPCNT" amd64/v3:-".*x86HasPOPCNT"
 141         // amd64:"POPCNTQ"
 142         // arm64:"VCNT","VUADDLV"
 143         // s390x:"POPCNT"
 144         // ppc64x:"POPCNTD"
 145         // wasm:"I64Popcnt"
 146         return bits.OnesCount(n)
 147 }
 148
 149 func OnesCount64(n uint64) int {
 150         // amd64/v2:-".*x86HasPOPCNT" amd64/v3:-".*x86HasPOPCNT"
 151         // amd64:"POPCNTQ"
 152         // arm64:"VCNT","VUADDLV"
 153         // s390x:"POPCNT"
 154         // ppc64x:"POPCNTD"
 155         // wasm:"I64Popcnt"
 156         return bits.OnesCount64(n)
 157 }
 158
 159 func OnesCount32(n uint32) int {
 160         // amd64/v2:-".*x86HasPOPCNT" amd64/v3:-".*x86HasPOPCNT"
 161         // amd64:"POPCNTL"
 162         // arm64:"VCNT","VUADDLV"
 163         // s390x:"POPCNT"
 164         // ppc64x:"POPCNTW"
 165         // wasm:"I64Popcnt"
 166         return bits.OnesCount32(n)
 167 }
 168
 169 func OnesCount16(n uint16) int {
 170         // amd64/v2:-".*x86HasPOPCNT" amd64/v3:-".*x86HasPOPCNT"
 171         // amd64:"POPCNTL"
 172         // arm64:"VCNT","VUADDLV"
 173         // s390x:"POPCNT"
 174         // ppc64x:"POPCNTW"
 175         // wasm:"I64Popcnt"
 176         return bits.OnesCount16(n)
 177 }
 178
 179 func OnesCount8(n uint8) int {
 180         // s390x:"POPCNT"
 181         // ppc64x:"POPCNTB"
 182         // wasm:"I64Popcnt"
 183         return bits.OnesCount8(n)
 184 }
 185
 186 // ----------------------- //
 187 //    bits.ReverseBytes    //
 188 // ----------------------- //
 189
 190 func ReverseBytes(n uint) uint {
 191         // amd64:"BSWAPQ"
 192         // 386:"BSWAPL"
 193         // s390x:"MOVDBR"
 194         // arm64:"REV"
 195         return bits.ReverseBytes(n)
 196 }
 197
 198 func ReverseBytes64(n uint64) uint64 {
 199         // amd64:"BSWAPQ"
 200         // 386:"BSWAPL"
 201         // s390x:"MOVDBR"
 202         // arm64:"REV"
 203         // ppc64x/power10: "BRD"
 204         return bits.ReverseBytes64(n)
 205 }
 206
 207 func ReverseBytes32(n uint32) uint32 {
 208         // amd64:"BSWAPL"
 209         // 386:"BSWAPL"
 210         // s390x:"MOVWBR"
 211         // arm64:"REVW"
 212         // ppc64x/power10: "BRW"
 213         return bits.ReverseBytes32(n)
 214 }
 215
 216 func ReverseBytes16(n uint16) uint16 {
 217         // amd64:"ROLW"
 218         // arm64:"REV16W",-"UBFX",-"ORR"
 219         // arm/5:"SLL","SRL","ORR"
 220         // arm/6:"REV16"
 221         // arm/7:"REV16"
 222         // ppc64x/power10: "BRH"
 223         return bits.ReverseBytes16(n)
 224 }
 225
 226 // --------------------- //
 227 //    bits.RotateLeft    //
 228 // --------------------- //
 229
 230 func RotateLeft64(n uint64) uint64 {
 231         // amd64:"ROLQ"
 232         // arm64:"ROR"
 233         // ppc64x:"ROTL"
 234         // s390x:"RISBGZ\t[$]0, [$]63, [$]37, "
 235         // wasm:"I64Rotl"
 236         return bits.RotateLeft64(n, 37)
 237 }
 238
 239 func RotateLeft32(n uint32) uint32 {
 240         // amd64:"ROLL" 386:"ROLL"
 241         // arm:`MOVW\tR[0-9]+@>23`
 242         // arm64:"RORW"
 243         // ppc64x:"ROTLW"
 244         // s390x:"RLL"
 245         // wasm:"I32Rotl"
 246         return bits.RotateLeft32(n, 9)
 247 }
 248
 249 func RotateLeft16(n uint16, s int) uint16 {
 250         // amd64:"ROLW" 386:"ROLW"
 251         // arm64:"RORW",-"CSEL"
 252         return bits.RotateLeft16(n, s)
 253 }
 254
 255 func RotateLeft8(n uint8, s int) uint8 {
 256         // amd64:"ROLB" 386:"ROLB"
 257         // arm64:"LSL","LSR",-"CSEL"
 258         return bits.RotateLeft8(n, s)
 259 }
 260
 261 func RotateLeftVariable(n uint, m int) uint {
 262         // amd64:"ROLQ"
 263         // arm64:"ROR"
 264         // ppc64x:"ROTL"
 265         // s390x:"RLLG"
 266         // wasm:"I64Rotl"
 267         return bits.RotateLeft(n, m)
 268 }
 269
 270 func RotateLeftVariable64(n uint64, m int) uint64 {
 271         // amd64:"ROLQ"
 272         // arm64:"ROR"
 273         // ppc64x:"ROTL"
 274         // s390x:"RLLG"
 275         // wasm:"I64Rotl"
 276         return bits.RotateLeft64(n, m)
 277 }
 278
 279 func RotateLeftVariable32(n uint32, m int) uint32 {
 280         // arm:`MOVW\tR[0-9]+@>R[0-9]+`
 281         // amd64:"ROLL"
 282         // arm64:"RORW"
 283         // ppc64x:"ROTLW"
 284         // s390x:"RLL"
 285         // wasm:"I32Rotl"
 286         return bits.RotateLeft32(n, m)
 287 }
 288
 289 // ------------------------ //
 290 //    bits.TrailingZeros    //
 291 // ------------------------ //
 292
 293 func TrailingZeros(n uint) int {
 294         // amd64/v1,amd64/v2:"BSFQ","MOVL\t\\$64","CMOVQEQ"
 295         // amd64/v3:"TZCNTQ"
 296         // 386:"BSFL"
 297         // arm:"CLZ"
 298         // arm64:"RBIT","CLZ"
 299         // s390x:"FLOGR"
 300         // ppc64x/power8:"ANDN","POPCNTD"
 301         // ppc64x/power9: "CNTTZD"
 302         // wasm:"I64Ctz"
 303         return bits.TrailingZeros(n)
 304 }
 305
 306 func TrailingZeros64(n uint64) int {
 307         // amd64/v1,amd64/v2:"BSFQ","MOVL\t\\$64","CMOVQEQ"
 308         // amd64/v3:"TZCNTQ"
 309         // 386:"BSFL"
 310         // arm64:"RBIT","CLZ"
 311         // s390x:"FLOGR"
 312         // ppc64x/power8:"ANDN","POPCNTD"
 313         // ppc64x/power9: "CNTTZD"
 314         // wasm:"I64Ctz"
 315         return bits.TrailingZeros64(n)
 316 }
 317
 318 func TrailingZeros64Subtract(n uint64) int {
 319         // ppc64x/power8:"NEG","SUBC","ANDN","POPCNTD"
 320         // ppc64x/power9:"SUBC","CNTTZD"
 321         return bits.TrailingZeros64(1 - n)
 322 }
 323
 324 func TrailingZeros32(n uint32) int {
 325         // amd64/v1,amd64/v2:"BTSQ\\t\\$32","BSFQ"
 326         // amd64/v3:"TZCNTL"
 327         // 386:"BSFL"
 328         // arm:"CLZ"
 329         // arm64:"RBITW","CLZW"
 330         // s390x:"FLOGR","MOVWZ"
 331         // ppc64x/power8:"ANDN","POPCNTW"
 332         // ppc64x/power9: "CNTTZW"
 333         // wasm:"I64Ctz"
 334         return bits.TrailingZeros32(n)
 335 }
 336
 337 func TrailingZeros16(n uint16) int {
 338         // amd64:"BSFL","ORL\\t\\$65536"
 339         // 386:"BSFL\t"
 340         // arm:"ORR\t\\$65536","CLZ",-"MOVHU\tR"
 341         // arm64:"ORR\t\\$65536","RBITW","CLZW",-"MOVHU\tR",-"RBIT\t",-"CLZ\t"
 342         // s390x:"FLOGR","OR\t\\$65536"
 343         // ppc64x/power8:"POPCNTD","ORIS\\t\\$1"
 344         // ppc64x/power9:"CNTTZD","ORIS\\t\\$1"
 345         // wasm:"I64Ctz"
 346         return bits.TrailingZeros16(n)
 347 }
 348
 349 func TrailingZeros8(n uint8) int {
 350         // amd64:"BSFL","ORL\\t\\$256"
 351         // 386:"BSFL"
 352         // arm:"ORR\t\\$256","CLZ",-"MOVBU\tR"
 353         // arm64:"ORR\t\\$256","RBITW","CLZW",-"MOVBU\tR",-"RBIT\t",-"CLZ\t"
 354         // s390x:"FLOGR","OR\t\\$256"
 355         // wasm:"I64Ctz"
 356         return bits.TrailingZeros8(n)
 357 }
 358
 359 // IterateBitsNN checks special handling of TrailingZerosNN when the input is known to be non-zero.
 360
 361 func IterateBits(n uint) int {
 362         i := 0
 363         for n != 0 {
 364                 // amd64/v1,amd64/v2:"BSFQ",-"CMOVEQ"
 365                 // amd64/v3:"TZCNTQ"
 366                 i += bits.TrailingZeros(n)
 367                 n &= n - 1
 368         }
 369         return i
 370 }
 371
 372 func IterateBits64(n uint64) int {
 373         i := 0
 374         for n != 0 {
 375                 // amd64/v1,amd64/v2:"BSFQ",-"CMOVEQ"
 376                 // amd64/v3:"TZCNTQ"
 377                 i += bits.TrailingZeros64(n)
 378                 n &= n - 1
 379         }
 380         return i
 381 }
 382
 383 func IterateBits32(n uint32) int {
 384         i := 0
 385         for n != 0 {
 386                 // amd64/v1,amd64/v2:"BSFL",-"BTSQ"
 387                 // amd64/v3:"TZCNTL"
 388                 i += bits.TrailingZeros32(n)
 389                 n &= n - 1
 390         }
 391         return i
 392 }
 393
 394 func IterateBits16(n uint16) int {
 395         i := 0
 396         for n != 0 {
 397                 // amd64/v1,amd64/v2:"BSFL",-"BTSL"
 398                 // amd64/v3:"TZCNTL"
 399                 // arm64:"RBITW","CLZW",-"ORR"
 400                 i += bits.TrailingZeros16(n)
 401                 n &= n - 1
 402         }
 403         return i
 404 }
 405
 406 func IterateBits8(n uint8) int {
 407         i := 0
 408         for n != 0 {
 409                 // amd64/v1,amd64/v2:"BSFL",-"BTSL"
 410                 // amd64/v3:"TZCNTL"
 411                 // arm64:"RBITW","CLZW",-"ORR"
 412                 i += bits.TrailingZeros8(n)
 413                 n &= n - 1
 414         }
 415         return i
 416 }
 417
 418 // --------------- //
 419 //    bits.Add*    //
 420 // --------------- //
 421
 422 func Add(x, y, ci uint) (r, co uint) {
 423         // arm64:"ADDS","ADCS","ADC",-"ADD\t",-"CMP"
 424         // amd64:"NEGL","ADCQ","SBBQ","NEGQ"
 425         // ppc64x: "ADDC", "ADDE", "ADDZE"
 426         // s390x:"ADDE","ADDC\t[$]-1,"
 427         // riscv64: "ADD","SLTU"
 428         return bits.Add(x, y, ci)
 429 }
 430
 431 func AddC(x, ci uint) (r, co uint) {
 432         // arm64:"ADDS","ADCS","ADC",-"ADD\t",-"CMP"
 433         // amd64:"NEGL","ADCQ","SBBQ","NEGQ"
 434         // loong64: "ADDV", "SGTU"
 435         // ppc64x: "ADDC", "ADDE", "ADDZE"
 436         // s390x:"ADDE","ADDC\t[$]-1,"
 437         // mips64:"ADDV","SGTU"
 438         // riscv64: "ADD","SLTU"
 439         return bits.Add(x, 7, ci)
 440 }
 441
 442 func AddZ(x, y uint) (r, co uint) {
 443         // arm64:"ADDS","ADC",-"ADCS",-"ADD\t",-"CMP"
 444         // amd64:"ADDQ","SBBQ","NEGQ",-"NEGL",-"ADCQ"
 445         // loong64: "ADDV", "SGTU"
 446         // ppc64x: "ADDC", -"ADDE", "ADDZE"
 447         // s390x:"ADDC",-"ADDC\t[$]-1,"
 448         // mips64:"ADDV","SGTU"
 449         // riscv64: "ADD","SLTU"
 450         return bits.Add(x, y, 0)
 451 }
 452
 453 func AddR(x, y, ci uint) uint {
 454         // arm64:"ADDS","ADCS",-"ADD\t",-"CMP"
 455         // amd64:"NEGL","ADCQ",-"SBBQ",-"NEGQ"
 456         // loong64: "ADDV", -"SGTU"
 457         // ppc64x: "ADDC", "ADDE", -"ADDZE"
 458         // s390x:"ADDE","ADDC\t[$]-1,"
 459         // mips64:"ADDV",-"SGTU"
 460         // riscv64: "ADD",-"SLTU"
 461         r, _ := bits.Add(x, y, ci)
 462         return r
 463 }
 464
 465 func AddM(p, q, r *[3]uint) {
 466         var c uint
 467         r[0], c = bits.Add(p[0], q[0], c)
 468         // arm64:"ADCS",-"ADD\t",-"CMP"
 469         // amd64:"ADCQ",-"NEGL",-"SBBQ",-"NEGQ"
 470         // s390x:"ADDE",-"ADDC\t[$]-1,"
 471         r[1], c = bits.Add(p[1], q[1], c)
 472         r[2], c = bits.Add(p[2], q[2], c)
 473 }
 474
 475 func Add64(x, y, ci uint64) (r, co uint64) {
 476         // arm64:"ADDS","ADCS","ADC",-"ADD\t",-"CMP"
 477         // amd64:"NEGL","ADCQ","SBBQ","NEGQ"
 478         // loong64: "ADDV", "SGTU"
 479         // ppc64x: "ADDC", "ADDE", "ADDZE"
 480         // s390x:"ADDE","ADDC\t[$]-1,"
 481         // mips64:"ADDV","SGTU"
 482         // riscv64: "ADD","SLTU"
 483         return bits.Add64(x, y, ci)
 484 }
 485
 486 func Add64C(x, ci uint64) (r, co uint64) {
 487         // arm64:"ADDS","ADCS","ADC",-"ADD\t",-"CMP"
 488         // amd64:"NEGL","ADCQ","SBBQ","NEGQ"
 489         // loong64: "ADDV", "SGTU"
 490         // ppc64x: "ADDC", "ADDE", "ADDZE"
 491         // s390x:"ADDE","ADDC\t[$]-1,"
 492         // mips64:"ADDV","SGTU"
 493         // riscv64: "ADD","SLTU"
 494         return bits.Add64(x, 7, ci)
 495 }
 496
 497 func Add64Z(x, y uint64) (r, co uint64) {
 498         // arm64:"ADDS","ADC",-"ADCS",-"ADD\t",-"CMP"
 499         // amd64:"ADDQ","SBBQ","NEGQ",-"NEGL",-"ADCQ"
 500         // loong64: "ADDV", "SGTU"
 501         // ppc64x: "ADDC", -"ADDE", "ADDZE"
 502         // s390x:"ADDC",-"ADDC\t[$]-1,"
 503         // mips64:"ADDV","SGTU"
 504         // riscv64: "ADD","SLTU"
 505         return bits.Add64(x, y, 0)
 506 }
 507
 508 func Add64R(x, y, ci uint64) uint64 {
 509         // arm64:"ADDS","ADCS",-"ADD\t",-"CMP"
 510         // amd64:"NEGL","ADCQ",-"SBBQ",-"NEGQ"
 511         // loong64: "ADDV", -"SGTU"
 512         // ppc64x: "ADDC", "ADDE", -"ADDZE"
 513         // s390x:"ADDE","ADDC\t[$]-1,"
 514         // mips64:"ADDV",-"SGTU"
 515         // riscv64: "ADD",-"SLTU"
 516         r, _ := bits.Add64(x, y, ci)
 517         return r
 518 }
 519 func Add64M(p, q, r *[3]uint64) {
 520         var c uint64
 521         r[0], c = bits.Add64(p[0], q[0], c)
 522         // arm64:"ADCS",-"ADD\t",-"CMP"
 523         // amd64:"ADCQ",-"NEGL",-"SBBQ",-"NEGQ"
 524         // ppc64x: -"ADDC", "ADDE", -"ADDZE"
 525         // s390x:"ADDE",-"ADDC\t[$]-1,"
 526         r[1], c = bits.Add64(p[1], q[1], c)
 527         r[2], c = bits.Add64(p[2], q[2], c)
 528 }
 529
 530 func Add64MSaveC(p, q, r, c *[2]uint64) {
 531         // ppc64x: "ADDC\tR", "ADDZE"
 532         r[0], c[0] = bits.Add64(p[0], q[0], 0)
 533         // ppc64x: "ADDC\t[$]-1", "ADDE", "ADDZE"
 534         r[1], c[1] = bits.Add64(p[1], q[1], c[0])
 535 }
 536
 537 func Add64PanicOnOverflowEQ(a, b uint64) uint64 {
 538         r, c := bits.Add64(a, b, 0)
 539         // s390x:"BRC\t[$]3,",-"ADDE"
 540         if c == 1 {
 541                 panic("overflow")
 542         }
 543         return r
 544 }
 545
 546 func Add64PanicOnOverflowNE(a, b uint64) uint64 {
 547         r, c := bits.Add64(a, b, 0)
 548         // s390x:"BRC\t[$]3,",-"ADDE"
 549         if c != 0 {
 550                 panic("overflow")
 551         }
 552         return r
 553 }
 554
 555 func Add64PanicOnOverflowGT(a, b uint64) uint64 {
 556         r, c := bits.Add64(a, b, 0)
 557         // s390x:"BRC\t[$]3,",-"ADDE"
 558         if c > 0 {
 559                 panic("overflow")
 560         }
 561         return r
 562 }
 563
 564 func Add64MPanicOnOverflowEQ(a, b [2]uint64) [2]uint64 {
 565         var r [2]uint64
 566         var c uint64
 567         r[0], c = bits.Add64(a[0], b[0], c)
 568         r[1], c = bits.Add64(a[1], b[1], c)
 569         // s390x:"BRC\t[$]3,"
 570         if c == 1 {
 571                 panic("overflow")
 572         }
 573         return r
 574 }
 575
 576 func Add64MPanicOnOverflowNE(a, b [2]uint64) [2]uint64 {
 577         var r [2]uint64
 578         var c uint64
 579         r[0], c = bits.Add64(a[0], b[0], c)
 580         r[1], c = bits.Add64(a[1], b[1], c)
 581         // s390x:"BRC\t[$]3,"
 582         if c != 0 {
 583                 panic("overflow")
 584         }
 585         return r
 586 }
 587
 588 func Add64MPanicOnOverflowGT(a, b [2]uint64) [2]uint64 {
 589         var r [2]uint64
 590         var c uint64
 591         r[0], c = bits.Add64(a[0], b[0], c)
 592         r[1], c = bits.Add64(a[1], b[1], c)
 593         // s390x:"BRC\t[$]3,"
 594         if c > 0 {
 595                 panic("overflow")
 596         }
 597         return r
 598 }
 599
 600 // Verify independent carry chain operations are scheduled efficiently
 601 // and do not cause unnecessary save/restore of the CA bit.
 602 //
 603 // This is an example of why CarryChainTail priority must be lower
 604 // (earlier in the block) than Memory. f[0]=f1 could be scheduled
 605 // after the first two lower 64 bit limb adds, but before either
 606 // high 64 bit limbs are added.
 607 //
 608 // This is what happened on PPC64 when compiling
 609 // crypto/internal/edwards25519/field.feMulGeneric.
 610 func Add64MultipleChains(a, b, c, d [2]uint64) {
 611         var cx, d1, d2 uint64
 612         a1, a2 := a[0], a[1]
 613         b1, b2 := b[0], b[1]
 614         c1, c2 := c[0], c[1]
 615
 616         // ppc64x: "ADDC\tR\\d+,", -"ADDE", -"MOVD\tXER"
 617         d1, cx = bits.Add64(a1, b1, 0)
 618         // ppc64x: "ADDE", -"ADDC", -"MOVD\t.*, XER"
 619         d2, _ = bits.Add64(a2, b2, cx)
 620
 621         // ppc64x: "ADDC\tR\\d+,", -"ADDE", -"MOVD\tXER"
 622         d1, cx = bits.Add64(c1, d1, 0)
 623         // ppc64x: "ADDE", -"ADDC", -"MOVD\t.*, XER"
 624         d2, _ = bits.Add64(c2, d2, cx)
 625         d[0] = d1
 626         d[1] = d2
 627 }
 628
 629 // --------------- //
 630 //    bits.Sub*    //
 631 // --------------- //
 632
 633 func Sub(x, y, ci uint) (r, co uint) {
 634         // amd64:"NEGL","SBBQ","NEGQ"
 635         // arm64:"NEGS","SBCS","NGC","NEG",-"ADD",-"SUB",-"CMP"
 636         // loong64:"SUBV","SGTU"
 637         // ppc64x:"SUBC", "SUBE", "SUBZE", "NEG"
 638         // s390x:"SUBE"
 639         // mips64:"SUBV","SGTU"
 640         // riscv64: "SUB","SLTU"
 641         return bits.Sub(x, y, ci)
 642 }
 643
 644 func SubC(x, ci uint) (r, co uint) {
 645         // amd64:"NEGL","SBBQ","NEGQ"
 646         // arm64:"NEGS","SBCS","NGC","NEG",-"ADD",-"SUB",-"CMP"
 647         // loong64:"SUBV","SGTU"
 648         // ppc64x:"SUBC", "SUBE", "SUBZE", "NEG"
 649         // s390x:"SUBE"
 650         // mips64:"SUBV","SGTU"
 651         // riscv64: "SUB","SLTU"
 652         return bits.Sub(x, 7, ci)
 653 }
 654
 655 func SubZ(x, y uint) (r, co uint) {
 656         // amd64:"SUBQ","SBBQ","NEGQ",-"NEGL"
 657         // arm64:"SUBS","NGC","NEG",-"SBCS",-"ADD",-"SUB\t",-"CMP"
 658         // loong64:"SUBV","SGTU"
 659         // ppc64x:"SUBC", -"SUBE", "SUBZE", "NEG"
 660         // s390x:"SUBC"
 661         // mips64:"SUBV","SGTU"
 662         // riscv64: "SUB","SLTU"
 663         return bits.Sub(x, y, 0)
 664 }
 665
 666 func SubR(x, y, ci uint) uint {
 667         // amd64:"NEGL","SBBQ",-"NEGQ"
 668         // arm64:"NEGS","SBCS",-"NGC",-"NEG\t",-"ADD",-"SUB",-"CMP"
 669         // loong64:"SUBV",-"SGTU"
 670         // ppc64x:"SUBC", "SUBE", -"SUBZE", -"NEG"
 671         // s390x:"SUBE"
 672         // riscv64: "SUB",-"SLTU"
 673         r, _ := bits.Sub(x, y, ci)
 674         return r
 675 }
 676 func SubM(p, q, r *[3]uint) {
 677         var c uint
 678         r[0], c = bits.Sub(p[0], q[0], c)
 679         // amd64:"SBBQ",-"NEGL",-"NEGQ"
 680         // arm64:"SBCS",-"NEGS",-"NGC",-"NEG",-"ADD",-"SUB",-"CMP"
 681         // ppc64x:-"SUBC", "SUBE", -"SUBZE", -"NEG"
 682         // s390x:"SUBE"
 683         r[1], c = bits.Sub(p[1], q[1], c)
 684         r[2], c = bits.Sub(p[2], q[2], c)
 685 }
 686
 687 func Sub64(x, y, ci uint64) (r, co uint64) {
 688         // amd64:"NEGL","SBBQ","NEGQ"
 689         // arm64:"NEGS","SBCS","NGC","NEG",-"ADD",-"SUB",-"CMP"
 690         // loong64:"SUBV","SGTU"
 691         // ppc64x:"SUBC", "SUBE", "SUBZE", "NEG"
 692         // s390x:"SUBE"
 693         // mips64:"SUBV","SGTU"
 694         // riscv64: "SUB","SLTU"
 695         return bits.Sub64(x, y, ci)
 696 }
 697
 698 func Sub64C(x, ci uint64) (r, co uint64) {
 699         // amd64:"NEGL","SBBQ","NEGQ"
 700         // arm64:"NEGS","SBCS","NGC","NEG",-"ADD",-"SUB",-"CMP"
 701         // loong64:"SUBV","SGTU"
 702         // ppc64x:"SUBC", "SUBE", "SUBZE", "NEG"
 703         // s390x:"SUBE"
 704         // mips64:"SUBV","SGTU"
 705         // riscv64: "SUB","SLTU"
 706         return bits.Sub64(x, 7, ci)
 707 }
 708
 709 func Sub64Z(x, y uint64) (r, co uint64) {
 710         // amd64:"SUBQ","SBBQ","NEGQ",-"NEGL"
 711         // arm64:"SUBS","NGC","NEG",-"SBCS",-"ADD",-"SUB\t",-"CMP"
 712         // loong64:"SUBV","SGTU"
 713         // ppc64x:"SUBC", -"SUBE", "SUBZE", "NEG"
 714         // s390x:"SUBC"
 715         // mips64:"SUBV","SGTU"
 716         // riscv64: "SUB","SLTU"
 717         return bits.Sub64(x, y, 0)
 718 }
 719
 720 func Sub64R(x, y, ci uint64) uint64 {
 721         // amd64:"NEGL","SBBQ",-"NEGQ"
 722         // arm64:"NEGS","SBCS",-"NGC",-"NEG\t",-"ADD",-"SUB",-"CMP"
 723         // loong64:"SUBV",-"SGTU"
 724         // ppc64x:"SUBC", "SUBE", -"SUBZE", -"NEG"
 725         // s390x:"SUBE"
 726         // riscv64: "SUB",-"SLTU"
 727         r, _ := bits.Sub64(x, y, ci)
 728         return r
 729 }
 730 func Sub64M(p, q, r *[3]uint64) {
 731         var c uint64
 732         r[0], c = bits.Sub64(p[0], q[0], c)
 733         // amd64:"SBBQ",-"NEGL",-"NEGQ"
 734         // arm64:"SBCS",-"NEGS",-"NGC",-"NEG",-"ADD",-"SUB",-"CMP"
 735         // s390x:"SUBE"
 736         r[1], c = bits.Sub64(p[1], q[1], c)
 737         r[2], c = bits.Sub64(p[2], q[2], c)
 738 }
 739
 740 func Sub64MSaveC(p, q, r, c *[2]uint64) {
 741         // ppc64x:"SUBC\tR\\d+, R\\d+,", "SUBZE", "NEG"
 742         r[0], c[0] = bits.Sub64(p[0], q[0], 0)
 743         // ppc64x:"SUBC\tR\\d+, [$]0,", "SUBE", "SUBZE", "NEG"
 744         r[1], c[1] = bits.Sub64(p[1], q[1], c[0])
 745 }
 746
 747 func Sub64PanicOnOverflowEQ(a, b uint64) uint64 {
 748         r, b := bits.Sub64(a, b, 0)
 749         // s390x:"BRC\t[$]12,",-"ADDE",-"SUBE"
 750         if b == 1 {
 751                 panic("overflow")
 752         }
 753         return r
 754 }
 755
 756 func Sub64PanicOnOverflowNE(a, b uint64) uint64 {
 757         r, b := bits.Sub64(a, b, 0)
 758         // s390x:"BRC\t[$]12,",-"ADDE",-"SUBE"
 759         if b != 0 {
 760                 panic("overflow")
 761         }
 762         return r
 763 }
 764
 765 func Sub64PanicOnOverflowGT(a, b uint64) uint64 {
 766         r, b := bits.Sub64(a, b, 0)
 767         // s390x:"BRC\t[$]12,",-"ADDE",-"SUBE"
 768         if b > 0 {
 769                 panic("overflow")
 770         }
 771         return r
 772 }
 773
 774 func Sub64MPanicOnOverflowEQ(a, b [2]uint64) [2]uint64 {
 775         var r [2]uint64
 776         var c uint64
 777         r[0], c = bits.Sub64(a[0], b[0], c)
 778         r[1], c = bits.Sub64(a[1], b[1], c)
 779         // s390x:"BRC\t[$]12,"
 780         if c == 1 {
 781                 panic("overflow")
 782         }
 783         return r
 784 }
 785
 786 func Sub64MPanicOnOverflowNE(a, b [2]uint64) [2]uint64 {
 787         var r [2]uint64
 788         var c uint64
 789         r[0], c = bits.Sub64(a[0], b[0], c)
 790         r[1], c = bits.Sub64(a[1], b[1], c)
 791         // s390x:"BRC\t[$]12,"
 792         if c != 0 {
 793                 panic("overflow")
 794         }
 795         return r
 796 }
 797
 798 func Sub64MPanicOnOverflowGT(a, b [2]uint64) [2]uint64 {
 799         var r [2]uint64
 800         var c uint64
 801         r[0], c = bits.Sub64(a[0], b[0], c)
 802         r[1], c = bits.Sub64(a[1], b[1], c)
 803         // s390x:"BRC\t[$]12,"
 804         if c > 0 {
 805                 panic("overflow")
 806         }
 807         return r
 808 }
 809
 810 // --------------- //
 811 //    bits.Mul*    //
 812 // --------------- //
 813
 814 func Mul(x, y uint) (hi, lo uint) {
 815         // amd64:"MULQ"
 816         // arm64:"UMULH","MUL"
 817         // ppc64x:"MULHDU","MULLD"
 818         // s390x:"MLGR"
 819         // mips64: "MULVU"
 820         return bits.Mul(x, y)
 821 }
 822
 823 func Mul64(x, y uint64) (hi, lo uint64) {
 824         // amd64:"MULQ"
 825         // arm64:"UMULH","MUL"
 826         // ppc64x:"MULHDU","MULLD"
 827         // s390x:"MLGR"
 828         // mips64: "MULVU"
 829         // riscv64:"MULHU","MUL"
 830         return bits.Mul64(x, y)
 831 }
 832
 833 func Mul64HiOnly(x, y uint64) uint64 {
 834         // arm64:"UMULH",-"MUL"
 835         // riscv64:"MULHU",-"MUL\t"
 836         hi, _ := bits.Mul64(x, y)
 837         return hi
 838 }
 839
 840 func Mul64LoOnly(x, y uint64) uint64 {
 841         // arm64:"MUL",-"UMULH"
 842         // riscv64:"MUL\t",-"MULHU"
 843         _, lo := bits.Mul64(x, y)
 844         return lo
 845 }
 846
 847 // --------------- //
 848 //    bits.Div*    //
 849 // --------------- //
 850
 851 func Div(hi, lo, x uint) (q, r uint) {
 852         // amd64:"DIVQ"
 853         return bits.Div(hi, lo, x)
 854 }
 855
 856 func Div32(hi, lo, x uint32) (q, r uint32) {
 857         // arm64:"ORR","UDIV","MSUB",-"UREM"
 858         return bits.Div32(hi, lo, x)
 859 }
 860
 861 func Div64(hi, lo, x uint64) (q, r uint64) {
 862         // amd64:"DIVQ"
 863         return bits.Div64(hi, lo, x)
 864 }
 865
 866 func Div64degenerate(x uint64) (q, r uint64) {
 867         // amd64:-"DIVQ"
 868         return bits.Div64(0, x, 5)
 869 }