test/codegen/mathbits.go

   1 // asmcheck
   2
   3 // Copyright 2018 The Go Authors. All rights reserved.
   4 // Use of this source code is governed by a BSD-style
   5 // license that can be found in the LICENSE file.
   6
   7 package codegen
   8
   9 import "math/bits"
  10
  11 // ----------------------- //
  12 //    bits.LeadingZeros    //
  13 // ----------------------- //
  14
  15 func LeadingZeros(n uint) int {
  16         // amd64/v1,amd64/v2:"BSRQ"
  17         // amd64/v3:"LZCNTQ", -"BSRQ"
  18         // s390x:"FLOGR"
  19         // arm:"CLZ" arm64:"CLZ"
  20         // mips:"CLZ"
  21         // wasm:"I64Clz"
  22         // ppc64x:"CNTLZD"
  23         return bits.LeadingZeros(n)
  24 }
  25
  26 func LeadingZeros64(n uint64) int {
  27         // amd64/v1,amd64/v2:"BSRQ"
  28         // amd64/v3:"LZCNTQ", -"BSRQ"
  29         // s390x:"FLOGR"
  30         // arm:"CLZ" arm64:"CLZ"
  31         // mips:"CLZ"
  32         // wasm:"I64Clz"
  33         // ppc64x:"CNTLZD"
  34         return bits.LeadingZeros64(n)
  35 }
  36
  37 func LeadingZeros32(n uint32) int {
  38         // amd64/v1,amd64/v2:"BSRQ","LEAQ",-"CMOVQEQ"
  39         // amd64/v3: "LZCNTL",- "BSRL"
  40         // s390x:"FLOGR"
  41         // arm:"CLZ" arm64:"CLZW"
  42         // mips:"CLZ"
  43         // wasm:"I64Clz"
  44         // ppc64x:"CNTLZW"
  45         return bits.LeadingZeros32(n)
  46 }
  47
  48 func LeadingZeros16(n uint16) int {
  49         // amd64/v1,amd64/v2:"BSRL","LEAL",-"CMOVQEQ"
  50         // amd64/v3: "LZCNTL",- "BSRL"
  51         // s390x:"FLOGR"
  52         // arm:"CLZ" arm64:"CLZ"
  53         // mips:"CLZ"
  54         // wasm:"I64Clz"
  55         // ppc64x:"CNTLZD"
  56         return bits.LeadingZeros16(n)
  57 }
  58
  59 func LeadingZeros8(n uint8) int {
  60         // amd64/v1,amd64/v2:"BSRL","LEAL",-"CMOVQEQ"
  61         // amd64/v3: "LZCNTL",- "BSRL"
  62         // s390x:"FLOGR"
  63         // arm:"CLZ" arm64:"CLZ"
  64         // mips:"CLZ"
  65         // wasm:"I64Clz"
  66         // ppc64x:"CNTLZD"
  67         return bits.LeadingZeros8(n)
  68 }
  69
  70 // --------------- //
  71 //    bits.Len*    //
  72 // --------------- //
  73
  74 func Len(n uint) int {
  75         // amd64/v1,amd64/v2:"BSRQ"
  76         // amd64/v3: "LZCNTQ"
  77         // s390x:"FLOGR"
  78         // arm:"CLZ" arm64:"CLZ"
  79         // mips:"CLZ"
  80         // wasm:"I64Clz"
  81         // ppc64x:"SUBC","CNTLZD"
  82         return bits.Len(n)
  83 }
  84
  85 func Len64(n uint64) int {
  86         // amd64/v1,amd64/v2:"BSRQ"
  87         // amd64/v3: "LZCNTQ"
  88         // s390x:"FLOGR"
  89         // arm:"CLZ" arm64:"CLZ"
  90         // mips:"CLZ"
  91         // wasm:"I64Clz"
  92         // ppc64x:"SUBC","CNTLZD"
  93         return bits.Len64(n)
  94 }
  95
  96 func SubFromLen64(n uint64) int {
  97         // ppc64x:"CNTLZD",-"SUBC"
  98         return 64 - bits.Len64(n)
  99 }
 100
 101 func Len32(n uint32) int {
 102         // amd64/v1,amd64/v2:"BSRQ","LEAQ",-"CMOVQEQ"
 103         // amd64/v3: "LZCNTL"
 104         // s390x:"FLOGR"
 105         // arm:"CLZ" arm64:"CLZ"
 106         // mips:"CLZ"
 107         // wasm:"I64Clz"
 108         // ppc64x: "CNTLZW"
 109         return bits.Len32(n)
 110 }
 111
 112 func Len16(n uint16) int {
 113         // amd64/v1,amd64/v2:"BSRL","LEAL",-"CMOVQEQ"
 114         // amd64/v3: "LZCNTL"
 115         // s390x:"FLOGR"
 116         // arm:"CLZ" arm64:"CLZ"
 117         // mips:"CLZ"
 118         // wasm:"I64Clz"
 119         // ppc64x:"SUBC","CNTLZD"
 120         return bits.Len16(n)
 121 }
 122
 123 func Len8(n uint8) int {
 124         // amd64/v1,amd64/v2:"BSRL","LEAL",-"CMOVQEQ"
 125         // amd64/v3: "LZCNTL"
 126         // s390x:"FLOGR"
 127         // arm:"CLZ" arm64:"CLZ"
 128         // mips:"CLZ"
 129         // wasm:"I64Clz"
 130         // ppc64x:"SUBC","CNTLZD"
 131         return bits.Len8(n)
 132 }
 133
 134 // -------------------- //
 135 //    bits.OnesCount    //
 136 // -------------------- //
 137
 138 // TODO(register args) Restore a m d 6 4 / v 1 :.*x86HasPOPCNT when only one ABI is tested.
 139 func OnesCount(n uint) int {
 140         // amd64/v2:-".*x86HasPOPCNT" amd64/v3:-".*x86HasPOPCNT"
 141         // amd64:"POPCNTQ"
 142         // arm64:"VCNT","VUADDLV"
 143         // s390x:"POPCNT"
 144         // ppc64x:"POPCNTD"
 145         // wasm:"I64Popcnt"
 146         return bits.OnesCount(n)
 147 }
 148
 149 func OnesCount64(n uint64) int {
 150         // amd64/v2:-".*x86HasPOPCNT" amd64/v3:-".*x86HasPOPCNT"
 151         // amd64:"POPCNTQ"
 152         // arm64:"VCNT","VUADDLV"
 153         // s390x:"POPCNT"
 154         // ppc64x:"POPCNTD"
 155         // wasm:"I64Popcnt"
 156         return bits.OnesCount64(n)
 157 }
 158
 159 func OnesCount32(n uint32) int {
 160         // amd64/v2:-".*x86HasPOPCNT" amd64/v3:-".*x86HasPOPCNT"
 161         // amd64:"POPCNTL"
 162         // arm64:"VCNT","VUADDLV"
 163         // s390x:"POPCNT"
 164         // ppc64x:"POPCNTW"
 165         // wasm:"I64Popcnt"
 166         return bits.OnesCount32(n)
 167 }
 168
 169 func OnesCount16(n uint16) int {
 170         // amd64/v2:-".*x86HasPOPCNT" amd64/v3:-".*x86HasPOPCNT"
 171         // amd64:"POPCNTL"
 172         // arm64:"VCNT","VUADDLV"
 173         // s390x:"POPCNT"
 174         // ppc64x:"POPCNTW"
 175         // wasm:"I64Popcnt"
 176         return bits.OnesCount16(n)
 177 }
 178
 179 func OnesCount8(n uint8) int {
 180         // s390x:"POPCNT"
 181         // ppc64x:"POPCNTB"
 182         // wasm:"I64Popcnt"
 183         return bits.OnesCount8(n)
 184 }
 185
 186 // ----------------------- //
 187 //    bits.ReverseBytes    //
 188 // ----------------------- //
 189
 190 func ReverseBytes(n uint) uint {
 191         // amd64:"BSWAPQ"
 192         // s390x:"MOVDBR"
 193         // arm64:"REV"
 194         return bits.ReverseBytes(n)
 195 }
 196
 197 func ReverseBytes64(n uint64) uint64 {
 198         // amd64:"BSWAPQ"
 199         // s390x:"MOVDBR"
 200         // arm64:"REV"
 201         // ppc64x/power10: "BRD"
 202         return bits.ReverseBytes64(n)
 203 }
 204
 205 func ReverseBytes32(n uint32) uint32 {
 206         // amd64:"BSWAPL"
 207         // s390x:"MOVWBR"
 208         // arm64:"REVW"
 209         // ppc64x/power10: "BRW"
 210         return bits.ReverseBytes32(n)
 211 }
 212
 213 func ReverseBytes16(n uint16) uint16 {
 214         // amd64:"ROLW"
 215         // arm64:"REV16W",-"UBFX",-"ORR"
 216         // arm/5:"SLL","SRL","ORR"
 217         // arm/6:"REV16"
 218         // arm/7:"REV16"
 219         // ppc64x/power10: "BRH"
 220         return bits.ReverseBytes16(n)
 221 }
 222
 223 // --------------------- //
 224 //    bits.RotateLeft    //
 225 // --------------------- //
 226
 227 func RotateLeft64(n uint64) uint64 {
 228         // amd64:"ROLQ"
 229         // arm64:"ROR"
 230         // ppc64x:"ROTL"
 231         // s390x:"RISBGZ\t[$]0, [$]63, [$]37, "
 232         // wasm:"I64Rotl"
 233         return bits.RotateLeft64(n, 37)
 234 }
 235
 236 func RotateLeft32(n uint32) uint32 {
 237         // amd64:"ROLL" 386:"ROLL"
 238         // arm:`MOVW\tR[0-9]+@>23`
 239         // arm64:"RORW"
 240         // ppc64x:"ROTLW"
 241         // s390x:"RLL"
 242         // wasm:"I32Rotl"
 243         return bits.RotateLeft32(n, 9)
 244 }
 245
 246 func RotateLeft16(n uint16, s int) uint16 {
 247         // amd64:"ROLW" 386:"ROLW"
 248         // arm64:"RORW",-"CSEL"
 249         return bits.RotateLeft16(n, s)
 250 }
 251
 252 func RotateLeft8(n uint8, s int) uint8 {
 253         // amd64:"ROLB" 386:"ROLB"
 254         // arm64:"LSL","LSR",-"CSEL"
 255         return bits.RotateLeft8(n, s)
 256 }
 257
 258 func RotateLeftVariable(n uint, m int) uint {
 259         // amd64:"ROLQ"
 260         // arm64:"ROR"
 261         // ppc64x:"ROTL"
 262         // s390x:"RLLG"
 263         // wasm:"I64Rotl"
 264         return bits.RotateLeft(n, m)
 265 }
 266
 267 func RotateLeftVariable64(n uint64, m int) uint64 {
 268         // amd64:"ROLQ"
 269         // arm64:"ROR"
 270         // ppc64x:"ROTL"
 271         // s390x:"RLLG"
 272         // wasm:"I64Rotl"
 273         return bits.RotateLeft64(n, m)
 274 }
 275
 276 func RotateLeftVariable32(n uint32, m int) uint32 {
 277         // arm:`MOVW\tR[0-9]+@>R[0-9]+`
 278         // amd64:"ROLL"
 279         // arm64:"RORW"
 280         // ppc64x:"ROTLW"
 281         // s390x:"RLL"
 282         // wasm:"I32Rotl"
 283         return bits.RotateLeft32(n, m)
 284 }
 285
 286 // ------------------------ //
 287 //    bits.TrailingZeros    //
 288 // ------------------------ //
 289
 290 func TrailingZeros(n uint) int {
 291         // amd64/v1,amd64/v2:"BSFQ","MOVL\t\\$64","CMOVQEQ"
 292         // amd64/v3:"TZCNTQ"
 293         // arm:"CLZ"
 294         // arm64:"RBIT","CLZ"
 295         // s390x:"FLOGR"
 296         // ppc64x/power8:"ANDN","POPCNTD"
 297         // ppc64x/power9: "CNTTZD"
 298         // wasm:"I64Ctz"
 299         return bits.TrailingZeros(n)
 300 }
 301
 302 func TrailingZeros64(n uint64) int {
 303         // amd64/v1,amd64/v2:"BSFQ","MOVL\t\\$64","CMOVQEQ"
 304         // amd64/v3:"TZCNTQ"
 305         // arm64:"RBIT","CLZ"
 306         // s390x:"FLOGR"
 307         // ppc64x/power8:"ANDN","POPCNTD"
 308         // ppc64x/power9: "CNTTZD"
 309         // wasm:"I64Ctz"
 310         return bits.TrailingZeros64(n)
 311 }
 312
 313 func TrailingZeros64Subtract(n uint64) int {
 314         // ppc64x/power8:"NEG","SUBC","ANDN","POPCNTD"
 315         // ppc64x/power9:"SUBC","CNTTZD"
 316         return bits.TrailingZeros64(1 - n)
 317 }
 318
 319 func TrailingZeros32(n uint32) int {
 320         // amd64/v1,amd64/v2:"BTSQ\\t\\$32","BSFQ"
 321         // amd64/v3:"TZCNTL"
 322         // arm:"CLZ"
 323         // arm64:"RBITW","CLZW"
 324         // s390x:"FLOGR","MOVWZ"
 325         // ppc64x/power8:"ANDN","POPCNTW"
 326         // ppc64x/power9: "CNTTZW"
 327         // wasm:"I64Ctz"
 328         return bits.TrailingZeros32(n)
 329 }
 330
 331 func TrailingZeros16(n uint16) int {
 332         // amd64:"BSFL","BTSL\\t\\$16"
 333         // 386:"BSFL\t"
 334         // arm:"ORR\t\\$65536","CLZ",-"MOVHU\tR"
 335         // arm64:"ORR\t\\$65536","RBITW","CLZW",-"MOVHU\tR",-"RBIT\t",-"CLZ\t"
 336         // s390x:"FLOGR","OR\t\\$65536"
 337         // ppc64x/power8:"POPCNTD","OR\\t\\$65536"
 338         // ppc64x/power9:"CNTTZD","OR\\t\\$65536"
 339         // wasm:"I64Ctz"
 340         return bits.TrailingZeros16(n)
 341 }
 342
 343 func TrailingZeros8(n uint8) int {
 344         // amd64:"BSFL","BTSL\\t\\$8"
 345         // arm:"ORR\t\\$256","CLZ",-"MOVBU\tR"
 346         // arm64:"ORR\t\\$256","RBITW","CLZW",-"MOVBU\tR",-"RBIT\t",-"CLZ\t"
 347         // s390x:"FLOGR","OR\t\\$256"
 348         // wasm:"I64Ctz"
 349         return bits.TrailingZeros8(n)
 350 }
 351
 352 // IterateBitsNN checks special handling of TrailingZerosNN when the input is known to be non-zero.
 353
 354 func IterateBits(n uint) int {
 355         i := 0
 356         for n != 0 {
 357                 // amd64/v1,amd64/v2:"BSFQ",-"CMOVEQ"
 358                 // amd64/v3:"TZCNTQ"
 359                 i += bits.TrailingZeros(n)
 360                 n &= n - 1
 361         }
 362         return i
 363 }
 364
 365 func IterateBits64(n uint64) int {
 366         i := 0
 367         for n != 0 {
 368                 // amd64/v1,amd64/v2:"BSFQ",-"CMOVEQ"
 369                 // amd64/v3:"TZCNTQ"
 370                 i += bits.TrailingZeros64(n)
 371                 n &= n - 1
 372         }
 373         return i
 374 }
 375
 376 func IterateBits32(n uint32) int {
 377         i := 0
 378         for n != 0 {
 379                 // amd64/v1,amd64/v2:"BSFL",-"BTSQ"
 380                 // amd64/v3:"TZCNTL"
 381                 i += bits.TrailingZeros32(n)
 382                 n &= n - 1
 383         }
 384         return i
 385 }
 386
 387 func IterateBits16(n uint16) int {
 388         i := 0
 389         for n != 0 {
 390                 // amd64/v1,amd64/v2:"BSFL",-"BTSL"
 391                 // amd64/v3:"TZCNTL"
 392                 // arm64:"RBITW","CLZW",-"ORR"
 393                 i += bits.TrailingZeros16(n)
 394                 n &= n - 1
 395         }
 396         return i
 397 }
 398
 399 func IterateBits8(n uint8) int {
 400         i := 0
 401         for n != 0 {
 402                 // amd64/v1,amd64/v2:"BSFL",-"BTSL"
 403                 // amd64/v3:"TZCNTL"
 404                 // arm64:"RBITW","CLZW",-"ORR"
 405                 i += bits.TrailingZeros8(n)
 406                 n &= n - 1
 407         }
 408         return i
 409 }
 410
 411 // --------------- //
 412 //    bits.Add*    //
 413 // --------------- //
 414
 415 func Add(x, y, ci uint) (r, co uint) {
 416         // arm64:"ADDS","ADCS","ADC",-"ADD\t",-"CMP"
 417         // amd64:"NEGL","ADCQ","SBBQ","NEGQ"
 418         // ppc64x: "ADDC", "ADDE", "ADDZE"
 419         // s390x:"ADDE","ADDC\t[$]-1,"
 420         // riscv64: "ADD","SLTU"
 421         return bits.Add(x, y, ci)
 422 }
 423
 424 func AddC(x, ci uint) (r, co uint) {
 425         // arm64:"ADDS","ADCS","ADC",-"ADD\t",-"CMP"
 426         // amd64:"NEGL","ADCQ","SBBQ","NEGQ"
 427         // loong64: "ADDV", "SGTU"
 428         // ppc64x: "ADDC", "ADDE", "ADDZE"
 429         // s390x:"ADDE","ADDC\t[$]-1,"
 430         // riscv64: "ADD","SLTU"
 431         return bits.Add(x, 7, ci)
 432 }
 433
 434 func AddZ(x, y uint) (r, co uint) {
 435         // arm64:"ADDS","ADC",-"ADCS",-"ADD\t",-"CMP"
 436         // amd64:"ADDQ","SBBQ","NEGQ",-"NEGL",-"ADCQ"
 437         // loong64: "ADDV", "SGTU"
 438         // ppc64x: "ADDC", -"ADDE", "ADDZE"
 439         // s390x:"ADDC",-"ADDC\t[$]-1,"
 440         // riscv64: "ADD","SLTU"
 441         return bits.Add(x, y, 0)
 442 }
 443
 444 func AddR(x, y, ci uint) uint {
 445         // arm64:"ADDS","ADCS",-"ADD\t",-"CMP"
 446         // amd64:"NEGL","ADCQ",-"SBBQ",-"NEGQ"
 447         // loong64: "ADDV", -"SGTU"
 448         // ppc64x: "ADDC", "ADDE", -"ADDZE"
 449         // s390x:"ADDE","ADDC\t[$]-1,"
 450         // riscv64: "ADD",-"SLTU"
 451         r, _ := bits.Add(x, y, ci)
 452         return r
 453 }
 454
 455 func AddM(p, q, r *[3]uint) {
 456         var c uint
 457         r[0], c = bits.Add(p[0], q[0], c)
 458         // arm64:"ADCS",-"ADD\t",-"CMP"
 459         // amd64:"ADCQ",-"NEGL",-"SBBQ",-"NEGQ"
 460         // s390x:"ADDE",-"ADDC\t[$]-1,"
 461         r[1], c = bits.Add(p[1], q[1], c)
 462         r[2], c = bits.Add(p[2], q[2], c)
 463 }
 464
 465 func Add64(x, y, ci uint64) (r, co uint64) {
 466         // arm64:"ADDS","ADCS","ADC",-"ADD\t",-"CMP"
 467         // amd64:"NEGL","ADCQ","SBBQ","NEGQ"
 468         // loong64: "ADDV", "SGTU"
 469         // ppc64x: "ADDC", "ADDE", "ADDZE"
 470         // s390x:"ADDE","ADDC\t[$]-1,"
 471         // riscv64: "ADD","SLTU"
 472         return bits.Add64(x, y, ci)
 473 }
 474
 475 func Add64C(x, ci uint64) (r, co uint64) {
 476         // arm64:"ADDS","ADCS","ADC",-"ADD\t",-"CMP"
 477         // amd64:"NEGL","ADCQ","SBBQ","NEGQ"
 478         // loong64: "ADDV", "SGTU"
 479         // ppc64x: "ADDC", "ADDE", "ADDZE"
 480         // s390x:"ADDE","ADDC\t[$]-1,"
 481         // riscv64: "ADD","SLTU"
 482         return bits.Add64(x, 7, ci)
 483 }
 484
 485 func Add64Z(x, y uint64) (r, co uint64) {
 486         // arm64:"ADDS","ADC",-"ADCS",-"ADD\t",-"CMP"
 487         // amd64:"ADDQ","SBBQ","NEGQ",-"NEGL",-"ADCQ"
 488         // loong64: "ADDV", "SGTU"
 489         // ppc64x: "ADDC", -"ADDE", "ADDZE"
 490         // s390x:"ADDC",-"ADDC\t[$]-1,"
 491         // riscv64: "ADD","SLTU"
 492         return bits.Add64(x, y, 0)
 493 }
 494
 495 func Add64R(x, y, ci uint64) uint64 {
 496         // arm64:"ADDS","ADCS",-"ADD\t",-"CMP"
 497         // amd64:"NEGL","ADCQ",-"SBBQ",-"NEGQ"
 498         // loong64: "ADDV", -"SGTU"
 499         // ppc64x: "ADDC", "ADDE", -"ADDZE"
 500         // s390x:"ADDE","ADDC\t[$]-1,"
 501         // riscv64: "ADD",-"SLTU"
 502         r, _ := bits.Add64(x, y, ci)
 503         return r
 504 }
 505 func Add64M(p, q, r *[3]uint64) {
 506         var c uint64
 507         r[0], c = bits.Add64(p[0], q[0], c)
 508         // arm64:"ADCS",-"ADD\t",-"CMP"
 509         // amd64:"ADCQ",-"NEGL",-"SBBQ",-"NEGQ"
 510         // ppc64x: -"ADDC", "ADDE", -"ADDZE"
 511         // s390x:"ADDE",-"ADDC\t[$]-1,"
 512         r[1], c = bits.Add64(p[1], q[1], c)
 513         r[2], c = bits.Add64(p[2], q[2], c)
 514 }
 515
 516 func Add64MSaveC(p, q, r, c *[2]uint64) {
 517         // ppc64x: "ADDC\tR", "ADDZE"
 518         r[0], c[0] = bits.Add64(p[0], q[0], 0)
 519         // ppc64x: "ADDC\t[$]-1", "ADDE", "ADDZE"
 520         r[1], c[1] = bits.Add64(p[1], q[1], c[0])
 521 }
 522
 523 func Add64PanicOnOverflowEQ(a, b uint64) uint64 {
 524         r, c := bits.Add64(a, b, 0)
 525         // s390x:"BRC\t[$]3,",-"ADDE"
 526         if c == 1 {
 527                 panic("overflow")
 528         }
 529         return r
 530 }
 531
 532 func Add64PanicOnOverflowNE(a, b uint64) uint64 {
 533         r, c := bits.Add64(a, b, 0)
 534         // s390x:"BRC\t[$]3,",-"ADDE"
 535         if c != 0 {
 536                 panic("overflow")
 537         }
 538         return r
 539 }
 540
 541 func Add64PanicOnOverflowGT(a, b uint64) uint64 {
 542         r, c := bits.Add64(a, b, 0)
 543         // s390x:"BRC\t[$]3,",-"ADDE"
 544         if c > 0 {
 545                 panic("overflow")
 546         }
 547         return r
 548 }
 549
 550 func Add64MPanicOnOverflowEQ(a, b [2]uint64) [2]uint64 {
 551         var r [2]uint64
 552         var c uint64
 553         r[0], c = bits.Add64(a[0], b[0], c)
 554         r[1], c = bits.Add64(a[1], b[1], c)
 555         // s390x:"BRC\t[$]3,"
 556         if c == 1 {
 557                 panic("overflow")
 558         }
 559         return r
 560 }
 561
 562 func Add64MPanicOnOverflowNE(a, b [2]uint64) [2]uint64 {
 563         var r [2]uint64
 564         var c uint64
 565         r[0], c = bits.Add64(a[0], b[0], c)
 566         r[1], c = bits.Add64(a[1], b[1], c)
 567         // s390x:"BRC\t[$]3,"
 568         if c != 0 {
 569                 panic("overflow")
 570         }
 571         return r
 572 }
 573
 574 func Add64MPanicOnOverflowGT(a, b [2]uint64) [2]uint64 {
 575         var r [2]uint64
 576         var c uint64
 577         r[0], c = bits.Add64(a[0], b[0], c)
 578         r[1], c = bits.Add64(a[1], b[1], c)
 579         // s390x:"BRC\t[$]3,"
 580         if c > 0 {
 581                 panic("overflow")
 582         }
 583         return r
 584 }
 585
 586 // Verify independent carry chain operations are scheduled efficiently
 587 // and do not cause unnecessary save/restore of the CA bit.
 588 //
 589 // This is an example of why CarryChainTail priority must be lower
 590 // (earlier in the block) than Memory. f[0]=f1 could be scheduled
 591 // after the first two lower 64 bit limb adds, but before either
 592 // high 64 bit limbs are added.
 593 //
 594 // This is what happened on PPC64 when compiling
 595 // crypto/internal/edwards25519/field.feMulGeneric.
 596 func Add64MultipleChains(a, b, c, d [2]uint64) {
 597         var cx, d1, d2 uint64
 598         a1, a2 := a[0], a[1]
 599         b1, b2 := b[0], b[1]
 600         c1, c2 := c[0], c[1]
 601
 602         // ppc64x: "ADDC\tR\\d+,", -"ADDE", -"MOVD\tXER"
 603         d1, cx = bits.Add64(a1, b1, 0)
 604         // ppc64x: "ADDE", -"ADDC", -"MOVD\t.*, XER"
 605         d2, _ = bits.Add64(a2, b2, cx)
 606
 607         // ppc64x: "ADDC\tR\\d+,", -"ADDE", -"MOVD\tXER"
 608         d1, cx = bits.Add64(c1, d1, 0)
 609         // ppc64x: "ADDE", -"ADDC", -"MOVD\t.*, XER"
 610         d2, _ = bits.Add64(c2, d2, cx)
 611         d[0] = d1
 612         d[1] = d2
 613 }
 614
 615 // --------------- //
 616 //    bits.Sub*    //
 617 // --------------- //
 618
 619 func Sub(x, y, ci uint) (r, co uint) {
 620         // amd64:"NEGL","SBBQ","NEGQ"
 621         // arm64:"NEGS","SBCS","NGC","NEG",-"ADD",-"SUB",-"CMP"
 622         // loong64:"SUBV","SGTU"
 623         // ppc64x:"SUBC", "SUBE", "SUBZE", "NEG"
 624         // s390x:"SUBE"
 625         // riscv64: "SUB","SLTU"
 626         return bits.Sub(x, y, ci)
 627 }
 628
 629 func SubC(x, ci uint) (r, co uint) {
 630         // amd64:"NEGL","SBBQ","NEGQ"
 631         // arm64:"NEGS","SBCS","NGC","NEG",-"ADD",-"SUB",-"CMP"
 632         // loong64:"SUBV","SGTU"
 633         // ppc64x:"SUBC", "SUBE", "SUBZE", "NEG"
 634         // s390x:"SUBE"
 635         // riscv64: "SUB","SLTU"
 636         return bits.Sub(x, 7, ci)
 637 }
 638
 639 func SubZ(x, y uint) (r, co uint) {
 640         // amd64:"SUBQ","SBBQ","NEGQ",-"NEGL"
 641         // arm64:"SUBS","NGC","NEG",-"SBCS",-"ADD",-"SUB\t",-"CMP"
 642         // loong64:"SUBV","SGTU"
 643         // ppc64x:"SUBC", -"SUBE", "SUBZE", "NEG"
 644         // s390x:"SUBC"
 645         // riscv64: "SUB","SLTU"
 646         return bits.Sub(x, y, 0)
 647 }
 648
 649 func SubR(x, y, ci uint) uint {
 650         // amd64:"NEGL","SBBQ",-"NEGQ"
 651         // arm64:"NEGS","SBCS",-"NGC",-"NEG\t",-"ADD",-"SUB",-"CMP"
 652         // loong64:"SUBV",-"SGTU"
 653         // ppc64x:"SUBC", "SUBE", -"SUBZE", -"NEG"
 654         // s390x:"SUBE"
 655         // riscv64: "SUB",-"SLTU"
 656         r, _ := bits.Sub(x, y, ci)
 657         return r
 658 }
 659 func SubM(p, q, r *[3]uint) {
 660         var c uint
 661         r[0], c = bits.Sub(p[0], q[0], c)
 662         // amd64:"SBBQ",-"NEGL",-"NEGQ"
 663         // arm64:"SBCS",-"NEGS",-"NGC",-"NEG",-"ADD",-"SUB",-"CMP"
 664         // ppc64x:-"SUBC", "SUBE", -"SUBZE", -"NEG"
 665         // s390x:"SUBE"
 666         r[1], c = bits.Sub(p[1], q[1], c)
 667         r[2], c = bits.Sub(p[2], q[2], c)
 668 }
 669
 670 func Sub64(x, y, ci uint64) (r, co uint64) {
 671         // amd64:"NEGL","SBBQ","NEGQ"
 672         // arm64:"NEGS","SBCS","NGC","NEG",-"ADD",-"SUB",-"CMP"
 673         // loong64:"SUBV","SGTU"
 674         // ppc64x:"SUBC", "SUBE", "SUBZE", "NEG"
 675         // s390x:"SUBE"
 676         // riscv64: "SUB","SLTU"
 677         return bits.Sub64(x, y, ci)
 678 }
 679
 680 func Sub64C(x, ci uint64) (r, co uint64) {
 681         // amd64:"NEGL","SBBQ","NEGQ"
 682         // arm64:"NEGS","SBCS","NGC","NEG",-"ADD",-"SUB",-"CMP"
 683         // loong64:"SUBV","SGTU"
 684         // ppc64x:"SUBC", "SUBE", "SUBZE", "NEG"
 685         // s390x:"SUBE"
 686         // riscv64: "SUB","SLTU"
 687         return bits.Sub64(x, 7, ci)
 688 }
 689
 690 func Sub64Z(x, y uint64) (r, co uint64) {
 691         // amd64:"SUBQ","SBBQ","NEGQ",-"NEGL"
 692         // arm64:"SUBS","NGC","NEG",-"SBCS",-"ADD",-"SUB\t",-"CMP"
 693         // loong64:"SUBV","SGTU"
 694         // ppc64x:"SUBC", -"SUBE", "SUBZE", "NEG"
 695         // s390x:"SUBC"
 696         // riscv64: "SUB","SLTU"
 697         return bits.Sub64(x, y, 0)
 698 }
 699
 700 func Sub64R(x, y, ci uint64) uint64 {
 701         // amd64:"NEGL","SBBQ",-"NEGQ"
 702         // arm64:"NEGS","SBCS",-"NGC",-"NEG\t",-"ADD",-"SUB",-"CMP"
 703         // loong64:"SUBV",-"SGTU"
 704         // ppc64x:"SUBC", "SUBE", -"SUBZE", -"NEG"
 705         // s390x:"SUBE"
 706         // riscv64: "SUB",-"SLTU"
 707         r, _ := bits.Sub64(x, y, ci)
 708         return r
 709 }
 710 func Sub64M(p, q, r *[3]uint64) {
 711         var c uint64
 712         r[0], c = bits.Sub64(p[0], q[0], c)
 713         // amd64:"SBBQ",-"NEGL",-"NEGQ"
 714         // arm64:"SBCS",-"NEGS",-"NGC",-"NEG",-"ADD",-"SUB",-"CMP"
 715         // s390x:"SUBE"
 716         r[1], c = bits.Sub64(p[1], q[1], c)
 717         r[2], c = bits.Sub64(p[2], q[2], c)
 718 }
 719
 720 func Sub64MSaveC(p, q, r, c *[2]uint64) {
 721         // ppc64x:"SUBC\tR\\d+, R\\d+,", "SUBZE", "NEG"
 722         r[0], c[0] = bits.Sub64(p[0], q[0], 0)
 723         // ppc64x:"SUBC\tR\\d+, [$]0,", "SUBE", "SUBZE", "NEG"
 724         r[1], c[1] = bits.Sub64(p[1], q[1], c[0])
 725 }
 726
 727 func Sub64PanicOnOverflowEQ(a, b uint64) uint64 {
 728         r, b := bits.Sub64(a, b, 0)
 729         // s390x:"BRC\t[$]12,",-"ADDE",-"SUBE"
 730         if b == 1 {
 731                 panic("overflow")
 732         }
 733         return r
 734 }
 735
 736 func Sub64PanicOnOverflowNE(a, b uint64) uint64 {
 737         r, b := bits.Sub64(a, b, 0)
 738         // s390x:"BRC\t[$]12,",-"ADDE",-"SUBE"
 739         if b != 0 {
 740                 panic("overflow")
 741         }
 742         return r
 743 }
 744
 745 func Sub64PanicOnOverflowGT(a, b uint64) uint64 {
 746         r, b := bits.Sub64(a, b, 0)
 747         // s390x:"BRC\t[$]12,",-"ADDE",-"SUBE"
 748         if b > 0 {
 749                 panic("overflow")
 750         }
 751         return r
 752 }
 753
 754 func Sub64MPanicOnOverflowEQ(a, b [2]uint64) [2]uint64 {
 755         var r [2]uint64
 756         var c uint64
 757         r[0], c = bits.Sub64(a[0], b[0], c)
 758         r[1], c = bits.Sub64(a[1], b[1], c)
 759         // s390x:"BRC\t[$]12,"
 760         if c == 1 {
 761                 panic("overflow")
 762         }
 763         return r
 764 }
 765
 766 func Sub64MPanicOnOverflowNE(a, b [2]uint64) [2]uint64 {
 767         var r [2]uint64
 768         var c uint64
 769         r[0], c = bits.Sub64(a[0], b[0], c)
 770         r[1], c = bits.Sub64(a[1], b[1], c)
 771         // s390x:"BRC\t[$]12,"
 772         if c != 0 {
 773                 panic("overflow")
 774         }
 775         return r
 776 }
 777
 778 func Sub64MPanicOnOverflowGT(a, b [2]uint64) [2]uint64 {
 779         var r [2]uint64
 780         var c uint64
 781         r[0], c = bits.Sub64(a[0], b[0], c)
 782         r[1], c = bits.Sub64(a[1], b[1], c)
 783         // s390x:"BRC\t[$]12,"
 784         if c > 0 {
 785                 panic("overflow")
 786         }
 787         return r
 788 }
 789
 790 // --------------- //
 791 //    bits.Mul*    //
 792 // --------------- //
 793
 794 func Mul(x, y uint) (hi, lo uint) {
 795         // amd64:"MULQ"
 796         // arm64:"UMULH","MUL"
 797         // ppc64x:"MULHDU","MULLD"
 798         // s390x:"MLGR"
 799         // mips64: "MULVU"
 800         return bits.Mul(x, y)
 801 }
 802
 803 func Mul64(x, y uint64) (hi, lo uint64) {
 804         // amd64:"MULQ"
 805         // arm64:"UMULH","MUL"
 806         // ppc64x:"MULHDU","MULLD"
 807         // s390x:"MLGR"
 808         // mips64: "MULVU"
 809         // riscv64:"MULHU","MUL"
 810         return bits.Mul64(x, y)
 811 }
 812
 813 func Mul64HiOnly(x, y uint64) uint64 {
 814         // arm64:"UMULH",-"MUL"
 815         // riscv64:"MULHU",-"MUL\t"
 816         hi, _ := bits.Mul64(x, y)
 817         return hi
 818 }
 819
 820 func Mul64LoOnly(x, y uint64) uint64 {
 821         // arm64:"MUL",-"UMULH"
 822         // riscv64:"MUL\t",-"MULHU"
 823         _, lo := bits.Mul64(x, y)
 824         return lo
 825 }
 826
 827 // --------------- //
 828 //    bits.Div*    //
 829 // --------------- //
 830
 831 func Div(hi, lo, x uint) (q, r uint) {
 832         // amd64:"DIVQ"
 833         return bits.Div(hi, lo, x)
 834 }
 835
 836 func Div32(hi, lo, x uint32) (q, r uint32) {
 837         // arm64:"ORR","UDIV","MSUB",-"UREM"
 838         return bits.Div32(hi, lo, x)
 839 }
 840
 841 func Div64(hi, lo, x uint64) (q, r uint64) {
 842         // amd64:"DIVQ"
 843         return bits.Div64(hi, lo, x)
 844 }
 845
 846 func Div64degenerate(x uint64) (q, r uint64) {
 847         // amd64:-"DIVQ"
 848         return bits.Div64(0, x, 5)
 849 }