nettrash-a 0.0.28

//! The aarch64 kernels of the curves' fields, in inline assembly. **This file is generated by
//! `kernels.py`, next to it: change that, not this.**
//!
//! Up to six limbs a kernel takes its operands and gives its result in registers, and is inlined,
//! so that a curve's formulas keep their elements in registers from one operation to the next:
//! measured on an Apple M4 Max, a product of P-256 took 33 cycles after the one before, an addition
//! 9 and a doubling 272, against 48, 22 and 357 with kernels that read their operands and wrote
//! their result in memory; OpenSSL's assembly takes 38 for the product and 257 for the doubling. The
//! compiler chooses the registers, as fixed ones left it none for the kernels of numbers' products
//! once both were inlined into one function.
//!
//! A row of a product is two carry chains, the low limbs of the row's products and then the high
//! ones one position up, each `mul` or `umulh` next to the addition that takes it, so that one
//! register holds them all and the carry flag goes through the row, as `mul` and `umulh` leave
//! it alone.
//!
//! A prime `2^k - c` folds a carry out of a sum, or the limb above a product's fold, back as a
//! product by `c`, `f = 2^(64 n)` modulo the prime, which carries out of the low limbs about once in
//! `2^64 / f`: that case goes to a branch around the end of the kernel, so that the other limbs are
//! done after one chain. An addition on secp256k1 so took 7 cycles after the one before against 12,
//! a subtraction of nine limbs 13 against 42, and a product and a square of Curve25519's field 31
//! and 29 against 37 and 33.
//!
//! - [`pseudo_mul_4`], [`pseudo_sqr_4`]: the eight limbs of the product, then the high half folded
//!   onto the low one by a product by `f`, for a prime with `2^256 = f`.
//! - [`p256_mul`], [`p256_sqr`]: Montgomery's reduction modulo P-256's prime `p`, where
//!   `-1 / p = 1` modulo `2^64`: the quotient of a round is the low limb `q` itself and `q p` is
//!   made of shifts of it, so that a round takes no product. With `l`, `h` the two limbs of
//!   `q 2^32`, a round adds `l` at the limb above and `h` above that, and the two limbs of
//!   `q (2^64 + 1) - q 2^32` above those.
//! - [`p384_mul`], [`p384_sqr`]: the same for P-384's prime, where `-1 / p = 2^32 + 1`, so that a
//!   round's `q` is a shift and an addition, and `q p = q 2^384 - q 2^128 - q 2^96 + q 2^32 - q`.
//! - [`pseudo_add_4`] and its like fold a carry back as `f`; [`mont_add_4`] and its like compare
//!   with the prime, and their subtractions take the difference or the difference plus the prime,
//!   the second chain a step behind the first, by the borrow.

use core::arch::asm;

/// `a b` modulo the prime with `2^256 = f`, `f < 2^62`, not reduced below it.
#[inline(always)]
pub(super) fn pseudo_mul_4(a: &[u64; 4], b: &[u64; 4], f: u64) -> [u64; 4] {
    let mut r = [0; 4];
    // SAFETY: the block reads and writes registers only.
    unsafe {
        asm!(
            "mul {x8}, {x4}, {x0}",
            "umulh {x9}, {x4}, {x0}",
            "mul {x17}, {x5}, {x0}",
            "umulh {x10}, {x5}, {x0}",
            "adds {x9}, {x9}, {x17}",
            "mul {x17}, {x6}, {x0}",
            "umulh {x11}, {x6}, {x0}",
            "adcs {x10}, {x10}, {x17}",
            "mul {x17}, {x7}, {x0}",
            "umulh {x12}, {x7}, {x0}",
            "adcs {x11}, {x11}, {x17}",
            "adc {x12}, {x12}, xzr",
            "mul {x17}, {x4}, {x1}",
            "adds {x9}, {x9}, {x17}",
            "mul {x17}, {x5}, {x1}",
            "adcs {x10}, {x10}, {x17}",
            "mul {x17}, {x6}, {x1}",
            "adcs {x11}, {x11}, {x17}",
            "mul {x17}, {x7}, {x1}",
            "adcs {x12}, {x12}, {x17}",
            "cset {x13}, cs",
            "umulh {x17}, {x4}, {x1}",
            "adds {x10}, {x10}, {x17}",
            "umulh {x17}, {x5}, {x1}",
            "adcs {x11}, {x11}, {x17}",
            "umulh {x17}, {x6}, {x1}",
            "adcs {x12}, {x12}, {x17}",
            "umulh {x17}, {x7}, {x1}",
            "adc {x13}, {x13}, {x17}",
            "mul {x17}, {x4}, {x2}",
            "adds {x10}, {x10}, {x17}",
            "mul {x17}, {x5}, {x2}",
            "adcs {x11}, {x11}, {x17}",
            "mul {x17}, {x6}, {x2}",
            "adcs {x12}, {x12}, {x17}",
            "mul {x17}, {x7}, {x2}",
            "adcs {x13}, {x13}, {x17}",
            "cset {x14}, cs",
            "umulh {x17}, {x4}, {x2}",
            "adds {x11}, {x11}, {x17}",
            "umulh {x17}, {x5}, {x2}",
            "adcs {x12}, {x12}, {x17}",
            "umulh {x17}, {x6}, {x2}",
            "adcs {x13}, {x13}, {x17}",
            "umulh {x17}, {x7}, {x2}",
            "adc {x14}, {x14}, {x17}",
            "mul {x17}, {x4}, {x20}",
            "adds {x11}, {x11}, {x17}",
            "mul {x17}, {x5}, {x20}",
            "adcs {x12}, {x12}, {x17}",
            "mul {x17}, {x6}, {x20}",
            "adcs {x13}, {x13}, {x17}",
            "mul {x17}, {x7}, {x20}",
            "adcs {x14}, {x14}, {x17}",
            "cset {x15}, cs",
            "umulh {x17}, {x4}, {x20}",
            "adds {x12}, {x12}, {x17}",
            "umulh {x17}, {x5}, {x20}",
            "adcs {x13}, {x13}, {x17}",
            "umulh {x17}, {x6}, {x20}",
            "adcs {x14}, {x14}, {x17}",
            "umulh {x17}, {x7}, {x20}",
            "adc {x15}, {x15}, {x17}",
            "mul {x17}, {x12}, {x3}",
            "adds {x8}, {x8}, {x17}",
            "mul {x17}, {x13}, {x3}",
            "adcs {x9}, {x9}, {x17}",
            "mul {x17}, {x14}, {x3}",
            "adcs {x10}, {x10}, {x17}",
            "mul {x17}, {x15}, {x3}",
            "adcs {x11}, {x11}, {x17}",
            "cset {x1}, cs",
            "umulh {x17}, {x12}, {x3}",
            "adds {x9}, {x9}, {x17}",
            "umulh {x17}, {x13}, {x3}",
            "adcs {x10}, {x10}, {x17}",
            "umulh {x17}, {x14}, {x3}",
            "adcs {x11}, {x11}, {x17}",
            "umulh {x17}, {x15}, {x3}",
            "adc {x1}, {x1}, {x17}",
            "mul {x17}, {x1}, {x3}",
            "umulh {x16}, {x1}, {x3}",
            "adds {x8}, {x8}, {x17}",
            "adcs {x9}, {x9}, {x16}",
            "b.cc 2f",
            "adcs {x10}, {x10}, xzr",
            "adcs {x11}, {x11}, xzr",
            "csel {x17}, {x3}, xzr, cs",
            "adds {x8}, {x8}, {x17}",
            "adc {x9}, {x9}, xzr",
            "2:",
            x0 = in(reg) b[0],
            x1 = inout(reg) b[1] => _,
            x2 = in(reg) b[2],
            x3 = in(reg) f,
            x4 = in(reg) a[0],
            x5 = in(reg) a[1],
            x6 = in(reg) a[2],
            x7 = in(reg) a[3],
            x8 = out(reg) r[0],
            x9 = out(reg) r[1],
            x10 = out(reg) r[2],
            x11 = out(reg) r[3],
            x12 = out(reg) _,
            x13 = out(reg) _,
            x14 = out(reg) _,
            x15 = out(reg) _,
            x16 = out(reg) _,
            x17 = out(reg) _,
            x20 = in(reg) b[3],
            options(pure, nomem, nostack),
        );
    }
    r
}

/// `a^2` modulo the prime with `2^256 = f`, `f < 2^62`, not reduced below it.
#[inline(always)]
pub(super) fn pseudo_sqr_4(a: &[u64; 4], f: u64) -> [u64; 4] {
    let mut r = [0; 4];
    // SAFETY: the block reads and writes registers only.
    unsafe {
        asm!(
            "mul {x9}, {x4}, {x5}",
            "umulh {x10}, {x4}, {x5}",
            "mul {x16}, {x4}, {x6}",
            "umulh {x11}, {x4}, {x6}",
            "adds {x10}, {x10}, {x16}",
            "mul {x16}, {x4}, {x7}",
            "umulh {x12}, {x4}, {x7}",
            "adcs {x11}, {x11}, {x16}",
            "adc {x12}, {x12}, xzr",
            "mul {x16}, {x5}, {x6}",
            "adds {x11}, {x11}, {x16}",
            "mul {x16}, {x5}, {x7}",
            "adcs {x12}, {x12}, {x16}",
            "cset {x13}, cs",
            "umulh {x16}, {x5}, {x6}",
            "adds {x12}, {x12}, {x16}",
            "umulh {x16}, {x5}, {x7}",
            "adc {x13}, {x13}, {x16}",
            "mul {x16}, {x6}, {x7}",
            "umulh {x14}, {x6}, {x7}",
            "adds {x13}, {x13}, {x16}",
            "adc {x14}, {x14}, xzr",
            "adds {x9}, {x9}, {x9}",
            "adcs {x10}, {x10}, {x10}",
            "adcs {x11}, {x11}, {x11}",
            "adcs {x12}, {x12}, {x12}",
            "adcs {x13}, {x13}, {x13}",
            "adcs {x14}, {x14}, {x14}",
            "cset {x15}, cs",
            "mul {x8}, {x4}, {x4}",
            "umulh {x16}, {x4}, {x4}",
            "adds {x9}, {x9}, {x16}",
            "mul {x16}, {x5}, {x5}",
            "adcs {x10}, {x10}, {x16}",
            "umulh {x16}, {x5}, {x5}",
            "adcs {x11}, {x11}, {x16}",
            "mul {x16}, {x6}, {x6}",
            "adcs {x12}, {x12}, {x16}",
            "umulh {x16}, {x6}, {x6}",
            "adcs {x13}, {x13}, {x16}",
            "mul {x16}, {x7}, {x7}",
            "adcs {x14}, {x14}, {x16}",
            "umulh {x16}, {x7}, {x7}",
            "adc {x15}, {x15}, {x16}",
            "mul {x16}, {x12}, {x3}",
            "adds {x8}, {x8}, {x16}",
            "mul {x16}, {x13}, {x3}",
            "adcs {x9}, {x9}, {x16}",
            "mul {x16}, {x14}, {x3}",
            "adcs {x10}, {x10}, {x16}",
            "mul {x16}, {x15}, {x3}",
            "adcs {x11}, {x11}, {x16}",
            "cset {x1}, cs",
            "umulh {x16}, {x12}, {x3}",
            "adds {x9}, {x9}, {x16}",
            "umulh {x16}, {x13}, {x3}",
            "adcs {x10}, {x10}, {x16}",
            "umulh {x16}, {x14}, {x3}",
            "adcs {x11}, {x11}, {x16}",
            "umulh {x16}, {x15}, {x3}",
            "adc {x1}, {x1}, {x16}",
            "mul {x16}, {x1}, {x3}",
            "umulh {x17}, {x1}, {x3}",
            "adds {x8}, {x8}, {x16}",
            "adcs {x9}, {x9}, {x17}",
            "b.cc 2f",
            "adcs {x10}, {x10}, xzr",
            "adcs {x11}, {x11}, xzr",
            "csel {x16}, {x3}, xzr, cs",
            "adds {x8}, {x8}, {x16}",
            "adc {x9}, {x9}, xzr",
            "2:",
            x1 = out(reg) _,
            x3 = in(reg) f,
            x4 = in(reg) a[0],
            x5 = in(reg) a[1],
            x6 = in(reg) a[2],
            x7 = in(reg) a[3],
            x8 = out(reg) r[0],
            x9 = out(reg) r[1],
            x10 = out(reg) r[2],
            x11 = out(reg) r[3],
            x12 = out(reg) _,
            x13 = out(reg) _,
            x14 = out(reg) _,
            x15 = out(reg) _,
            x16 = out(reg) _,
            x17 = out(reg) _,
            options(pure, nomem, nostack),
        );
    }
    r
}

/// `a b / 2^256` modulo P-256's prime, below it, for `a` and `b` below it.
#[inline(always)]
pub(super) fn p256_mul(a: &[u64; 4], b: &[u64; 4]) -> [u64; 4] {
    let mut r = [0; 4];
    // SAFETY: the block reads and writes registers only.
    unsafe {
        asm!(
            "mul {x8}, {x4}, {x0}",
            "umulh {x9}, {x4}, {x0}",
            "mul {x1}, {x5}, {x0}",
            "umulh {x10}, {x5}, {x0}",
            "adds {x9}, {x9}, {x1}",
            "mul {x1}, {x6}, {x0}",
            "umulh {x11}, {x6}, {x0}",
            "adcs {x10}, {x10}, {x1}",
            "mul {x1}, {x7}, {x0}",
            "umulh {x12}, {x7}, {x0}",
            "adcs {x11}, {x11}, {x1}",
            "adc {x12}, {x12}, xzr",
            "mov {x13}, xzr",
            "lsl {x14}, {x8}, #32",
            "lsr {x15}, {x8}, #32",
            "subs {x16}, {x8}, {x14}",
            "sbc {x17}, {x8}, {x15}",
            "adds {x9}, {x9}, {x14}",
            "adcs {x10}, {x10}, {x15}",
            "adcs {x11}, {x11}, {x16}",
            "adcs {x12}, {x12}, {x17}",
            "adc {x13}, {x13}, xzr",
            "mul {x1}, {x4}, {x2}",
            "adds {x9}, {x9}, {x1}",
            "mul {x1}, {x5}, {x2}",
            "adcs {x10}, {x10}, {x1}",
            "mul {x1}, {x6}, {x2}",
            "adcs {x11}, {x11}, {x1}",
            "mul {x1}, {x7}, {x2}",
            "adcs {x12}, {x12}, {x1}",
            "adcs {x13}, {x13}, xzr",
            "cset {x8}, cs",
            "umulh {x1}, {x4}, {x2}",
            "adds {x10}, {x10}, {x1}",
            "umulh {x1}, {x5}, {x2}",
            "adcs {x11}, {x11}, {x1}",
            "umulh {x1}, {x6}, {x2}",
            "adcs {x12}, {x12}, {x1}",
            "umulh {x1}, {x7}, {x2}",
            "adcs {x13}, {x13}, {x1}",
            "adc {x8}, {x8}, xzr",
            "lsl {x14}, {x9}, #32",
            "lsr {x15}, {x9}, #32",
            "subs {x16}, {x9}, {x14}",
            "sbc {x17}, {x9}, {x15}",
            "adds {x10}, {x10}, {x14}",
            "adcs {x11}, {x11}, {x15}",
            "adcs {x12}, {x12}, {x16}",
            "adcs {x13}, {x13}, {x17}",
            "adc {x8}, {x8}, xzr",
            "mul {x1}, {x4}, {x3}",
            "adds {x10}, {x10}, {x1}",
            "mul {x1}, {x5}, {x3}",
            "adcs {x11}, {x11}, {x1}",
            "mul {x1}, {x6}, {x3}",
            "adcs {x12}, {x12}, {x1}",
            "mul {x1}, {x7}, {x3}",
            "adcs {x13}, {x13}, {x1}",
            "adcs {x8}, {x8}, xzr",
            "cset {x9}, cs",
            "umulh {x1}, {x4}, {x3}",
            "adds {x11}, {x11}, {x1}",
            "umulh {x1}, {x5}, {x3}",
            "adcs {x12}, {x12}, {x1}",
            "umulh {x1}, {x6}, {x3}",
            "adcs {x13}, {x13}, {x1}",
            "umulh {x1}, {x7}, {x3}",
            "adcs {x8}, {x8}, {x1}",
            "adc {x9}, {x9}, xzr",
            "lsl {x14}, {x10}, #32",
            "lsr {x15}, {x10}, #32",
            "subs {x16}, {x10}, {x14}",
            "sbc {x17}, {x10}, {x15}",
            "adds {x11}, {x11}, {x14}",
            "adcs {x12}, {x12}, {x15}",
            "adcs {x13}, {x13}, {x16}",
            "adcs {x8}, {x8}, {x17}",
            "adc {x9}, {x9}, xzr",
            "mul {x1}, {x4}, {x20}",
            "adds {x11}, {x11}, {x1}",
            "mul {x1}, {x5}, {x20}",
            "adcs {x12}, {x12}, {x1}",
            "mul {x1}, {x6}, {x20}",
            "adcs {x13}, {x13}, {x1}",
            "mul {x1}, {x7}, {x20}",
            "adcs {x8}, {x8}, {x1}",
            "adcs {x9}, {x9}, xzr",
            "cset {x10}, cs",
            "umulh {x1}, {x4}, {x20}",
            "adds {x12}, {x12}, {x1}",
            "umulh {x1}, {x5}, {x20}",
            "adcs {x13}, {x13}, {x1}",
            "umulh {x1}, {x6}, {x20}",
            "adcs {x8}, {x8}, {x1}",
            "umulh {x1}, {x7}, {x20}",
            "adcs {x9}, {x9}, {x1}",
            "adc {x10}, {x10}, xzr",
            "lsl {x14}, {x11}, #32",
            "lsr {x15}, {x11}, #32",
            "subs {x16}, {x11}, {x14}",
            "sbc {x17}, {x11}, {x15}",
            "adds {x12}, {x12}, {x14}",
            "adcs {x13}, {x13}, {x15}",
            "adcs {x8}, {x8}, {x16}",
            "adcs {x9}, {x9}, {x17}",
            "adc {x10}, {x10}, xzr",
            "mov {x14}, #-1",
            "mov {x15}, #0xffffffff",
            "mov {x17}, #0xffffffff00000001",
            "subs {x16}, {x12}, {x14}",
            "sbcs {x1}, {x13}, {x15}",
            "sbcs {x3}, {x8}, xzr",
            "sbcs {x11}, {x9}, {x17}",
            "sbcs xzr, {x10}, xzr",
            "csel {x12}, {x12}, {x16}, lo",
            "csel {x13}, {x13}, {x1}, lo",
            "csel {x8}, {x8}, {x3}, lo",
            "csel {x9}, {x9}, {x11}, lo",
            x0 = in(reg) b[0],
            x1 = out(reg) _,
            x2 = in(reg) b[1],
            x3 = inout(reg) b[2] => _,
            x4 = in(reg) a[0],
            x5 = in(reg) a[1],
            x6 = in(reg) a[2],
            x7 = in(reg) a[3],
            x8 = out(reg) r[2],
            x9 = out(reg) r[3],
            x10 = out(reg) _,
            x11 = out(reg) _,
            x12 = out(reg) r[0],
            x13 = out(reg) r[1],
            x14 = out(reg) _,
            x15 = out(reg) _,
            x16 = out(reg) _,
            x17 = out(reg) _,
            x20 = in(reg) b[3],
            options(pure, nomem, nostack),
        );
    }
    r
}

/// `a^2 / 2^256` modulo P-256's prime, below it, for `a` below it.
#[inline(always)]
pub(super) fn p256_sqr(a: &[u64; 4]) -> [u64; 4] {
    let mut r = [0; 4];
    // SAFETY: the block reads and writes registers only.
    unsafe {
        asm!(
            "mul {x9}, {x4}, {x5}",
            "umulh {x10}, {x4}, {x5}",
            "mul {x3}, {x4}, {x6}",
            "umulh {x11}, {x4}, {x6}",
            "adds {x10}, {x10}, {x3}",
            "mul {x3}, {x4}, {x7}",
            "umulh {x12}, {x4}, {x7}",
            "adcs {x11}, {x11}, {x3}",
            "adc {x12}, {x12}, xzr",
            "mul {x3}, {x5}, {x6}",
            "adds {x11}, {x11}, {x3}",
            "mul {x3}, {x5}, {x7}",
            "adcs {x12}, {x12}, {x3}",
            "cset {x13}, cs",
            "umulh {x3}, {x5}, {x6}",
            "adds {x12}, {x12}, {x3}",
            "umulh {x3}, {x5}, {x7}",
            "adc {x13}, {x13}, {x3}",
            "mul {x3}, {x6}, {x7}",
            "umulh {x14}, {x6}, {x7}",
            "adds {x13}, {x13}, {x3}",
            "adc {x14}, {x14}, xzr",
            "adds {x9}, {x9}, {x9}",
            "adcs {x10}, {x10}, {x10}",
            "adcs {x11}, {x11}, {x11}",
            "adcs {x12}, {x12}, {x12}",
            "adcs {x13}, {x13}, {x13}",
            "adcs {x14}, {x14}, {x14}",
            "cset {x15}, cs",
            "mul {x8}, {x4}, {x4}",
            "umulh {x3}, {x4}, {x4}",
            "adds {x9}, {x9}, {x3}",
            "mul {x3}, {x5}, {x5}",
            "adcs {x10}, {x10}, {x3}",
            "umulh {x3}, {x5}, {x5}",
            "adcs {x11}, {x11}, {x3}",
            "mul {x3}, {x6}, {x6}",
            "adcs {x12}, {x12}, {x3}",
            "umulh {x3}, {x6}, {x6}",
            "adcs {x13}, {x13}, {x3}",
            "mul {x3}, {x7}, {x7}",
            "adcs {x14}, {x14}, {x3}",
            "umulh {x3}, {x7}, {x7}",
            "adc {x15}, {x15}, {x3}",
            "lsl {x16}, {x8}, #32",
            "lsr {x17}, {x8}, #32",
            "subs {x1}, {x8}, {x16}",
            "sbc {x2}, {x8}, {x17}",
            "adds {x8}, {x9}, {x16}",
            "adcs {x9}, {x10}, {x17}",
            "adcs {x10}, {x11}, {x1}",
            "adc {x11}, {x2}, xzr",
            "lsl {x16}, {x8}, #32",
            "lsr {x17}, {x8}, #32",
            "subs {x1}, {x8}, {x16}",
            "sbc {x2}, {x8}, {x17}",
            "adds {x8}, {x9}, {x16}",
            "adcs {x9}, {x10}, {x17}",
            "adcs {x10}, {x11}, {x1}",
            "adc {x11}, {x2}, xzr",
            "lsl {x16}, {x8}, #32",
            "lsr {x17}, {x8}, #32",
            "subs {x1}, {x8}, {x16}",
            "sbc {x2}, {x8}, {x17}",
            "adds {x8}, {x9}, {x16}",
            "adcs {x9}, {x10}, {x17}",
            "adcs {x10}, {x11}, {x1}",
            "adc {x11}, {x2}, xzr",
            "lsl {x16}, {x8}, #32",
            "lsr {x17}, {x8}, #32",
            "subs {x1}, {x8}, {x16}",
            "sbc {x2}, {x8}, {x17}",
            "adds {x8}, {x9}, {x16}",
            "adcs {x9}, {x10}, {x17}",
            "adcs {x10}, {x11}, {x1}",
            "adc {x11}, {x2}, xzr",
            "adds {x8}, {x8}, {x12}",
            "adcs {x9}, {x9}, {x13}",
            "adcs {x10}, {x10}, {x14}",
            "adcs {x11}, {x11}, {x15}",
            "cset {x12}, cs",
            "mov {x16}, #-1",
            "mov {x17}, #0xffffffff",
            "mov {x2}, #0xffffffff00000001",
            "subs {x1}, {x8}, {x16}",
            "sbcs {x3}, {x9}, {x17}",
            "sbcs {x13}, {x10}, xzr",
            "sbcs {x14}, {x11}, {x2}",
            "sbcs xzr, {x12}, xzr",
            "csel {x8}, {x8}, {x1}, lo",
            "csel {x9}, {x9}, {x3}, lo",
            "csel {x10}, {x10}, {x13}, lo",
            "csel {x11}, {x11}, {x14}, lo",
            x1 = out(reg) _,
            x2 = out(reg) _,
            x3 = out(reg) _,
            x4 = in(reg) a[0],
            x5 = in(reg) a[1],
            x6 = in(reg) a[2],
            x7 = in(reg) a[3],
            x8 = out(reg) r[0],
            x9 = out(reg) r[1],
            x10 = out(reg) r[2],
            x11 = out(reg) r[3],
            x12 = out(reg) _,
            x13 = out(reg) _,
            x14 = out(reg) _,
            x15 = out(reg) _,
            x16 = out(reg) _,
            x17 = out(reg) _,
            options(pure, nomem, nostack),
        );
    }
    r
}

/// `a b / 2^384` modulo P-384's prime, below it, for `a` and `b` below it.
#[inline(always)]
pub(super) fn p384_mul(a: &[u64; 6], b: &[u64; 6]) -> [u64; 6] {
    let mut r = [0; 6];
    // SAFETY: the block reads and writes registers only.
    unsafe {
        asm!(
            "mul {x10}, {x4}, {x0}",
            "umulh {x11}, {x4}, {x0}",
            "mul {x1}, {x5}, {x0}",
            "umulh {x12}, {x5}, {x0}",
            "adds {x11}, {x11}, {x1}",
            "mul {x1}, {x6}, {x0}",
            "umulh {x13}, {x6}, {x0}",
            "adcs {x12}, {x12}, {x1}",
            "mul {x1}, {x7}, {x0}",
            "umulh {x14}, {x7}, {x0}",
            "adcs {x13}, {x13}, {x1}",
            "mul {x1}, {x8}, {x0}",
            "umulh {x15}, {x8}, {x0}",
            "adcs {x14}, {x14}, {x1}",
            "mul {x1}, {x9}, {x0}",
            "umulh {x16}, {x9}, {x0}",
            "adcs {x15}, {x15}, {x1}",
            "adc {x16}, {x16}, xzr",
            "mov {x17}, xzr",
            "add {x23}, {x10}, {x10}, lsl #32",
            "lsl {x20}, {x23}, #32",
            "lsr {x21}, {x23}, #32",
            "subs {x22}, {x20}, {x23}",
            "sbc {x24}, {x21}, xzr",
            "cmn {x10}, {x22}",
            "adcs {x11}, {x11}, {x24}",
            "adcs {x12}, {x12}, xzr",
            "adcs {x13}, {x13}, xzr",
            "adcs {x14}, {x14}, xzr",
            "adcs {x15}, {x15}, xzr",
            "adcs {x16}, {x16}, {x23}",
            "adc {x17}, {x17}, xzr",
            "adds {x22}, {x21}, {x23}",
            "cset {x21}, cs",
            "subs {x11}, {x11}, {x20}",
            "sbcs {x12}, {x12}, {x22}",
            "sbcs {x13}, {x13}, {x21}",
            "sbcs {x14}, {x14}, xzr",
            "sbcs {x15}, {x15}, xzr",
            "sbcs {x16}, {x16}, xzr",
            "sbc {x17}, {x17}, xzr",
            "mov {x10}, xzr",
            "mul {x1}, {x4}, {x2}",
            "adds {x11}, {x11}, {x1}",
            "mul {x1}, {x5}, {x2}",
            "adcs {x12}, {x12}, {x1}",
            "mul {x1}, {x6}, {x2}",
            "adcs {x13}, {x13}, {x1}",
            "mul {x1}, {x7}, {x2}",
            "adcs {x14}, {x14}, {x1}",
            "mul {x1}, {x8}, {x2}",
            "adcs {x15}, {x15}, {x1}",
            "mul {x1}, {x9}, {x2}",
            "adcs {x16}, {x16}, {x1}",
            "adcs {x17}, {x17}, xzr",
            "cset {x10}, cs",
            "umulh {x1}, {x4}, {x2}",
            "adds {x12}, {x12}, {x1}",
            "umulh {x1}, {x5}, {x2}",
            "adcs {x13}, {x13}, {x1}",
            "umulh {x1}, {x6}, {x2}",
            "adcs {x14}, {x14}, {x1}",
            "umulh {x1}, {x7}, {x2}",
            "adcs {x15}, {x15}, {x1}",
            "umulh {x1}, {x8}, {x2}",
            "adcs {x16}, {x16}, {x1}",
            "umulh {x1}, {x9}, {x2}",
            "adcs {x17}, {x17}, {x1}",
            "adc {x10}, {x10}, xzr",
            "add {x23}, {x11}, {x11}, lsl #32",
            "lsl {x20}, {x23}, #32",
            "lsr {x21}, {x23}, #32",
            "subs {x22}, {x20}, {x23}",
            "sbc {x24}, {x21}, xzr",
            "cmn {x11}, {x22}",
            "adcs {x12}, {x12}, {x24}",
            "adcs {x13}, {x13}, xzr",
            "adcs {x14}, {x14}, xzr",
            "adcs {x15}, {x15}, xzr",
            "adcs {x16}, {x16}, xzr",
            "adcs {x17}, {x17}, {x23}",
            "adc {x10}, {x10}, xzr",
            "adds {x22}, {x21}, {x23}",
            "cset {x21}, cs",
            "subs {x12}, {x12}, {x20}",
            "sbcs {x13}, {x13}, {x22}",
            "sbcs {x14}, {x14}, {x21}",
            "sbcs {x15}, {x15}, xzr",
            "sbcs {x16}, {x16}, xzr",
            "sbcs {x17}, {x17}, xzr",
            "sbc {x10}, {x10}, xzr",
            "mov {x11}, xzr",
            "mul {x1}, {x4}, {x25}",
            "adds {x12}, {x12}, {x1}",
            "mul {x1}, {x5}, {x25}",
            "adcs {x13}, {x13}, {x1}",
            "mul {x1}, {x6}, {x25}",
            "adcs {x14}, {x14}, {x1}",
            "mul {x1}, {x7}, {x25}",
            "adcs {x15}, {x15}, {x1}",
            "mul {x1}, {x8}, {x25}",
            "adcs {x16}, {x16}, {x1}",
            "mul {x1}, {x9}, {x25}",
            "adcs {x17}, {x17}, {x1}",
            "adcs {x10}, {x10}, xzr",
            "cset {x11}, cs",
            "umulh {x1}, {x4}, {x25}",
            "adds {x13}, {x13}, {x1}",
            "umulh {x1}, {x5}, {x25}",
            "adcs {x14}, {x14}, {x1}",
            "umulh {x1}, {x6}, {x25}",
            "adcs {x15}, {x15}, {x1}",
            "umulh {x1}, {x7}, {x25}",
            "adcs {x16}, {x16}, {x1}",
            "umulh {x1}, {x8}, {x25}",
            "adcs {x17}, {x17}, {x1}",
            "umulh {x1}, {x9}, {x25}",
            "adcs {x10}, {x10}, {x1}",
            "adc {x11}, {x11}, xzr",
            "add {x23}, {x12}, {x12}, lsl #32",
            "lsl {x20}, {x23}, #32",
            "lsr {x21}, {x23}, #32",
            "subs {x22}, {x20}, {x23}",
            "sbc {x24}, {x21}, xzr",
            "cmn {x12}, {x22}",
            "adcs {x13}, {x13}, {x24}",
            "adcs {x14}, {x14}, xzr",
            "adcs {x15}, {x15}, xzr",
            "adcs {x16}, {x16}, xzr",
            "adcs {x17}, {x17}, xzr",
            "adcs {x10}, {x10}, {x23}",
            "adc {x11}, {x11}, xzr",
            "adds {x22}, {x21}, {x23}",
            "cset {x21}, cs",
            "subs {x13}, {x13}, {x20}",
            "sbcs {x14}, {x14}, {x22}",
            "sbcs {x15}, {x15}, {x21}",
            "sbcs {x16}, {x16}, xzr",
            "sbcs {x17}, {x17}, xzr",
            "sbcs {x10}, {x10}, xzr",
            "sbc {x11}, {x11}, xzr",
            "mov {x12}, xzr",
            "mul {x1}, {x4}, {x26}",
            "adds {x13}, {x13}, {x1}",
            "mul {x1}, {x5}, {x26}",
            "adcs {x14}, {x14}, {x1}",
            "mul {x1}, {x6}, {x26}",
            "adcs {x15}, {x15}, {x1}",
            "mul {x1}, {x7}, {x26}",
            "adcs {x16}, {x16}, {x1}",
            "mul {x1}, {x8}, {x26}",
            "adcs {x17}, {x17}, {x1}",
            "mul {x1}, {x9}, {x26}",
            "adcs {x10}, {x10}, {x1}",
            "adcs {x11}, {x11}, xzr",
            "cset {x12}, cs",
            "umulh {x1}, {x4}, {x26}",
            "adds {x14}, {x14}, {x1}",
            "umulh {x1}, {x5}, {x26}",
            "adcs {x15}, {x15}, {x1}",
            "umulh {x1}, {x6}, {x26}",
            "adcs {x16}, {x16}, {x1}",
            "umulh {x1}, {x7}, {x26}",
            "adcs {x17}, {x17}, {x1}",
            "umulh {x1}, {x8}, {x26}",
            "adcs {x10}, {x10}, {x1}",
            "umulh {x1}, {x9}, {x26}",
            "adcs {x11}, {x11}, {x1}",
            "adc {x12}, {x12}, xzr",
            "add {x23}, {x13}, {x13}, lsl #32",
            "lsl {x20}, {x23}, #32",
            "lsr {x21}, {x23}, #32",
            "subs {x22}, {x20}, {x23}",
            "sbc {x24}, {x21}, xzr",
            "cmn {x13}, {x22}",
            "adcs {x14}, {x14}, {x24}",
            "adcs {x15}, {x15}, xzr",
            "adcs {x16}, {x16}, xzr",
            "adcs {x17}, {x17}, xzr",
            "adcs {x10}, {x10}, xzr",
            "adcs {x11}, {x11}, {x23}",
            "adc {x12}, {x12}, xzr",
            "adds {x22}, {x21}, {x23}",
            "cset {x21}, cs",
            "subs {x14}, {x14}, {x20}",
            "sbcs {x15}, {x15}, {x22}",
            "sbcs {x16}, {x16}, {x21}",
            "sbcs {x17}, {x17}, xzr",
            "sbcs {x10}, {x10}, xzr",
            "sbcs {x11}, {x11}, xzr",
            "sbc {x12}, {x12}, xzr",
            "mov {x13}, xzr",
            "mul {x1}, {x4}, {x27}",
            "adds {x14}, {x14}, {x1}",
            "mul {x1}, {x5}, {x27}",
            "adcs {x15}, {x15}, {x1}",
            "mul {x1}, {x6}, {x27}",
            "adcs {x16}, {x16}, {x1}",
            "mul {x1}, {x7}, {x27}",
            "adcs {x17}, {x17}, {x1}",
            "mul {x1}, {x8}, {x27}",
            "adcs {x10}, {x10}, {x1}",
            "mul {x1}, {x9}, {x27}",
            "adcs {x11}, {x11}, {x1}",
            "adcs {x12}, {x12}, xzr",
            "cset {x13}, cs",
            "umulh {x1}, {x4}, {x27}",
            "adds {x15}, {x15}, {x1}",
            "umulh {x1}, {x5}, {x27}",
            "adcs {x16}, {x16}, {x1}",
            "umulh {x1}, {x6}, {x27}",
            "adcs {x17}, {x17}, {x1}",
            "umulh {x1}, {x7}, {x27}",
            "adcs {x10}, {x10}, {x1}",
            "umulh {x1}, {x8}, {x27}",
            "adcs {x11}, {x11}, {x1}",
            "umulh {x1}, {x9}, {x27}",
            "adcs {x12}, {x12}, {x1}",
            "adc {x13}, {x13}, xzr",
            "add {x23}, {x14}, {x14}, lsl #32",
            "lsl {x20}, {x23}, #32",
            "lsr {x21}, {x23}, #32",
            "subs {x22}, {x20}, {x23}",
            "sbc {x24}, {x21}, xzr",
            "cmn {x14}, {x22}",
            "adcs {x15}, {x15}, {x24}",
            "adcs {x16}, {x16}, xzr",
            "adcs {x17}, {x17}, xzr",
            "adcs {x10}, {x10}, xzr",
            "adcs {x11}, {x11}, xzr",
            "adcs {x12}, {x12}, {x23}",
            "adc {x13}, {x13}, xzr",
            "adds {x22}, {x21}, {x23}",
            "cset {x21}, cs",
            "subs {x15}, {x15}, {x20}",
            "sbcs {x16}, {x16}, {x22}",
            "sbcs {x17}, {x17}, {x21}",
            "sbcs {x10}, {x10}, xzr",
            "sbcs {x11}, {x11}, xzr",
            "sbcs {x12}, {x12}, xzr",
            "sbc {x13}, {x13}, xzr",
            "mov {x14}, xzr",
            "mul {x1}, {x4}, {x28}",
            "adds {x15}, {x15}, {x1}",
            "mul {x1}, {x5}, {x28}",
            "adcs {x16}, {x16}, {x1}",
            "mul {x1}, {x6}, {x28}",
            "adcs {x17}, {x17}, {x1}",
            "mul {x1}, {x7}, {x28}",
            "adcs {x10}, {x10}, {x1}",
            "mul {x1}, {x8}, {x28}",
            "adcs {x11}, {x11}, {x1}",
            "mul {x1}, {x9}, {x28}",
            "adcs {x12}, {x12}, {x1}",
            "adcs {x13}, {x13}, xzr",
            "cset {x14}, cs",
            "umulh {x1}, {x4}, {x28}",
            "adds {x16}, {x16}, {x1}",
            "umulh {x1}, {x5}, {x28}",
            "adcs {x17}, {x17}, {x1}",
            "umulh {x1}, {x6}, {x28}",
            "adcs {x10}, {x10}, {x1}",
            "umulh {x1}, {x7}, {x28}",
            "adcs {x11}, {x11}, {x1}",
            "umulh {x1}, {x8}, {x28}",
            "adcs {x12}, {x12}, {x1}",
            "umulh {x1}, {x9}, {x28}",
            "adcs {x13}, {x13}, {x1}",
            "adc {x14}, {x14}, xzr",
            "add {x23}, {x15}, {x15}, lsl #32",
            "lsl {x20}, {x23}, #32",
            "lsr {x21}, {x23}, #32",
            "subs {x22}, {x20}, {x23}",
            "sbc {x24}, {x21}, xzr",
            "cmn {x15}, {x22}",
            "adcs {x16}, {x16}, {x24}",
            "adcs {x17}, {x17}, xzr",
            "adcs {x10}, {x10}, xzr",
            "adcs {x11}, {x11}, xzr",
            "adcs {x12}, {x12}, xzr",
            "adcs {x13}, {x13}, {x23}",
            "adc {x14}, {x14}, xzr",
            "adds {x22}, {x21}, {x23}",
            "cset {x21}, cs",
            "subs {x16}, {x16}, {x20}",
            "sbcs {x17}, {x17}, {x22}",
            "sbcs {x10}, {x10}, {x21}",
            "sbcs {x11}, {x11}, xzr",
            "sbcs {x12}, {x12}, xzr",
            "sbcs {x13}, {x13}, xzr",
            "sbc {x14}, {x14}, xzr",
            "mov {x15}, xzr",
            "mov {x7}, #0xffffffff",
            "mov {x8}, #0xffffffff00000000",
            "mov {x9}, #-2",
            "mov {x20}, #-1",
            "subs {x1}, {x16}, {x7}",
            "sbcs {x2}, {x17}, {x8}",
            "sbcs {x3}, {x10}, {x9}",
            "sbcs {x4}, {x11}, {x20}",
            "sbcs {x5}, {x12}, {x20}",
            "sbcs {x6}, {x13}, {x20}",
            "sbcs xzr, {x14}, xzr",
            "csel {x16}, {x16}, {x1}, lo",
            "csel {x17}, {x17}, {x2}, lo",
            "csel {x10}, {x10}, {x3}, lo",
            "csel {x11}, {x11}, {x4}, lo",
            "csel {x12}, {x12}, {x5}, lo",
            "csel {x13}, {x13}, {x6}, lo",
            x0 = in(reg) b[0],
            x1 = out(reg) _,
            x2 = inout(reg) b[1] => _,
            x3 = out(reg) _,
            x4 = inout(reg) a[0] => _,
            x5 = inout(reg) a[1] => _,
            x6 = inout(reg) a[2] => _,
            x7 = inout(reg) a[3] => _,
            x8 = inout(reg) a[4] => _,
            x9 = inout(reg) a[5] => _,
            x10 = out(reg) r[2],
            x11 = out(reg) r[3],
            x12 = out(reg) r[4],
            x13 = out(reg) r[5],
            x14 = out(reg) _,
            x15 = out(reg) _,
            x16 = out(reg) r[0],
            x17 = out(reg) r[1],
            x20 = out(reg) _,
            x21 = out(reg) _,
            x22 = out(reg) _,
            x23 = out(reg) _,
            x24 = out(reg) _,
            x25 = in(reg) b[2],
            x26 = in(reg) b[3],
            x27 = in(reg) b[4],
            x28 = in(reg) b[5],
            options(pure, nomem, nostack),
        );
    }
    r
}

/// `a^2 / 2^384` modulo P-384's prime, below it, for `a` below it.
#[inline(always)]
pub(super) fn p384_sqr(a: &[u64; 6]) -> [u64; 6] {
    let mut r = [0; 6];
    // SAFETY: the block reads and writes registers only.
    unsafe {
        asm!(
            "mul {x11}, {x4}, {x5}",
            "umulh {x12}, {x4}, {x5}",
            "mul {x3}, {x4}, {x6}",
            "umulh {x13}, {x4}, {x6}",
            "adds {x12}, {x12}, {x3}",
            "mul {x3}, {x4}, {x7}",
            "umulh {x14}, {x4}, {x7}",
            "adcs {x13}, {x13}, {x3}",
            "mul {x3}, {x4}, {x8}",
            "umulh {x15}, {x4}, {x8}",
            "adcs {x14}, {x14}, {x3}",
            "mul {x3}, {x4}, {x9}",
            "umulh {x16}, {x4}, {x9}",
            "adcs {x15}, {x15}, {x3}",
            "adc {x16}, {x16}, xzr",
            "mul {x3}, {x5}, {x6}",
            "adds {x13}, {x13}, {x3}",
            "mul {x3}, {x5}, {x7}",
            "adcs {x14}, {x14}, {x3}",
            "mul {x3}, {x5}, {x8}",
            "adcs {x15}, {x15}, {x3}",
            "mul {x3}, {x5}, {x9}",
            "adcs {x16}, {x16}, {x3}",
            "cset {x17}, cs",
            "umulh {x3}, {x5}, {x6}",
            "adds {x14}, {x14}, {x3}",
            "umulh {x3}, {x5}, {x7}",
            "adcs {x15}, {x15}, {x3}",
            "umulh {x3}, {x5}, {x8}",
            "adcs {x16}, {x16}, {x3}",
            "umulh {x3}, {x5}, {x9}",
            "adc {x17}, {x17}, {x3}",
            "mul {x3}, {x6}, {x7}",
            "adds {x15}, {x15}, {x3}",
            "mul {x3}, {x6}, {x8}",
            "adcs {x16}, {x16}, {x3}",
            "mul {x3}, {x6}, {x9}",
            "adcs {x17}, {x17}, {x3}",
            "cset {x20}, cs",
            "umulh {x3}, {x6}, {x7}",
            "adds {x16}, {x16}, {x3}",
            "umulh {x3}, {x6}, {x8}",
            "adcs {x17}, {x17}, {x3}",
            "umulh {x3}, {x6}, {x9}",
            "adc {x20}, {x20}, {x3}",
            "mul {x3}, {x7}, {x8}",
            "adds {x17}, {x17}, {x3}",
            "mul {x3}, {x7}, {x9}",
            "adcs {x20}, {x20}, {x3}",
            "cset {x21}, cs",
            "umulh {x3}, {x7}, {x8}",
            "adds {x20}, {x20}, {x3}",
            "umulh {x3}, {x7}, {x9}",
            "adc {x21}, {x21}, {x3}",
            "mul {x3}, {x8}, {x9}",
            "adds {x21}, {x21}, {x3}",
            "cset {x22}, cs",
            "umulh {x3}, {x8}, {x9}",
            "add {x22}, {x22}, {x3}",
            "adds {x11}, {x11}, {x11}",
            "adcs {x12}, {x12}, {x12}",
            "adcs {x13}, {x13}, {x13}",
            "adcs {x14}, {x14}, {x14}",
            "adcs {x15}, {x15}, {x15}",
            "adcs {x16}, {x16}, {x16}",
            "adcs {x17}, {x17}, {x17}",
            "adcs {x20}, {x20}, {x20}",
            "adcs {x21}, {x21}, {x21}",
            "adcs {x22}, {x22}, {x22}",
            "cset {x23}, cs",
            "mul {x10}, {x4}, {x4}",
            "umulh {x3}, {x4}, {x4}",
            "adds {x11}, {x11}, {x3}",
            "mul {x3}, {x5}, {x5}",
            "adcs {x12}, {x12}, {x3}",
            "umulh {x3}, {x5}, {x5}",
            "adcs {x13}, {x13}, {x3}",
            "mul {x3}, {x6}, {x6}",
            "adcs {x14}, {x14}, {x3}",
            "umulh {x3}, {x6}, {x6}",
            "adcs {x15}, {x15}, {x3}",
            "mul {x3}, {x7}, {x7}",
            "adcs {x16}, {x16}, {x3}",
            "umulh {x3}, {x7}, {x7}",
            "adcs {x17}, {x17}, {x3}",
            "mul {x3}, {x8}, {x8}",
            "adcs {x20}, {x20}, {x3}",
            "umulh {x3}, {x8}, {x8}",
            "adcs {x21}, {x21}, {x3}",
            "mul {x3}, {x9}, {x9}",
            "adcs {x22}, {x22}, {x3}",
            "umulh {x3}, {x9}, {x9}",
            "adc {x23}, {x23}, {x3}",
            "add {x27}, {x10}, {x10}, lsl #32",
            "lsl {x24}, {x27}, #32",
            "lsr {x25}, {x27}, #32",
            "subs {x26}, {x24}, {x27}",
            "sbc {x28}, {x25}, xzr",
            "cmn {x10}, {x26}",
            "adcs {x11}, {x11}, {x28}",
            "adcs {x12}, {x12}, xzr",
            "adcs {x13}, {x13}, xzr",
            "adcs {x14}, {x14}, xzr",
            "adcs {x15}, {x15}, xzr",
            "adc {x10}, {x27}, xzr",
            "adds {x1}, {x25}, {x27}",
            "cset {x25}, cs",
            "subs {x11}, {x11}, {x24}",
            "sbcs {x12}, {x12}, {x1}",
            "sbcs {x13}, {x13}, {x25}",
            "sbcs {x14}, {x14}, xzr",
            "sbcs {x15}, {x15}, xzr",
            "sbc {x10}, {x10}, xzr",
            "add {x27}, {x11}, {x11}, lsl #32",
            "lsl {x24}, {x27}, #32",
            "lsr {x25}, {x27}, #32",
            "subs {x26}, {x24}, {x27}",
            "sbc {x28}, {x25}, xzr",
            "cmn {x11}, {x26}",
            "adcs {x12}, {x12}, {x28}",
            "adcs {x13}, {x13}, xzr",
            "adcs {x14}, {x14}, xzr",
            "adcs {x15}, {x15}, xzr",
            "adcs {x10}, {x10}, xzr",
            "adc {x11}, {x27}, xzr",
            "adds {x1}, {x25}, {x27}",
            "cset {x25}, cs",
            "subs {x12}, {x12}, {x24}",
            "sbcs {x13}, {x13}, {x1}",
            "sbcs {x14}, {x14}, {x25}",
            "sbcs {x15}, {x15}, xzr",
            "sbcs {x10}, {x10}, xzr",
            "sbc {x11}, {x11}, xzr",
            "add {x27}, {x12}, {x12}, lsl #32",
            "lsl {x24}, {x27}, #32",
            "lsr {x25}, {x27}, #32",
            "subs {x26}, {x24}, {x27}",
            "sbc {x28}, {x25}, xzr",
            "cmn {x12}, {x26}",
            "adcs {x13}, {x13}, {x28}",
            "adcs {x14}, {x14}, xzr",
            "adcs {x15}, {x15}, xzr",
            "adcs {x10}, {x10}, xzr",
            "adcs {x11}, {x11}, xzr",
            "adc {x12}, {x27}, xzr",
            "adds {x1}, {x25}, {x27}",
            "cset {x25}, cs",
            "subs {x13}, {x13}, {x24}",
            "sbcs {x14}, {x14}, {x1}",
            "sbcs {x15}, {x15}, {x25}",
            "sbcs {x10}, {x10}, xzr",
            "sbcs {x11}, {x11}, xzr",
            "sbc {x12}, {x12}, xzr",
            "add {x27}, {x13}, {x13}, lsl #32",
            "lsl {x24}, {x27}, #32",
            "lsr {x25}, {x27}, #32",
            "subs {x26}, {x24}, {x27}",
            "sbc {x28}, {x25}, xzr",
            "cmn {x13}, {x26}",
            "adcs {x14}, {x14}, {x28}",
            "adcs {x15}, {x15}, xzr",
            "adcs {x10}, {x10}, xzr",
            "adcs {x11}, {x11}, xzr",
            "adcs {x12}, {x12}, xzr",
            "adc {x13}, {x27}, xzr",
            "adds {x1}, {x25}, {x27}",
            "cset {x25}, cs",
            "subs {x14}, {x14}, {x24}",
            "sbcs {x15}, {x15}, {x1}",
            "sbcs {x10}, {x10}, {x25}",
            "sbcs {x11}, {x11}, xzr",
            "sbcs {x12}, {x12}, xzr",
            "sbc {x13}, {x13}, xzr",
            "add {x27}, {x14}, {x14}, lsl #32",
            "lsl {x24}, {x27}, #32",
            "lsr {x25}, {x27}, #32",
            "subs {x26}, {x24}, {x27}",
            "sbc {x28}, {x25}, xzr",
            "cmn {x14}, {x26}",
            "adcs {x15}, {x15}, {x28}",
            "adcs {x10}, {x10}, xzr",
            "adcs {x11}, {x11}, xzr",
            "adcs {x12}, {x12}, xzr",
            "adcs {x13}, {x13}, xzr",
            "adc {x14}, {x27}, xzr",
            "adds {x1}, {x25}, {x27}",
            "cset {x25}, cs",
            "subs {x15}, {x15}, {x24}",
            "sbcs {x10}, {x10}, {x1}",
            "sbcs {x11}, {x11}, {x25}",
            "sbcs {x12}, {x12}, xzr",
            "sbcs {x13}, {x13}, xzr",
            "sbc {x14}, {x14}, xzr",
            "add {x27}, {x15}, {x15}, lsl #32",
            "lsl {x24}, {x27}, #32",
            "lsr {x25}, {x27}, #32",
            "subs {x26}, {x24}, {x27}",
            "sbc {x28}, {x25}, xzr",
            "cmn {x15}, {x26}",
            "adcs {x10}, {x10}, {x28}",
            "adcs {x11}, {x11}, xzr",
            "adcs {x12}, {x12}, xzr",
            "adcs {x13}, {x13}, xzr",
            "adcs {x14}, {x14}, xzr",
            "adc {x15}, {x27}, xzr",
            "adds {x1}, {x25}, {x27}",
            "cset {x25}, cs",
            "subs {x10}, {x10}, {x24}",
            "sbcs {x11}, {x11}, {x1}",
            "sbcs {x12}, {x12}, {x25}",
            "sbcs {x13}, {x13}, xzr",
            "sbcs {x14}, {x14}, xzr",
            "sbc {x15}, {x15}, xzr",
            "adds {x10}, {x10}, {x16}",
            "adcs {x11}, {x11}, {x17}",
            "adcs {x12}, {x12}, {x20}",
            "adcs {x13}, {x13}, {x21}",
            "adcs {x14}, {x14}, {x22}",
            "adcs {x15}, {x15}, {x23}",
            "cset {x16}, cs",
            "mov {x4}, #0xffffffff",
            "mov {x5}, #0xffffffff00000000",
            "mov {x6}, #-2",
            "mov {x7}, #-1",
            "subs {x8}, {x10}, {x4}",
            "sbcs {x9}, {x11}, {x5}",
            "sbcs {x24}, {x12}, {x6}",
            "sbcs {x25}, {x13}, {x7}",
            "sbcs {x26}, {x14}, {x7}",
            "sbcs {x27}, {x15}, {x7}",
            "sbcs xzr, {x16}, xzr",
            "csel {x10}, {x10}, {x8}, lo",
            "csel {x11}, {x11}, {x9}, lo",
            "csel {x12}, {x12}, {x24}, lo",
            "csel {x13}, {x13}, {x25}, lo",
            "csel {x14}, {x14}, {x26}, lo",
            "csel {x15}, {x15}, {x27}, lo",
            x1 = out(reg) _,
            x3 = out(reg) _,
            x4 = inout(reg) a[0] => _,
            x5 = inout(reg) a[1] => _,
            x6 = inout(reg) a[2] => _,
            x7 = inout(reg) a[3] => _,
            x8 = inout(reg) a[4] => _,
            x9 = inout(reg) a[5] => _,
            x10 = out(reg) r[0],
            x11 = out(reg) r[1],
            x12 = out(reg) r[2],
            x13 = out(reg) r[3],
            x14 = out(reg) r[4],
            x15 = out(reg) r[5],
            x16 = out(reg) _,
            x17 = out(reg) _,
            x20 = out(reg) _,
            x21 = out(reg) _,
            x22 = out(reg) _,
            x23 = out(reg) _,
            x24 = out(reg) _,
            x25 = out(reg) _,
            x26 = out(reg) _,
            x27 = out(reg) _,
            x28 = out(reg) _,
            options(pure, nomem, nostack),
        );
    }
    r
}

/// `a + b` modulo the prime with `2^256 = f`, `f < 2^62`.
#[inline(always)]
pub(super) fn pseudo_add_4(a: &[u64; 4], b: &[u64; 4], f: u64) -> [u64; 4] {
    let mut r = [0; 4];
    // SAFETY: the block reads and writes registers only.
    unsafe {
        asm!(
            "adds {x4}, {x4}, {x10}",
            "adcs {x5}, {x5}, {x11}",
            "adcs {x6}, {x6}, {x12}",
            "adcs {x7}, {x7}, {x13}",
            "csel {x16}, {x3}, xzr, cs",
            "adds {x4}, {x4}, {x16}",
            "b.cc 2f",
            "adcs {x5}, {x5}, xzr",
            "adcs {x6}, {x6}, xzr",
            "adcs {x7}, {x7}, xzr",
            "csel {x16}, {x3}, xzr, cs",
            "add {x4}, {x4}, {x16}",
            "2:",
            x3 = in(reg) f,
            x4 = inout(reg) a[0] => r[0],
            x5 = inout(reg) a[1] => r[1],
            x6 = inout(reg) a[2] => r[2],
            x7 = inout(reg) a[3] => r[3],
            x10 = in(reg) b[0],
            x11 = in(reg) b[1],
            x12 = in(reg) b[2],
            x13 = in(reg) b[3],
            x16 = out(reg) _,
            options(pure, nomem, nostack),
        );
    }
    r
}

/// `a - b` modulo the prime with `2^256 = f`, `f < 2^62`.
#[inline(always)]
pub(super) fn pseudo_sub_4(a: &[u64; 4], b: &[u64; 4], f: u64) -> [u64; 4] {
    let mut r = [0; 4];
    // SAFETY: the block reads and writes registers only.
    unsafe {
        asm!(
            "subs {x4}, {x4}, {x10}",
            "sbcs {x5}, {x5}, {x11}",
            "sbcs {x6}, {x6}, {x12}",
            "sbcs {x7}, {x7}, {x13}",
            "csel {x16}, {x3}, xzr, cc",
            "subs {x4}, {x4}, {x16}",
            "b.cs 2f",
            "sbcs {x5}, {x5}, xzr",
            "sbcs {x6}, {x6}, xzr",
            "sbcs {x7}, {x7}, xzr",
            "csel {x16}, {x3}, xzr, cc",
            "subs {x4}, {x4}, {x16}",
            "sbcs {x5}, {x5}, xzr",
            "sbcs {x6}, {x6}, xzr",
            "sbcs {x7}, {x7}, xzr",
            "2:",
            x3 = in(reg) f,
            x4 = inout(reg) a[0] => r[0],
            x5 = inout(reg) a[1] => r[1],
            x6 = inout(reg) a[2] => r[2],
            x7 = inout(reg) a[3] => r[3],
            x10 = in(reg) b[0],
            x11 = in(reg) b[1],
            x12 = in(reg) b[2],
            x13 = in(reg) b[3],
            x16 = out(reg) _,
            options(pure, nomem, nostack),
        );
    }
    r
}

/// `a + b` modulo `m`, for `a` and `b` below `m`.
#[inline(always)]
pub(super) fn mont_add_4(a: &[u64; 4], b: &[u64; 4], m: &[u64; 4]) -> [u64; 4] {
    let mut r = [0; 4];
    // SAFETY: the block reads and writes registers only.
    unsafe {
        asm!(
            "adds {x4}, {x4}, {x10}",
            "adcs {x5}, {x5}, {x11}",
            "adcs {x6}, {x6}, {x12}",
            "adcs {x7}, {x7}, {x13}",
            "cset {x1}, cs",
            "subs {x10}, {x4}, {x16}",
            "sbcs {x11}, {x5}, {x17}",
            "sbcs {x12}, {x6}, {x20}",
            "sbcs {x13}, {x7}, {x21}",
            "sbcs xzr, {x1}, xzr",
            "csel {x4}, {x4}, {x10}, lo",
            "csel {x5}, {x5}, {x11}, lo",
            "csel {x6}, {x6}, {x12}, lo",
            "csel {x7}, {x7}, {x13}, lo",
            x1 = out(reg) _,
            x4 = inout(reg) a[0] => r[0],
            x5 = inout(reg) a[1] => r[1],
            x6 = inout(reg) a[2] => r[2],
            x7 = inout(reg) a[3] => r[3],
            x10 = inout(reg) b[0] => _,
            x11 = inout(reg) b[1] => _,
            x12 = inout(reg) b[2] => _,
            x13 = inout(reg) b[3] => _,
            x16 = in(reg) m[0],
            x17 = in(reg) m[1],
            x20 = in(reg) m[2],
            x21 = in(reg) m[3],
            options(pure, nomem, nostack),
        );
    }
    r
}

/// `a - b` modulo `m`, for `a` and `b` below `m`.
#[inline(always)]
pub(super) fn mont_sub_4(a: &[u64; 4], b: &[u64; 4], m: &[u64; 4]) -> [u64; 4] {
    let mut r = [0; 4];
    // SAFETY: the block reads and writes registers only.
    unsafe {
        asm!(
            "subs {x4}, {x4}, {x10}",
            "sbcs {x5}, {x5}, {x11}",
            "sbcs {x6}, {x6}, {x12}",
            "sbcs {x7}, {x7}, {x13}",
            "csetm {x1}, cc",
            "adds {x10}, {x4}, {x16}",
            "adcs {x11}, {x5}, {x17}",
            "adcs {x12}, {x6}, {x20}",
            "adcs {x13}, {x7}, {x21}",
            "cmp {x1}, xzr",
            "csel {x4}, {x10}, {x4}, ne",
            "csel {x5}, {x11}, {x5}, ne",
            "csel {x6}, {x12}, {x6}, ne",
            "csel {x7}, {x13}, {x7}, ne",
            x1 = out(reg) _,
            x4 = inout(reg) a[0] => r[0],
            x5 = inout(reg) a[1] => r[1],
            x6 = inout(reg) a[2] => r[2],
            x7 = inout(reg) a[3] => r[3],
            x10 = inout(reg) b[0] => _,
            x11 = inout(reg) b[1] => _,
            x12 = inout(reg) b[2] => _,
            x13 = inout(reg) b[3] => _,
            x16 = in(reg) m[0],
            x17 = in(reg) m[1],
            x20 = in(reg) m[2],
            x21 = in(reg) m[3],
            options(pure, nomem, nostack),
        );
    }
    r
}

/// `a + b` modulo the prime with `2^384 = f`, `f < 2^62`.
#[inline(always)]
pub(super) fn pseudo_add_6(a: &[u64; 6], b: &[u64; 6], f: u64) -> [u64; 6] {
    let mut r = [0; 6];
    // SAFETY: the block reads and writes registers only.
    unsafe {
        asm!(
            "adds {x4}, {x4}, {x10}",
            "adcs {x5}, {x5}, {x11}",
            "adcs {x6}, {x6}, {x12}",
            "adcs {x7}, {x7}, {x13}",
            "adcs {x8}, {x8}, {x14}",
            "adcs {x9}, {x9}, {x15}",
            "csel {x16}, {x3}, xzr, cs",
            "adds {x4}, {x4}, {x16}",
            "b.cc 2f",
            "adcs {x5}, {x5}, xzr",
            "adcs {x6}, {x6}, xzr",
            "adcs {x7}, {x7}, xzr",
            "adcs {x8}, {x8}, xzr",
            "adcs {x9}, {x9}, xzr",
            "csel {x16}, {x3}, xzr, cs",
            "add {x4}, {x4}, {x16}",
            "2:",
            x3 = in(reg) f,
            x4 = inout(reg) a[0] => r[0],
            x5 = inout(reg) a[1] => r[1],
            x6 = inout(reg) a[2] => r[2],
            x7 = inout(reg) a[3] => r[3],
            x8 = inout(reg) a[4] => r[4],
            x9 = inout(reg) a[5] => r[5],
            x10 = in(reg) b[0],
            x11 = in(reg) b[1],
            x12 = in(reg) b[2],
            x13 = in(reg) b[3],
            x14 = in(reg) b[4],
            x15 = in(reg) b[5],
            x16 = out(reg) _,
            options(pure, nomem, nostack),
        );
    }
    r
}

/// `a - b` modulo the prime with `2^384 = f`, `f < 2^62`.
#[inline(always)]
pub(super) fn pseudo_sub_6(a: &[u64; 6], b: &[u64; 6], f: u64) -> [u64; 6] {
    let mut r = [0; 6];
    // SAFETY: the block reads and writes registers only.
    unsafe {
        asm!(
            "subs {x4}, {x4}, {x10}",
            "sbcs {x5}, {x5}, {x11}",
            "sbcs {x6}, {x6}, {x12}",
            "sbcs {x7}, {x7}, {x13}",
            "sbcs {x8}, {x8}, {x14}",
            "sbcs {x9}, {x9}, {x15}",
            "csel {x16}, {x3}, xzr, cc",
            "subs {x4}, {x4}, {x16}",
            "b.cs 2f",
            "sbcs {x5}, {x5}, xzr",
            "sbcs {x6}, {x6}, xzr",
            "sbcs {x7}, {x7}, xzr",
            "sbcs {x8}, {x8}, xzr",
            "sbcs {x9}, {x9}, xzr",
            "csel {x16}, {x3}, xzr, cc",
            "subs {x4}, {x4}, {x16}",
            "sbcs {x5}, {x5}, xzr",
            "sbcs {x6}, {x6}, xzr",
            "sbcs {x7}, {x7}, xzr",
            "sbcs {x8}, {x8}, xzr",
            "sbcs {x9}, {x9}, xzr",
            "2:",
            x3 = in(reg) f,
            x4 = inout(reg) a[0] => r[0],
            x5 = inout(reg) a[1] => r[1],
            x6 = inout(reg) a[2] => r[2],
            x7 = inout(reg) a[3] => r[3],
            x8 = inout(reg) a[4] => r[4],
            x9 = inout(reg) a[5] => r[5],
            x10 = in(reg) b[0],
            x11 = in(reg) b[1],
            x12 = in(reg) b[2],
            x13 = in(reg) b[3],
            x14 = in(reg) b[4],
            x15 = in(reg) b[5],
            x16 = out(reg) _,
            options(pure, nomem, nostack),
        );
    }
    r
}

/// `a + b` modulo `m`, for `a` and `b` below `m`.
#[inline(always)]
pub(super) fn mont_add_6(a: &[u64; 6], b: &[u64; 6], m: &[u64; 6]) -> [u64; 6] {
    let mut r = [0; 6];
    // SAFETY: the block reads and writes registers only.
    unsafe {
        asm!(
            "adds {x4}, {x4}, {x10}",
            "adcs {x5}, {x5}, {x11}",
            "adcs {x6}, {x6}, {x12}",
            "adcs {x7}, {x7}, {x13}",
            "adcs {x8}, {x8}, {x14}",
            "adcs {x9}, {x9}, {x15}",
            "cset {x1}, cs",
            "subs {x10}, {x4}, {x16}",
            "sbcs {x11}, {x5}, {x17}",
            "sbcs {x12}, {x6}, {x20}",
            "sbcs {x13}, {x7}, {x21}",
            "sbcs {x14}, {x8}, {x22}",
            "sbcs {x15}, {x9}, {x23}",
            "sbcs xzr, {x1}, xzr",
            "csel {x4}, {x4}, {x10}, lo",
            "csel {x5}, {x5}, {x11}, lo",
            "csel {x6}, {x6}, {x12}, lo",
            "csel {x7}, {x7}, {x13}, lo",
            "csel {x8}, {x8}, {x14}, lo",
            "csel {x9}, {x9}, {x15}, lo",
            x1 = out(reg) _,
            x4 = inout(reg) a[0] => r[0],
            x5 = inout(reg) a[1] => r[1],
            x6 = inout(reg) a[2] => r[2],
            x7 = inout(reg) a[3] => r[3],
            x8 = inout(reg) a[4] => r[4],
            x9 = inout(reg) a[5] => r[5],
            x10 = inout(reg) b[0] => _,
            x11 = inout(reg) b[1] => _,
            x12 = inout(reg) b[2] => _,
            x13 = inout(reg) b[3] => _,
            x14 = inout(reg) b[4] => _,
            x15 = inout(reg) b[5] => _,
            x16 = in(reg) m[0],
            x17 = in(reg) m[1],
            x20 = in(reg) m[2],
            x21 = in(reg) m[3],
            x22 = in(reg) m[4],
            x23 = in(reg) m[5],
            options(pure, nomem, nostack),
        );
    }
    r
}

/// `a - b` modulo `m`, for `a` and `b` below `m`.
#[inline(always)]
pub(super) fn mont_sub_6(a: &[u64; 6], b: &[u64; 6], m: &[u64; 6]) -> [u64; 6] {
    let mut r = [0; 6];
    // SAFETY: the block reads and writes registers only.
    unsafe {
        asm!(
            "subs {x4}, {x4}, {x10}",
            "sbcs {x5}, {x5}, {x11}",
            "sbcs {x6}, {x6}, {x12}",
            "sbcs {x7}, {x7}, {x13}",
            "sbcs {x8}, {x8}, {x14}",
            "sbcs {x9}, {x9}, {x15}",
            "csetm {x1}, cc",
            "adds {x10}, {x4}, {x16}",
            "adcs {x11}, {x5}, {x17}",
            "adcs {x12}, {x6}, {x20}",
            "adcs {x13}, {x7}, {x21}",
            "adcs {x14}, {x8}, {x22}",
            "adcs {x15}, {x9}, {x23}",
            "cmp {x1}, xzr",
            "csel {x4}, {x10}, {x4}, ne",
            "csel {x5}, {x11}, {x5}, ne",
            "csel {x6}, {x12}, {x6}, ne",
            "csel {x7}, {x13}, {x7}, ne",
            "csel {x8}, {x14}, {x8}, ne",
            "csel {x9}, {x15}, {x9}, ne",
            x1 = out(reg) _,
            x4 = inout(reg) a[0] => r[0],
            x5 = inout(reg) a[1] => r[1],
            x6 = inout(reg) a[2] => r[2],
            x7 = inout(reg) a[3] => r[3],
            x8 = inout(reg) a[4] => r[4],
            x9 = inout(reg) a[5] => r[5],
            x10 = inout(reg) b[0] => _,
            x11 = inout(reg) b[1] => _,
            x12 = inout(reg) b[2] => _,
            x13 = inout(reg) b[3] => _,
            x14 = inout(reg) b[4] => _,
            x15 = inout(reg) b[5] => _,
            x16 = in(reg) m[0],
            x17 = in(reg) m[1],
            x20 = in(reg) m[2],
            x21 = in(reg) m[3],
            x22 = in(reg) m[4],
            x23 = in(reg) m[5],
            options(pure, nomem, nostack),
        );
    }
    r
}

/// `a + b` modulo the prime with `2^576 = f`, `f < 2^62`.
#[inline(always)]
pub(super) fn pseudo_add_9(a: &[u64; 9], b: &[u64; 9], f: u64) -> [u64; 9] {
    let mut r = [0; 9];
    // SAFETY: the block reads and writes registers only.
    unsafe {
        asm!(
            "adds {x4}, {x4}, {x13}",
            "adcs {x5}, {x5}, {x14}",
            "adcs {x6}, {x6}, {x15}",
            "adcs {x7}, {x7}, {x17}",
            "adcs {x8}, {x8}, {x20}",
            "adcs {x9}, {x9}, {x21}",
            "adcs {x10}, {x10}, {x22}",
            "adcs {x11}, {x11}, {x23}",
            "adcs {x12}, {x12}, {x24}",
            "csel {x16}, {x3}, xzr, cs",
            "adds {x4}, {x4}, {x16}",
            "b.cc 2f",
            "adcs {x5}, {x5}, xzr",
            "adcs {x6}, {x6}, xzr",
            "adcs {x7}, {x7}, xzr",
            "adcs {x8}, {x8}, xzr",
            "adcs {x9}, {x9}, xzr",
            "adcs {x10}, {x10}, xzr",
            "adcs {x11}, {x11}, xzr",
            "adcs {x12}, {x12}, xzr",
            "csel {x16}, {x3}, xzr, cs",
            "add {x4}, {x4}, {x16}",
            "2:",
            x3 = in(reg) f,
            x4 = inout(reg) a[0] => r[0],
            x5 = inout(reg) a[1] => r[1],
            x6 = inout(reg) a[2] => r[2],
            x7 = inout(reg) a[3] => r[3],
            x8 = inout(reg) a[4] => r[4],
            x9 = inout(reg) a[5] => r[5],
            x10 = inout(reg) a[6] => r[6],
            x11 = inout(reg) a[7] => r[7],
            x12 = inout(reg) a[8] => r[8],
            x13 = in(reg) b[0],
            x14 = in(reg) b[1],
            x15 = in(reg) b[2],
            x16 = out(reg) _,
            x17 = in(reg) b[3],
            x20 = in(reg) b[4],
            x21 = in(reg) b[5],
            x22 = in(reg) b[6],
            x23 = in(reg) b[7],
            x24 = in(reg) b[8],
            options(pure, nomem, nostack),
        );
    }
    r
}

/// `a - b` modulo the prime with `2^576 = f`, `f < 2^62`.
#[inline(always)]
pub(super) fn pseudo_sub_9(a: &[u64; 9], b: &[u64; 9], f: u64) -> [u64; 9] {
    let mut r = [0; 9];
    // SAFETY: the block reads and writes registers only.
    unsafe {
        asm!(
            "subs {x4}, {x4}, {x13}",
            "sbcs {x5}, {x5}, {x14}",
            "sbcs {x6}, {x6}, {x15}",
            "sbcs {x7}, {x7}, {x17}",
            "sbcs {x8}, {x8}, {x20}",
            "sbcs {x9}, {x9}, {x21}",
            "sbcs {x10}, {x10}, {x22}",
            "sbcs {x11}, {x11}, {x23}",
            "sbcs {x12}, {x12}, {x24}",
            "csel {x16}, {x3}, xzr, cc",
            "subs {x4}, {x4}, {x16}",
            "b.cs 2f",
            "sbcs {x5}, {x5}, xzr",
            "sbcs {x6}, {x6}, xzr",
            "sbcs {x7}, {x7}, xzr",
            "sbcs {x8}, {x8}, xzr",
            "sbcs {x9}, {x9}, xzr",
            "sbcs {x10}, {x10}, xzr",
            "sbcs {x11}, {x11}, xzr",
            "sbcs {x12}, {x12}, xzr",
            "csel {x16}, {x3}, xzr, cc",
            "subs {x4}, {x4}, {x16}",
            "sbcs {x5}, {x5}, xzr",
            "sbcs {x6}, {x6}, xzr",
            "sbcs {x7}, {x7}, xzr",
            "sbcs {x8}, {x8}, xzr",
            "sbcs {x9}, {x9}, xzr",
            "sbcs {x10}, {x10}, xzr",
            "sbcs {x11}, {x11}, xzr",
            "sbcs {x12}, {x12}, xzr",
            "2:",
            x3 = in(reg) f,
            x4 = inout(reg) a[0] => r[0],
            x5 = inout(reg) a[1] => r[1],
            x6 = inout(reg) a[2] => r[2],
            x7 = inout(reg) a[3] => r[3],
            x8 = inout(reg) a[4] => r[4],
            x9 = inout(reg) a[5] => r[5],
            x10 = inout(reg) a[6] => r[6],
            x11 = inout(reg) a[7] => r[7],
            x12 = inout(reg) a[8] => r[8],
            x13 = in(reg) b[0],
            x14 = in(reg) b[1],
            x15 = in(reg) b[2],
            x16 = out(reg) _,
            x17 = in(reg) b[3],
            x20 = in(reg) b[4],
            x21 = in(reg) b[5],
            x22 = in(reg) b[6],
            x23 = in(reg) b[7],
            x24 = in(reg) b[8],
            options(pure, nomem, nostack),
        );
    }
    r
}