ic-ec 0.3.2

//! Montgomery multiplication modulo P-384's group order, one `asm!` block
//! over `mulx`/`adcx`/`adox`.
//!
//! GENERATED by `scripts/gen_mont_x86.py`; edit that, not this.
//!
//! The same function as the generic `arith::wide::mont_mul` up to its last
//! step, which the caller takes the same way: CIOS over six words with the
//! modulus read through a RIP-relative address. Nothing branches and nothing
//! is indexed by data. The tests hold it to the generic word loop. The
//! prime's multiplication is not here: it is specialised to the prime's
//! shape, in `p384_x86_point`.

use core::arch::asm;

/// `-m^-1 mod 2^64`, by Newton iteration, as `arith::compute_neg_inv`.
const fn neg_inv(m0: u64) -> u64 {
    let mut inv = m0;
    let mut i = 0;
    while i < 6 {
        inv = inv.wrapping_mul(2u64.wrapping_sub(m0.wrapping_mul(inv)));
        i += 1;
    }
    inv.wrapping_neg()
}

/// P-384's group order, little-endian.
const P384_N_WORDS: [u64; 6] = [
    0xecec_196a_ccc5_2973,
    0x581a_0db2_48b0_a77a,
    0xc763_4d81_f437_2ddf,
    0xffff_ffff_ffff_ffff,
    0xffff_ffff_ffff_ffff,
    0xffff_ffff_ffff_ffff,
];

/// The order, for the RIP-relative reads.
static P384_N: [u64; 6] = P384_N_WORDS;

/// Montgomery multiplication modulo P-384's group order.
///
/// Returns the six words of `a * b * 2^-384 mod m`, not yet below `m`,
/// and the word above them; the value is below `2m`.
///
/// # Safety
///
/// The CPU must support BMI2 and ADX; see `super::p256_x86::has_adx`.
/// `a` and `b` must be below the modulus.
#[inline]
pub unsafe fn mul_p384_n(a: &[u64; 6], b: &[u64; 6]) -> ([u64; 6], u64) {
    let (r0, r1, r2, r3, r4, r5, top): (u64, u64, u64, u64, u64, u64, u64);
    // SAFETY: the caller guarantees the instructions exist. The block reads
    // the 48 bytes behind each of `a` and `b`, live references, and the
    // 48 bytes of `P384_N`, a static; it writes only the registers it
    // declares and touches neither the stack nor memory.
    unsafe {
        asm!(
            ".p2align 6",
            "xor {w0:e}, {w0:e}",
            "xor {w1:e}, {w1:e}",
            "xor {w2:e}, {w2:e}",
            "xor {w3:e}, {w3:e}",
            "xor {w4:e}, {w4:e}",
            "xor {w5:e}, {w5:e}",
            "xor {w6:e}, {w6:e}",
            // Row 0: window += a * b0.
            "mov rdx, qword ptr [{b} + 0]",
            "xor {w7:e}, {w7:e}",
            "mulx {c}, rax, qword ptr [{a}]",
            "adcx {w0}, rax",
            "adox {w1}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 8]",
            "adcx {w1}, rax",
            "adox {w2}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 16]",
            "adcx {w2}, rax",
            "adox {w3}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 24]",
            "adcx {w3}, rax",
            "adox {w4}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 32]",
            "adcx {w4}, rax",
            "adox {w5}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 40]",
            "adcx {w5}, rax",
            "adox {w6}, {c}",
            "mov eax, 0",
            "adcx {w6}, rax",
            "adox {w7}, rax",
            "adcx {w7}, rax",
            // Step: u = t0 * -m^-1, window += u * m; t0 becomes zero.
            "mov rdx, {ninv}",
            "imul rdx, {w0}",
            "xor eax, eax",
            "mulx {c}, rax, qword ptr [rip + {m}]",
            "adcx {w0}, rax",
            "adox {w1}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 8]",
            "adcx {w1}, rax",
            "adox {w2}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 16]",
            "adcx {w2}, rax",
            "adox {w3}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 24]",
            "adcx {w3}, rax",
            "adox {w4}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 32]",
            "adcx {w4}, rax",
            "adox {w5}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 40]",
            "adcx {w5}, rax",
            "adox {w6}, {c}",
            "mov eax, 0",
            "adcx {w6}, rax",
            "adox {w7}, rax",
            "adcx {w7}, rax",
            // Row 1: window += a * b1.
            "mov rdx, qword ptr [{b} + 8]",
            "xor {w0:e}, {w0:e}",
            "mulx {c}, rax, qword ptr [{a}]",
            "adcx {w1}, rax",
            "adox {w2}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 8]",
            "adcx {w2}, rax",
            "adox {w3}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 16]",
            "adcx {w3}, rax",
            "adox {w4}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 24]",
            "adcx {w4}, rax",
            "adox {w5}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 32]",
            "adcx {w5}, rax",
            "adox {w6}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 40]",
            "adcx {w6}, rax",
            "adox {w7}, {c}",
            "mov eax, 0",
            "adcx {w7}, rax",
            "adox {w0}, rax",
            "adcx {w0}, rax",
            // Step: u = t0 * -m^-1, window += u * m; t0 becomes zero.
            "mov rdx, {ninv}",
            "imul rdx, {w1}",
            "xor eax, eax",
            "mulx {c}, rax, qword ptr [rip + {m}]",
            "adcx {w1}, rax",
            "adox {w2}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 8]",
            "adcx {w2}, rax",
            "adox {w3}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 16]",
            "adcx {w3}, rax",
            "adox {w4}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 24]",
            "adcx {w4}, rax",
            "adox {w5}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 32]",
            "adcx {w5}, rax",
            "adox {w6}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 40]",
            "adcx {w6}, rax",
            "adox {w7}, {c}",
            "mov eax, 0",
            "adcx {w7}, rax",
            "adox {w0}, rax",
            "adcx {w0}, rax",
            // Row 2: window += a * b2.
            "mov rdx, qword ptr [{b} + 16]",
            "xor {w1:e}, {w1:e}",
            "mulx {c}, rax, qword ptr [{a}]",
            "adcx {w2}, rax",
            "adox {w3}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 8]",
            "adcx {w3}, rax",
            "adox {w4}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 16]",
            "adcx {w4}, rax",
            "adox {w5}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 24]",
            "adcx {w5}, rax",
            "adox {w6}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 32]",
            "adcx {w6}, rax",
            "adox {w7}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 40]",
            "adcx {w7}, rax",
            "adox {w0}, {c}",
            "mov eax, 0",
            "adcx {w0}, rax",
            "adox {w1}, rax",
            "adcx {w1}, rax",
            // Step: u = t0 * -m^-1, window += u * m; t0 becomes zero.
            "mov rdx, {ninv}",
            "imul rdx, {w2}",
            "xor eax, eax",
            "mulx {c}, rax, qword ptr [rip + {m}]",
            "adcx {w2}, rax",
            "adox {w3}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 8]",
            "adcx {w3}, rax",
            "adox {w4}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 16]",
            "adcx {w4}, rax",
            "adox {w5}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 24]",
            "adcx {w5}, rax",
            "adox {w6}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 32]",
            "adcx {w6}, rax",
            "adox {w7}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 40]",
            "adcx {w7}, rax",
            "adox {w0}, {c}",
            "mov eax, 0",
            "adcx {w0}, rax",
            "adox {w1}, rax",
            "adcx {w1}, rax",
            // Row 3: window += a * b3.
            "mov rdx, qword ptr [{b} + 24]",
            "xor {w2:e}, {w2:e}",
            "mulx {c}, rax, qword ptr [{a}]",
            "adcx {w3}, rax",
            "adox {w4}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 8]",
            "adcx {w4}, rax",
            "adox {w5}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 16]",
            "adcx {w5}, rax",
            "adox {w6}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 24]",
            "adcx {w6}, rax",
            "adox {w7}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 32]",
            "adcx {w7}, rax",
            "adox {w0}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 40]",
            "adcx {w0}, rax",
            "adox {w1}, {c}",
            "mov eax, 0",
            "adcx {w1}, rax",
            "adox {w2}, rax",
            "adcx {w2}, rax",
            // Step: u = t0 * -m^-1, window += u * m; t0 becomes zero.
            "mov rdx, {ninv}",
            "imul rdx, {w3}",
            "xor eax, eax",
            "mulx {c}, rax, qword ptr [rip + {m}]",
            "adcx {w3}, rax",
            "adox {w4}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 8]",
            "adcx {w4}, rax",
            "adox {w5}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 16]",
            "adcx {w5}, rax",
            "adox {w6}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 24]",
            "adcx {w6}, rax",
            "adox {w7}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 32]",
            "adcx {w7}, rax",
            "adox {w0}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 40]",
            "adcx {w0}, rax",
            "adox {w1}, {c}",
            "mov eax, 0",
            "adcx {w1}, rax",
            "adox {w2}, rax",
            "adcx {w2}, rax",
            // Row 4: window += a * b4.
            "mov rdx, qword ptr [{b} + 32]",
            "xor {w3:e}, {w3:e}",
            "mulx {c}, rax, qword ptr [{a}]",
            "adcx {w4}, rax",
            "adox {w5}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 8]",
            "adcx {w5}, rax",
            "adox {w6}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 16]",
            "adcx {w6}, rax",
            "adox {w7}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 24]",
            "adcx {w7}, rax",
            "adox {w0}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 32]",
            "adcx {w0}, rax",
            "adox {w1}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 40]",
            "adcx {w1}, rax",
            "adox {w2}, {c}",
            "mov eax, 0",
            "adcx {w2}, rax",
            "adox {w3}, rax",
            "adcx {w3}, rax",
            // Step: u = t0 * -m^-1, window += u * m; t0 becomes zero.
            "mov rdx, {ninv}",
            "imul rdx, {w4}",
            "xor eax, eax",
            "mulx {c}, rax, qword ptr [rip + {m}]",
            "adcx {w4}, rax",
            "adox {w5}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 8]",
            "adcx {w5}, rax",
            "adox {w6}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 16]",
            "adcx {w6}, rax",
            "adox {w7}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 24]",
            "adcx {w7}, rax",
            "adox {w0}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 32]",
            "adcx {w0}, rax",
            "adox {w1}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 40]",
            "adcx {w1}, rax",
            "adox {w2}, {c}",
            "mov eax, 0",
            "adcx {w2}, rax",
            "adox {w3}, rax",
            "adcx {w3}, rax",
            // Row 5: window += a * b5.
            "mov rdx, qword ptr [{b} + 40]",
            "xor {w4:e}, {w4:e}",
            "mulx {c}, rax, qword ptr [{a}]",
            "adcx {w5}, rax",
            "adox {w6}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 8]",
            "adcx {w6}, rax",
            "adox {w7}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 16]",
            "adcx {w7}, rax",
            "adox {w0}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 24]",
            "adcx {w0}, rax",
            "adox {w1}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 32]",
            "adcx {w1}, rax",
            "adox {w2}, {c}",
            "mulx {c}, rax, qword ptr [{a} + 40]",
            "adcx {w2}, rax",
            "adox {w3}, {c}",
            "mov eax, 0",
            "adcx {w3}, rax",
            "adox {w4}, rax",
            "adcx {w4}, rax",
            // Step: u = t0 * -m^-1, window += u * m; t0 becomes zero.
            "mov rdx, {ninv}",
            "imul rdx, {w5}",
            "xor eax, eax",
            "mulx {c}, rax, qword ptr [rip + {m}]",
            "adcx {w5}, rax",
            "adox {w6}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 8]",
            "adcx {w6}, rax",
            "adox {w7}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 16]",
            "adcx {w7}, rax",
            "adox {w0}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 24]",
            "adcx {w0}, rax",
            "adox {w1}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 32]",
            "adcx {w1}, rax",
            "adox {w2}, {c}",
            "mulx {c}, rax, qword ptr [rip + {m} + 40]",
            "adcx {w2}, rax",
            "adox {w3}, {c}",
            "mov eax, 0",
            "adcx {w3}, rax",
            "adox {w4}, rax",
            "adcx {w4}, rax",
            a = in(reg) a.as_ptr(),
            b = in(reg) b.as_ptr(),
            m = sym P384_N,
            ninv = const neg_inv(P384_N_WORDS[0]),
            w0 = out(reg) r2,
            w1 = out(reg) r3,
            w2 = out(reg) r4,
            w3 = out(reg) r5,
            w4 = out(reg) top,
            w5 = out(reg) _,
            w6 = out(reg) r0,
            w7 = out(reg) r1,
            c = out(reg) _,
            out("rax") _,
            out("rdx") _,
            options(pure, readonly, nostack),
        );
    }
    ([r0, r1, r2, r3, r4, r5], top)
}