rind 0.1.2 - Docs.rs

// generated source. do not edit.
#![allow(non_upper_case_globals, unused_macros, unused_imports)]
use crate::low::macros::*;

// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT-0

// ----------------------------------------------------------------------------
// Point doubling on NIST curve P-521 in Jacobian coordinates
//
//    extern void p521_jdouble_alt(uint64_t p3[static 27],
//                                 const uint64_t p1[static 27]);
//
// Does p3 := 2 * p1 where all points are regarded as Jacobian triples.
// A Jacobian triple (x,y,z) represents affine point (x/z^2,y/z^3).
// It is assumed that all coordinates of the input point are fully
// reduced mod p_521 and that the z coordinate is not zero.
//
// Standard ARM ABI: X0 = p3, X1 = p1
// ----------------------------------------------------------------------------

// Size of individual field elements

macro_rules! NUMSIZE {
    () => {
        "72"
    };
}

// Stable homes for input arguments during main code sequence

macro_rules! input_z {
    () => {
        "x26"
    };
}
macro_rules! input_x {
    () => {
        "x27"
    };
}

// Pointer-offset pairs for inputs and outputs

macro_rules! x_1 { () => { Q!(input_x!() ", #0") } }
macro_rules! y_1 { () => { Q!(input_x!() ", # " NUMSIZE!()) } }
macro_rules! z_1 { () => { Q!(input_x!() ", # (2 * " NUMSIZE!() ")") } }

macro_rules! x_3 { () => { Q!(input_z!() ", #0") } }
macro_rules! y_3 { () => { Q!(input_z!() ", # " NUMSIZE!()) } }
macro_rules! z_3 { () => { Q!(input_z!() ", # (2 * " NUMSIZE!() ")") } }

// Pointer-offset pairs for temporaries

macro_rules! z2 { () => { Q!("sp, # (" NUMSIZE!() "* 0)") } }
macro_rules! y2 { () => { Q!("sp, # (" NUMSIZE!() "* 1)") } }
macro_rules! x2p { () => { Q!("sp, # (" NUMSIZE!() "* 2)") } }
macro_rules! xy2 { () => { Q!("sp, # (" NUMSIZE!() "* 3)") } }

macro_rules! y4 { () => { Q!("sp, # (" NUMSIZE!() "* 4)") } }
macro_rules! t2 { () => { Q!("sp, # (" NUMSIZE!() "* 4)") } }

macro_rules! dx2 { () => { Q!("sp, # (" NUMSIZE!() "* 5)") } }
macro_rules! t1 { () => { Q!("sp, # (" NUMSIZE!() "* 5)") } }

macro_rules! d { () => { Q!("sp, # (" NUMSIZE!() "* 6)") } }
macro_rules! x4p { () => { Q!("sp, # (" NUMSIZE!() "* 6)") } }

// NUMSIZE*7 is not 16-aligned so we round it up

macro_rules! NSPACE {
    () => {
        "512"
    };
}

// Corresponds exactly to bignum_mul_p521_alt

macro_rules! mul_p521 {
    ($P0:expr, $P1:expr, $P2:expr) => { Q!(
        "ldp x3, x4, [" $P1 "];\n"
        "ldp x5, x6, [" $P2 "];\n"
        "mul x15, x3, x5;\n"
        "umulh x16, x3, x5;\n"
        "mul x14, x3, x6;\n"
        "umulh x17, x3, x6;\n"
        "adds x16, x16, x14;\n"
        "ldp x7, x8, [" $P2 "+ 16];\n"
        "mul x14, x3, x7;\n"
        "umulh x19, x3, x7;\n"
        "adcs x17, x17, x14;\n"
        "mul x14, x3, x8;\n"
        "umulh x20, x3, x8;\n"
        "adcs x19, x19, x14;\n"
        "ldp x9, x10, [" $P2 "+ 32];\n"
        "mul x14, x3, x9;\n"
        "umulh x21, x3, x9;\n"
        "adcs x20, x20, x14;\n"
        "mul x14, x3, x10;\n"
        "umulh x22, x3, x10;\n"
        "adcs x21, x21, x14;\n"
        "ldp x11, x12, [" $P2 "+ 48];\n"
        "mul x14, x3, x11;\n"
        "umulh x23, x3, x11;\n"
        "adcs x22, x22, x14;\n"
        "ldr x13, [" $P2 "+ 64];\n"
        "mul x14, x3, x12;\n"
        "umulh x24, x3, x12;\n"
        "adcs x23, x23, x14;\n"
        "mul x14, x3, x13;\n"
        "umulh x1, x3, x13;\n"
        "adcs x24, x24, x14;\n"
        "adc x1, x1, xzr;\n"
        "mul x14, x4, x5;\n"
        "adds x16, x16, x14;\n"
        "mul x14, x4, x6;\n"
        "adcs x17, x17, x14;\n"
        "mul x14, x4, x7;\n"
        "adcs x19, x19, x14;\n"
        "mul x14, x4, x8;\n"
        "adcs x20, x20, x14;\n"
        "mul x14, x4, x9;\n"
        "adcs x21, x21, x14;\n"
        "mul x14, x4, x10;\n"
        "adcs x22, x22, x14;\n"
        "mul x14, x4, x11;\n"
        "adcs x23, x23, x14;\n"
        "mul x14, x4, x12;\n"
        "adcs x24, x24, x14;\n"
        "mul x14, x4, x13;\n"
        "adcs x1, x1, x14;\n"
        "cset x0, hs;\n"
        "umulh x14, x4, x5;\n"
        "adds x17, x17, x14;\n"
        "umulh x14, x4, x6;\n"
        "adcs x19, x19, x14;\n"
        "umulh x14, x4, x7;\n"
        "adcs x20, x20, x14;\n"
        "umulh x14, x4, x8;\n"
        "adcs x21, x21, x14;\n"
        "umulh x14, x4, x9;\n"
        "adcs x22, x22, x14;\n"
        "umulh x14, x4, x10;\n"
        "adcs x23, x23, x14;\n"
        "umulh x14, x4, x11;\n"
        "adcs x24, x24, x14;\n"
        "umulh x14, x4, x12;\n"
        "adcs x1, x1, x14;\n"
        "umulh x14, x4, x13;\n"
        "adc x0, x0, x14;\n"
        "stp x15, x16, [" $P0 "];\n"
        "ldp x3, x4, [" $P1 "+ 16];\n"
        "mul x14, x3, x5;\n"
        "adds x17, x17, x14;\n"
        "mul x14, x3, x6;\n"
        "adcs x19, x19, x14;\n"
        "mul x14, x3, x7;\n"
        "adcs x20, x20, x14;\n"
        "mul x14, x3, x8;\n"
        "adcs x21, x21, x14;\n"
        "mul x14, x3, x9;\n"
        "adcs x22, x22, x14;\n"
        "mul x14, x3, x10;\n"
        "adcs x23, x23, x14;\n"
        "mul x14, x3, x11;\n"
        "adcs x24, x24, x14;\n"
        "mul x14, x3, x12;\n"
        "adcs x1, x1, x14;\n"
        "mul x14, x3, x13;\n"
        "adcs x0, x0, x14;\n"
        "cset x15, hs;\n"
        "umulh x14, x3, x5;\n"
        "adds x19, x19, x14;\n"
        "umulh x14, x3, x6;\n"
        "adcs x20, x20, x14;\n"
        "umulh x14, x3, x7;\n"
        "adcs x21, x21, x14;\n"
        "umulh x14, x3, x8;\n"
        "adcs x22, x22, x14;\n"
        "umulh x14, x3, x9;\n"
        "adcs x23, x23, x14;\n"
        "umulh x14, x3, x10;\n"
        "adcs x24, x24, x14;\n"
        "umulh x14, x3, x11;\n"
        "adcs x1, x1, x14;\n"
        "umulh x14, x3, x12;\n"
        "adcs x0, x0, x14;\n"
        "umulh x14, x3, x13;\n"
        "adc x15, x15, x14;\n"
        "mul x14, x4, x5;\n"
        "adds x19, x19, x14;\n"
        "mul x14, x4, x6;\n"
        "adcs x20, x20, x14;\n"
        "mul x14, x4, x7;\n"
        "adcs x21, x21, x14;\n"
        "mul x14, x4, x8;\n"
        "adcs x22, x22, x14;\n"
        "mul x14, x4, x9;\n"
        "adcs x23, x23, x14;\n"
        "mul x14, x4, x10;\n"
        "adcs x24, x24, x14;\n"
        "mul x14, x4, x11;\n"
        "adcs x1, x1, x14;\n"
        "mul x14, x4, x12;\n"
        "adcs x0, x0, x14;\n"
        "mul x14, x4, x13;\n"
        "adcs x15, x15, x14;\n"
        "cset x16, hs;\n"
        "umulh x14, x4, x5;\n"
        "adds x20, x20, x14;\n"
        "umulh x14, x4, x6;\n"
        "adcs x21, x21, x14;\n"
        "umulh x14, x4, x7;\n"
        "adcs x22, x22, x14;\n"
        "umulh x14, x4, x8;\n"
        "adcs x23, x23, x14;\n"
        "umulh x14, x4, x9;\n"
        "adcs x24, x24, x14;\n"
        "umulh x14, x4, x10;\n"
        "adcs x1, x1, x14;\n"
        "umulh x14, x4, x11;\n"
        "adcs x0, x0, x14;\n"
        "umulh x14, x4, x12;\n"
        "adcs x15, x15, x14;\n"
        "umulh x14, x4, x13;\n"
        "adc x16, x16, x14;\n"
        "stp x17, x19, [" $P0 "+ 16];\n"
        "ldp x3, x4, [" $P1 "+ 32];\n"
        "mul x14, x3, x5;\n"
        "adds x20, x20, x14;\n"
        "mul x14, x3, x6;\n"
        "adcs x21, x21, x14;\n"
        "mul x14, x3, x7;\n"
        "adcs x22, x22, x14;\n"
        "mul x14, x3, x8;\n"
        "adcs x23, x23, x14;\n"
        "mul x14, x3, x9;\n"
        "adcs x24, x24, x14;\n"
        "mul x14, x3, x10;\n"
        "adcs x1, x1, x14;\n"
        "mul x14, x3, x11;\n"
        "adcs x0, x0, x14;\n"
        "mul x14, x3, x12;\n"
        "adcs x15, x15, x14;\n"
        "mul x14, x3, x13;\n"
        "adcs x16, x16, x14;\n"
        "cset x17, hs;\n"
        "umulh x14, x3, x5;\n"
        "adds x21, x21, x14;\n"
        "umulh x14, x3, x6;\n"
        "adcs x22, x22, x14;\n"
        "umulh x14, x3, x7;\n"
        "adcs x23, x23, x14;\n"
        "umulh x14, x3, x8;\n"
        "adcs x24, x24, x14;\n"
        "umulh x14, x3, x9;\n"
        "adcs x1, x1, x14;\n"
        "umulh x14, x3, x10;\n"
        "adcs x0, x0, x14;\n"
        "umulh x14, x3, x11;\n"
        "adcs x15, x15, x14;\n"
        "umulh x14, x3, x12;\n"
        "adcs x16, x16, x14;\n"
        "umulh x14, x3, x13;\n"
        "adc x17, x17, x14;\n"
        "mul x14, x4, x5;\n"
        "adds x21, x21, x14;\n"
        "mul x14, x4, x6;\n"
        "adcs x22, x22, x14;\n"
        "mul x14, x4, x7;\n"
        "adcs x23, x23, x14;\n"
        "mul x14, x4, x8;\n"
        "adcs x24, x24, x14;\n"
        "mul x14, x4, x9;\n"
        "adcs x1, x1, x14;\n"
        "mul x14, x4, x10;\n"
        "adcs x0, x0, x14;\n"
        "mul x14, x4, x11;\n"
        "adcs x15, x15, x14;\n"
        "mul x14, x4, x12;\n"
        "adcs x16, x16, x14;\n"
        "mul x14, x4, x13;\n"
        "adcs x17, x17, x14;\n"
        "cset x19, hs;\n"
        "umulh x14, x4, x5;\n"
        "adds x22, x22, x14;\n"
        "umulh x14, x4, x6;\n"
        "adcs x23, x23, x14;\n"
        "umulh x14, x4, x7;\n"
        "adcs x24, x24, x14;\n"
        "umulh x14, x4, x8;\n"
        "adcs x1, x1, x14;\n"
        "umulh x14, x4, x9;\n"
        "adcs x0, x0, x14;\n"
        "umulh x14, x4, x10;\n"
        "adcs x15, x15, x14;\n"
        "umulh x14, x4, x11;\n"
        "adcs x16, x16, x14;\n"
        "umulh x14, x4, x12;\n"
        "adcs x17, x17, x14;\n"
        "umulh x14, x4, x13;\n"
        "adc x19, x19, x14;\n"
        "stp x20, x21, [" $P0 "+ 32];\n"
        "ldp x3, x4, [" $P1 "+ 48];\n"
        "mul x14, x3, x5;\n"
        "adds x22, x22, x14;\n"
        "mul x14, x3, x6;\n"
        "adcs x23, x23, x14;\n"
        "mul x14, x3, x7;\n"
        "adcs x24, x24, x14;\n"
        "mul x14, x3, x8;\n"
        "adcs x1, x1, x14;\n"
        "mul x14, x3, x9;\n"
        "adcs x0, x0, x14;\n"
        "mul x14, x3, x10;\n"
        "adcs x15, x15, x14;\n"
        "mul x14, x3, x11;\n"
        "adcs x16, x16, x14;\n"
        "mul x14, x3, x12;\n"
        "adcs x17, x17, x14;\n"
        "mul x14, x3, x13;\n"
        "adcs x19, x19, x14;\n"
        "cset x20, hs;\n"
        "umulh x14, x3, x5;\n"
        "adds x23, x23, x14;\n"
        "umulh x14, x3, x6;\n"
        "adcs x24, x24, x14;\n"
        "umulh x14, x3, x7;\n"
        "adcs x1, x1, x14;\n"
        "umulh x14, x3, x8;\n"
        "adcs x0, x0, x14;\n"
        "umulh x14, x3, x9;\n"
        "adcs x15, x15, x14;\n"
        "umulh x14, x3, x10;\n"
        "adcs x16, x16, x14;\n"
        "umulh x14, x3, x11;\n"
        "adcs x17, x17, x14;\n"
        "umulh x14, x3, x12;\n"
        "adcs x19, x19, x14;\n"
        "umulh x14, x3, x13;\n"
        "adc x20, x20, x14;\n"
        "mul x14, x4, x5;\n"
        "adds x23, x23, x14;\n"
        "mul x14, x4, x6;\n"
        "adcs x24, x24, x14;\n"
        "mul x14, x4, x7;\n"
        "adcs x1, x1, x14;\n"
        "mul x14, x4, x8;\n"
        "adcs x0, x0, x14;\n"
        "mul x14, x4, x9;\n"
        "adcs x15, x15, x14;\n"
        "mul x14, x4, x10;\n"
        "adcs x16, x16, x14;\n"
        "mul x14, x4, x11;\n"
        "adcs x17, x17, x14;\n"
        "mul x14, x4, x12;\n"
        "adcs x19, x19, x14;\n"
        "mul x14, x4, x13;\n"
        "adcs x20, x20, x14;\n"
        "cset x21, hs;\n"
        "umulh x14, x4, x5;\n"
        "adds x24, x24, x14;\n"
        "umulh x14, x4, x6;\n"
        "adcs x1, x1, x14;\n"
        "umulh x14, x4, x7;\n"
        "adcs x0, x0, x14;\n"
        "umulh x14, x4, x8;\n"
        "adcs x15, x15, x14;\n"
        "umulh x14, x4, x9;\n"
        "adcs x16, x16, x14;\n"
        "umulh x14, x4, x10;\n"
        "adcs x17, x17, x14;\n"
        "umulh x14, x4, x11;\n"
        "adcs x19, x19, x14;\n"
        "umulh x14, x4, x12;\n"
        "adcs x20, x20, x14;\n"
        "umulh x14, x4, x13;\n"
        "adc x21, x21, x14;\n"
        "stp x22, x23, [" $P0 "+ 48];\n"
        "ldr x3, [" $P1 "+ 64];\n"
        "mul x14, x3, x5;\n"
        "adds x24, x24, x14;\n"
        "mul x14, x3, x6;\n"
        "adcs x1, x1, x14;\n"
        "mul x14, x3, x7;\n"
        "adcs x0, x0, x14;\n"
        "mul x14, x3, x8;\n"
        "adcs x15, x15, x14;\n"
        "mul x14, x3, x9;\n"
        "adcs x16, x16, x14;\n"
        "mul x14, x3, x10;\n"
        "adcs x17, x17, x14;\n"
        "mul x14, x3, x11;\n"
        "adcs x19, x19, x14;\n"
        "mul x14, x3, x12;\n"
        "adcs x20, x20, x14;\n"
        "mul x14, x3, x13;\n"
        "adc x21, x21, x14;\n"
        "umulh x14, x3, x5;\n"
        "adds x1, x1, x14;\n"
        "umulh x14, x3, x6;\n"
        "adcs x0, x0, x14;\n"
        "umulh x14, x3, x7;\n"
        "adcs x15, x15, x14;\n"
        "umulh x14, x3, x8;\n"
        "adcs x16, x16, x14;\n"
        "umulh x14, x3, x9;\n"
        "adcs x17, x17, x14;\n"
        "umulh x14, x3, x10;\n"
        "adcs x19, x19, x14;\n"
        "umulh x14, x3, x11;\n"
        "adcs x20, x20, x14;\n"
        "umulh x14, x3, x12;\n"
        "adc x21, x21, x14;\n"
        "cmp xzr, xzr;\n"
        "ldp x5, x6, [" $P0 "];\n"
        "extr x14, x1, x24, #9;\n"
        "adcs x5, x5, x14;\n"
        "extr x14, x0, x1, #9;\n"
        "adcs x6, x6, x14;\n"
        "ldp x7, x8, [" $P0 "+ 16];\n"
        "extr x14, x15, x0, #9;\n"
        "adcs x7, x7, x14;\n"
        "extr x14, x16, x15, #9;\n"
        "adcs x8, x8, x14;\n"
        "ldp x9, x10, [" $P0 "+ 32];\n"
        "extr x14, x17, x16, #9;\n"
        "adcs x9, x9, x14;\n"
        "extr x14, x19, x17, #9;\n"
        "adcs x10, x10, x14;\n"
        "ldp x11, x12, [" $P0 "+ 48];\n"
        "extr x14, x20, x19, #9;\n"
        "adcs x11, x11, x14;\n"
        "extr x14, x21, x20, #9;\n"
        "adcs x12, x12, x14;\n"
        "orr x13, x24, #0xfffffffffffffe00;\n"
        "lsr x14, x21, #9;\n"
        "adcs x13, x13, x14;\n"
        "sbcs x5, x5, xzr;\n"
        "sbcs x6, x6, xzr;\n"
        "sbcs x7, x7, xzr;\n"
        "sbcs x8, x8, xzr;\n"
        "sbcs x9, x9, xzr;\n"
        "sbcs x10, x10, xzr;\n"
        "sbcs x11, x11, xzr;\n"
        "sbcs x12, x12, xzr;\n"
        "sbc x13, x13, xzr;\n"
        "and x13, x13, #0x1ff;\n"
        "stp x5, x6, [" $P0 "];\n"
        "stp x7, x8, [" $P0 "+ 16];\n"
        "stp x9, x10, [" $P0 "+ 32];\n"
        "stp x11, x12, [" $P0 "+ 48];\n"
        "str x13, [" $P0 "+ 64]"
    )}
}

// Corresponds exactly to bignum_sqr_p521_alt

macro_rules! sqr_p521 {
    ($P0:expr, $P1:expr) => { Q!(
        "ldp x2, x3, [" $P1 "];\n"
        "mul x11, x2, x3;\n"
        "umulh x12, x2, x3;\n"
        "ldp x4, x5, [" $P1 "+ 16];\n"
        "mul x10, x2, x4;\n"
        "umulh x13, x2, x4;\n"
        "adds x12, x12, x10;\n"
        "ldp x6, x7, [" $P1 "+ 32];\n"
        "mul x10, x2, x5;\n"
        "umulh x14, x2, x5;\n"
        "adcs x13, x13, x10;\n"
        "ldp x8, x9, [" $P1 "+ 48];\n"
        "mul x10, x2, x6;\n"
        "umulh x15, x2, x6;\n"
        "adcs x14, x14, x10;\n"
        "mul x10, x2, x7;\n"
        "umulh x16, x2, x7;\n"
        "adcs x15, x15, x10;\n"
        "mul x10, x2, x8;\n"
        "umulh x17, x2, x8;\n"
        "adcs x16, x16, x10;\n"
        "mul x10, x2, x9;\n"
        "umulh x19, x2, x9;\n"
        "adcs x17, x17, x10;\n"
        "adc x19, x19, xzr;\n"
        "mul x10, x3, x4;\n"
        "adds x13, x13, x10;\n"
        "mul x10, x3, x5;\n"
        "adcs x14, x14, x10;\n"
        "mul x10, x3, x6;\n"
        "adcs x15, x15, x10;\n"
        "mul x10, x3, x7;\n"
        "adcs x16, x16, x10;\n"
        "mul x10, x3, x8;\n"
        "adcs x17, x17, x10;\n"
        "mul x10, x3, x9;\n"
        "adcs x19, x19, x10;\n"
        "cset x20, hs;\n"
        "umulh x10, x3, x4;\n"
        "adds x14, x14, x10;\n"
        "umulh x10, x3, x5;\n"
        "adcs x15, x15, x10;\n"
        "umulh x10, x3, x6;\n"
        "adcs x16, x16, x10;\n"
        "umulh x10, x3, x7;\n"
        "adcs x17, x17, x10;\n"
        "umulh x10, x3, x8;\n"
        "adcs x19, x19, x10;\n"
        "umulh x10, x3, x9;\n"
        "adc x20, x20, x10;\n"
        "mul x10, x6, x7;\n"
        "umulh x21, x6, x7;\n"
        "adds x20, x20, x10;\n"
        "adc x21, x21, xzr;\n"
        "mul x10, x4, x5;\n"
        "adds x15, x15, x10;\n"
        "mul x10, x4, x6;\n"
        "adcs x16, x16, x10;\n"
        "mul x10, x4, x7;\n"
        "adcs x17, x17, x10;\n"
        "mul x10, x4, x8;\n"
        "adcs x19, x19, x10;\n"
        "mul x10, x4, x9;\n"
        "adcs x20, x20, x10;\n"
        "mul x10, x6, x8;\n"
        "adcs x21, x21, x10;\n"
        "cset x22, hs;\n"
        "umulh x10, x4, x5;\n"
        "adds x16, x16, x10;\n"
        "umulh x10, x4, x6;\n"
        "adcs x17, x17, x10;\n"
        "umulh x10, x4, x7;\n"
        "adcs x19, x19, x10;\n"
        "umulh x10, x4, x8;\n"
        "adcs x20, x20, x10;\n"
        "umulh x10, x4, x9;\n"
        "adcs x21, x21, x10;\n"
        "umulh x10, x6, x8;\n"
        "adc x22, x22, x10;\n"
        "mul x10, x7, x8;\n"
        "umulh x23, x7, x8;\n"
        "adds x22, x22, x10;\n"
        "adc x23, x23, xzr;\n"
        "mul x10, x5, x6;\n"
        "adds x17, x17, x10;\n"
        "mul x10, x5, x7;\n"
        "adcs x19, x19, x10;\n"
        "mul x10, x5, x8;\n"
        "adcs x20, x20, x10;\n"
        "mul x10, x5, x9;\n"
        "adcs x21, x21, x10;\n"
        "mul x10, x6, x9;\n"
        "adcs x22, x22, x10;\n"
        "mul x10, x7, x9;\n"
        "adcs x23, x23, x10;\n"
        "cset x24, hs;\n"
        "umulh x10, x5, x6;\n"
        "adds x19, x19, x10;\n"
        "umulh x10, x5, x7;\n"
        "adcs x20, x20, x10;\n"
        "umulh x10, x5, x8;\n"
        "adcs x21, x21, x10;\n"
        "umulh x10, x5, x9;\n"
        "adcs x22, x22, x10;\n"
        "umulh x10, x6, x9;\n"
        "adcs x23, x23, x10;\n"
        "umulh x10, x7, x9;\n"
        "adc x24, x24, x10;\n"
        "mul x10, x8, x9;\n"
        "umulh x25, x8, x9;\n"
        "adds x24, x24, x10;\n"
        "adc x25, x25, xzr;\n"
        "adds x11, x11, x11;\n"
        "adcs x12, x12, x12;\n"
        "adcs x13, x13, x13;\n"
        "adcs x14, x14, x14;\n"
        "adcs x15, x15, x15;\n"
        "adcs x16, x16, x16;\n"
        "adcs x17, x17, x17;\n"
        "adcs x19, x19, x19;\n"
        "adcs x20, x20, x20;\n"
        "adcs x21, x21, x21;\n"
        "adcs x22, x22, x22;\n"
        "adcs x23, x23, x23;\n"
        "adcs x24, x24, x24;\n"
        "adcs x25, x25, x25;\n"
        "cset x0, hs;\n"
        "umulh x10, x2, x2;\n"
        "adds x11, x11, x10;\n"
        "mul x10, x3, x3;\n"
        "adcs x12, x12, x10;\n"
        "umulh x10, x3, x3;\n"
        "adcs x13, x13, x10;\n"
        "mul x10, x4, x4;\n"
        "adcs x14, x14, x10;\n"
        "umulh x10, x4, x4;\n"
        "adcs x15, x15, x10;\n"
        "mul x10, x5, x5;\n"
        "adcs x16, x16, x10;\n"
        "umulh x10, x5, x5;\n"
        "adcs x17, x17, x10;\n"
        "mul x10, x6, x6;\n"
        "adcs x19, x19, x10;\n"
        "umulh x10, x6, x6;\n"
        "adcs x20, x20, x10;\n"
        "mul x10, x7, x7;\n"
        "adcs x21, x21, x10;\n"
        "umulh x10, x7, x7;\n"
        "adcs x22, x22, x10;\n"
        "mul x10, x8, x8;\n"
        "adcs x23, x23, x10;\n"
        "umulh x10, x8, x8;\n"
        "adcs x24, x24, x10;\n"
        "mul x10, x9, x9;\n"
        "adcs x25, x25, x10;\n"
        "umulh x10, x9, x9;\n"
        "adc x0, x0, x10;\n"
        "ldr x1, [" $P1 "+ 64];\n"
        "add x1, x1, x1;\n"
        "mul x10, x1, x2;\n"
        "adds x19, x19, x10;\n"
        "umulh x10, x1, x2;\n"
        "adcs x20, x20, x10;\n"
        "mul x10, x1, x4;\n"
        "adcs x21, x21, x10;\n"
        "umulh x10, x1, x4;\n"
        "adcs x22, x22, x10;\n"
        "mul x10, x1, x6;\n"
        "adcs x23, x23, x10;\n"
        "umulh x10, x1, x6;\n"
        "adcs x24, x24, x10;\n"
        "mul x10, x1, x8;\n"
        "adcs x25, x25, x10;\n"
        "umulh x10, x1, x8;\n"
        "adcs x0, x0, x10;\n"
        "lsr x4, x1, #1;\n"
        "mul x4, x4, x4;\n"
        "adc x4, x4, xzr;\n"
        "mul x10, x1, x3;\n"
        "adds x20, x20, x10;\n"
        "umulh x10, x1, x3;\n"
        "adcs x21, x21, x10;\n"
        "mul x10, x1, x5;\n"
        "adcs x22, x22, x10;\n"
        "umulh x10, x1, x5;\n"
        "adcs x23, x23, x10;\n"
        "mul x10, x1, x7;\n"
        "adcs x24, x24, x10;\n"
        "umulh x10, x1, x7;\n"
        "adcs x25, x25, x10;\n"
        "mul x10, x1, x9;\n"
        "adcs x0, x0, x10;\n"
        "umulh x10, x1, x9;\n"
        "adc x4, x4, x10;\n"
        "mul x2, x2, x2;\n"
        "cmp xzr, xzr;\n"
        "extr x10, x20, x19, #9;\n"
        "adcs x2, x2, x10;\n"
        "extr x10, x21, x20, #9;\n"
        "adcs x11, x11, x10;\n"
        "extr x10, x22, x21, #9;\n"
        "adcs x12, x12, x10;\n"
        "extr x10, x23, x22, #9;\n"
        "adcs x13, x13, x10;\n"
        "extr x10, x24, x23, #9;\n"
        "adcs x14, x14, x10;\n"
        "extr x10, x25, x24, #9;\n"
        "adcs x15, x15, x10;\n"
        "extr x10, x0, x25, #9;\n"
        "adcs x16, x16, x10;\n"
        "extr x10, x4, x0, #9;\n"
        "adcs x17, x17, x10;\n"
        "orr x19, x19, #0xfffffffffffffe00;\n"
        "lsr x10, x4, #9;\n"
        "adcs x19, x19, x10;\n"
        "sbcs x2, x2, xzr;\n"
        "sbcs x11, x11, xzr;\n"
        "sbcs x12, x12, xzr;\n"
        "sbcs x13, x13, xzr;\n"
        "sbcs x14, x14, xzr;\n"
        "sbcs x15, x15, xzr;\n"
        "sbcs x16, x16, xzr;\n"
        "sbcs x17, x17, xzr;\n"
        "sbc x19, x19, xzr;\n"
        "and x19, x19, #0x1ff;\n"
        "stp x2, x11, [" $P0 "];\n"
        "stp x12, x13, [" $P0 "+ 16];\n"
        "stp x14, x15, [" $P0 "+ 32];\n"
        "stp x16, x17, [" $P0 "+ 48];\n"
        "str x19, [" $P0 "+ 64]"
    )}
}

// Corresponds exactly to bignum_add_p521

macro_rules! add_p521 {
    ($P0:expr, $P1:expr, $P2:expr) => { Q!(
        "cmp xzr, xzr;\n"
        "ldp x5, x6, [" $P1 "];\n"
        "ldp x4, x3, [" $P2 "];\n"
        "adcs x5, x5, x4;\n"
        "adcs x6, x6, x3;\n"
        "ldp x7, x8, [" $P1 "+ 16];\n"
        "ldp x4, x3, [" $P2 "+ 16];\n"
        "adcs x7, x7, x4;\n"
        "adcs x8, x8, x3;\n"
        "ldp x9, x10, [" $P1 "+ 32];\n"
        "ldp x4, x3, [" $P2 "+ 32];\n"
        "adcs x9, x9, x4;\n"
        "adcs x10, x10, x3;\n"
        "ldp x11, x12, [" $P1 "+ 48];\n"
        "ldp x4, x3, [" $P2 "+ 48];\n"
        "adcs x11, x11, x4;\n"
        "adcs x12, x12, x3;\n"
        "ldr x13, [" $P1 "+ 64];\n"
        "ldr x4, [" $P2 "+ 64];\n"
        "adc x13, x13, x4;\n"
        "subs x4, x13, #512;\n"
        "csetm x4, hs;\n"
        "sbcs x5, x5, xzr;\n"
        "and x4, x4, #0x200;\n"
        "sbcs x6, x6, xzr;\n"
        "sbcs x7, x7, xzr;\n"
        "sbcs x8, x8, xzr;\n"
        "sbcs x9, x9, xzr;\n"
        "sbcs x10, x10, xzr;\n"
        "sbcs x11, x11, xzr;\n"
        "sbcs x12, x12, xzr;\n"
        "sbc x13, x13, x4;\n"
        "stp x5, x6, [" $P0 "];\n"
        "stp x7, x8, [" $P0 "+ 16];\n"
        "stp x9, x10, [" $P0 "+ 32];\n"
        "stp x11, x12, [" $P0 "+ 48];\n"
        "str x13, [" $P0 "+ 64]"
    )}
}

// Corresponds exactly to bignum_sub_p521

macro_rules! sub_p521 {
    ($P0:expr, $P1:expr, $P2:expr) => { Q!(
        "ldp x5, x6, [" $P1 "];\n"
        "ldp x4, x3, [" $P2 "];\n"
        "subs x5, x5, x4;\n"
        "sbcs x6, x6, x3;\n"
        "ldp x7, x8, [" $P1 "+ 16];\n"
        "ldp x4, x3, [" $P2 "+ 16];\n"
        "sbcs x7, x7, x4;\n"
        "sbcs x8, x8, x3;\n"
        "ldp x9, x10, [" $P1 "+ 32];\n"
        "ldp x4, x3, [" $P2 "+ 32];\n"
        "sbcs x9, x9, x4;\n"
        "sbcs x10, x10, x3;\n"
        "ldp x11, x12, [" $P1 "+ 48];\n"
        "ldp x4, x3, [" $P2 "+ 48];\n"
        "sbcs x11, x11, x4;\n"
        "sbcs x12, x12, x3;\n"
        "ldr x13, [" $P1 "+ 64];\n"
        "ldr x4, [" $P2 "+ 64];\n"
        "sbcs x13, x13, x4;\n"
        "sbcs x5, x5, xzr;\n"
        "sbcs x6, x6, xzr;\n"
        "sbcs x7, x7, xzr;\n"
        "sbcs x8, x8, xzr;\n"
        "sbcs x9, x9, xzr;\n"
        "sbcs x10, x10, xzr;\n"
        "sbcs x11, x11, xzr;\n"
        "sbcs x12, x12, xzr;\n"
        "sbcs x13, x13, xzr;\n"
        "and x13, x13, #0x1ff;\n"
        "stp x5, x6, [" $P0 "];\n"
        "stp x7, x8, [" $P0 "+ 16];\n"
        "stp x9, x10, [" $P0 "+ 32];\n"
        "stp x11, x12, [" $P0 "+ 48];\n"
        "str x13, [" $P0 "+ 64]"
    )}
}

// Weak multiplication not fully reducing

macro_rules! weakmul_p521 {
    ($P0:expr, $P1:expr, $P2:expr) => { Q!(
        "ldp x3, x4, [" $P1 "];\n"
        "ldp x5, x6, [" $P2 "];\n"
        "mul x15, x3, x5;\n"
        "umulh x16, x3, x5;\n"
        "mul x14, x3, x6;\n"
        "umulh x17, x3, x6;\n"
        "adds x16, x16, x14;\n"
        "ldp x7, x8, [" $P2 "+ 16];\n"
        "mul x14, x3, x7;\n"
        "umulh x19, x3, x7;\n"
        "adcs x17, x17, x14;\n"
        "mul x14, x3, x8;\n"
        "umulh x20, x3, x8;\n"
        "adcs x19, x19, x14;\n"
        "ldp x9, x10, [" $P2 "+ 32];\n"
        "mul x14, x3, x9;\n"
        "umulh x21, x3, x9;\n"
        "adcs x20, x20, x14;\n"
        "mul x14, x3, x10;\n"
        "umulh x22, x3, x10;\n"
        "adcs x21, x21, x14;\n"
        "ldp x11, x12, [" $P2 "+ 48];\n"
        "mul x14, x3, x11;\n"
        "umulh x23, x3, x11;\n"
        "adcs x22, x22, x14;\n"
        "ldr x13, [" $P2 "+ 64];\n"
        "mul x14, x3, x12;\n"
        "umulh x24, x3, x12;\n"
        "adcs x23, x23, x14;\n"
        "mul x14, x3, x13;\n"
        "umulh x1, x3, x13;\n"
        "adcs x24, x24, x14;\n"
        "adc x1, x1, xzr;\n"
        "mul x14, x4, x5;\n"
        "adds x16, x16, x14;\n"
        "mul x14, x4, x6;\n"
        "adcs x17, x17, x14;\n"
        "mul x14, x4, x7;\n"
        "adcs x19, x19, x14;\n"
        "mul x14, x4, x8;\n"
        "adcs x20, x20, x14;\n"
        "mul x14, x4, x9;\n"
        "adcs x21, x21, x14;\n"
        "mul x14, x4, x10;\n"
        "adcs x22, x22, x14;\n"
        "mul x14, x4, x11;\n"
        "adcs x23, x23, x14;\n"
        "mul x14, x4, x12;\n"
        "adcs x24, x24, x14;\n"
        "mul x14, x4, x13;\n"
        "adcs x1, x1, x14;\n"
        "cset x0, hs;\n"
        "umulh x14, x4, x5;\n"
        "adds x17, x17, x14;\n"
        "umulh x14, x4, x6;\n"
        "adcs x19, x19, x14;\n"
        "umulh x14, x4, x7;\n"
        "adcs x20, x20, x14;\n"
        "umulh x14, x4, x8;\n"
        "adcs x21, x21, x14;\n"
        "umulh x14, x4, x9;\n"
        "adcs x22, x22, x14;\n"
        "umulh x14, x4, x10;\n"
        "adcs x23, x23, x14;\n"
        "umulh x14, x4, x11;\n"
        "adcs x24, x24, x14;\n"
        "umulh x14, x4, x12;\n"
        "adcs x1, x1, x14;\n"
        "umulh x14, x4, x13;\n"
        "adc x0, x0, x14;\n"
        "stp x15, x16, [" $P0 "];\n"
        "ldp x3, x4, [" $P1 "+ 16];\n"
        "mul x14, x3, x5;\n"
        "adds x17, x17, x14;\n"
        "mul x14, x3, x6;\n"
        "adcs x19, x19, x14;\n"
        "mul x14, x3, x7;\n"
        "adcs x20, x20, x14;\n"
        "mul x14, x3, x8;\n"
        "adcs x21, x21, x14;\n"
        "mul x14, x3, x9;\n"
        "adcs x22, x22, x14;\n"
        "mul x14, x3, x10;\n"
        "adcs x23, x23, x14;\n"
        "mul x14, x3, x11;\n"
        "adcs x24, x24, x14;\n"
        "mul x14, x3, x12;\n"
        "adcs x1, x1, x14;\n"
        "mul x14, x3, x13;\n"
        "adcs x0, x0, x14;\n"
        "cset x15, hs;\n"
        "umulh x14, x3, x5;\n"
        "adds x19, x19, x14;\n"
        "umulh x14, x3, x6;\n"
        "adcs x20, x20, x14;\n"
        "umulh x14, x3, x7;\n"
        "adcs x21, x21, x14;\n"
        "umulh x14, x3, x8;\n"
        "adcs x22, x22, x14;\n"
        "umulh x14, x3, x9;\n"
        "adcs x23, x23, x14;\n"
        "umulh x14, x3, x10;\n"
        "adcs x24, x24, x14;\n"
        "umulh x14, x3, x11;\n"
        "adcs x1, x1, x14;\n"
        "umulh x14, x3, x12;\n"
        "adcs x0, x0, x14;\n"
        "umulh x14, x3, x13;\n"
        "adc x15, x15, x14;\n"
        "mul x14, x4, x5;\n"
        "adds x19, x19, x14;\n"
        "mul x14, x4, x6;\n"
        "adcs x20, x20, x14;\n"
        "mul x14, x4, x7;\n"
        "adcs x21, x21, x14;\n"
        "mul x14, x4, x8;\n"
        "adcs x22, x22, x14;\n"
        "mul x14, x4, x9;\n"
        "adcs x23, x23, x14;\n"
        "mul x14, x4, x10;\n"
        "adcs x24, x24, x14;\n"
        "mul x14, x4, x11;\n"
        "adcs x1, x1, x14;\n"
        "mul x14, x4, x12;\n"
        "adcs x0, x0, x14;\n"
        "mul x14, x4, x13;\n"
        "adcs x15, x15, x14;\n"
        "cset x16, hs;\n"
        "umulh x14, x4, x5;\n"
        "adds x20, x20, x14;\n"
        "umulh x14, x4, x6;\n"
        "adcs x21, x21, x14;\n"
        "umulh x14, x4, x7;\n"
        "adcs x22, x22, x14;\n"
        "umulh x14, x4, x8;\n"
        "adcs x23, x23, x14;\n"
        "umulh x14, x4, x9;\n"
        "adcs x24, x24, x14;\n"
        "umulh x14, x4, x10;\n"
        "adcs x1, x1, x14;\n"
        "umulh x14, x4, x11;\n"
        "adcs x0, x0, x14;\n"
        "umulh x14, x4, x12;\n"
        "adcs x15, x15, x14;\n"
        "umulh x14, x4, x13;\n"
        "adc x16, x16, x14;\n"
        "stp x17, x19, [" $P0 "+ 16];\n"
        "ldp x3, x4, [" $P1 "+ 32];\n"
        "mul x14, x3, x5;\n"
        "adds x20, x20, x14;\n"
        "mul x14, x3, x6;\n"
        "adcs x21, x21, x14;\n"
        "mul x14, x3, x7;\n"
        "adcs x22, x22, x14;\n"
        "mul x14, x3, x8;\n"
        "adcs x23, x23, x14;\n"
        "mul x14, x3, x9;\n"
        "adcs x24, x24, x14;\n"
        "mul x14, x3, x10;\n"
        "adcs x1, x1, x14;\n"
        "mul x14, x3, x11;\n"
        "adcs x0, x0, x14;\n"
        "mul x14, x3, x12;\n"
        "adcs x15, x15, x14;\n"
        "mul x14, x3, x13;\n"
        "adcs x16, x16, x14;\n"
        "cset x17, hs;\n"
        "umulh x14, x3, x5;\n"
        "adds x21, x21, x14;\n"
        "umulh x14, x3, x6;\n"
        "adcs x22, x22, x14;\n"
        "umulh x14, x3, x7;\n"
        "adcs x23, x23, x14;\n"
        "umulh x14, x3, x8;\n"
        "adcs x24, x24, x14;\n"
        "umulh x14, x3, x9;\n"
        "adcs x1, x1, x14;\n"
        "umulh x14, x3, x10;\n"
        "adcs x0, x0, x14;\n"
        "umulh x14, x3, x11;\n"
        "adcs x15, x15, x14;\n"
        "umulh x14, x3, x12;\n"
        "adcs x16, x16, x14;\n"
        "umulh x14, x3, x13;\n"
        "adc x17, x17, x14;\n"
        "mul x14, x4, x5;\n"
        "adds x21, x21, x14;\n"
        "mul x14, x4, x6;\n"
        "adcs x22, x22, x14;\n"
        "mul x14, x4, x7;\n"
        "adcs x23, x23, x14;\n"
        "mul x14, x4, x8;\n"
        "adcs x24, x24, x14;\n"
        "mul x14, x4, x9;\n"
        "adcs x1, x1, x14;\n"
        "mul x14, x4, x10;\n"
        "adcs x0, x0, x14;\n"
        "mul x14, x4, x11;\n"
        "adcs x15, x15, x14;\n"
        "mul x14, x4, x12;\n"
        "adcs x16, x16, x14;\n"
        "mul x14, x4, x13;\n"
        "adcs x17, x17, x14;\n"
        "cset x19, hs;\n"
        "umulh x14, x4, x5;\n"
        "adds x22, x22, x14;\n"
        "umulh x14, x4, x6;\n"
        "adcs x23, x23, x14;\n"
        "umulh x14, x4, x7;\n"
        "adcs x24, x24, x14;\n"
        "umulh x14, x4, x8;\n"
        "adcs x1, x1, x14;\n"
        "umulh x14, x4, x9;\n"
        "adcs x0, x0, x14;\n"
        "umulh x14, x4, x10;\n"
        "adcs x15, x15, x14;\n"
        "umulh x14, x4, x11;\n"
        "adcs x16, x16, x14;\n"
        "umulh x14, x4, x12;\n"
        "adcs x17, x17, x14;\n"
        "umulh x14, x4, x13;\n"
        "adc x19, x19, x14;\n"
        "stp x20, x21, [" $P0 "+ 32];\n"
        "ldp x3, x4, [" $P1 "+ 48];\n"
        "mul x14, x3, x5;\n"
        "adds x22, x22, x14;\n"
        "mul x14, x3, x6;\n"
        "adcs x23, x23, x14;\n"
        "mul x14, x3, x7;\n"
        "adcs x24, x24, x14;\n"
        "mul x14, x3, x8;\n"
        "adcs x1, x1, x14;\n"
        "mul x14, x3, x9;\n"
        "adcs x0, x0, x14;\n"
        "mul x14, x3, x10;\n"
        "adcs x15, x15, x14;\n"
        "mul x14, x3, x11;\n"
        "adcs x16, x16, x14;\n"
        "mul x14, x3, x12;\n"
        "adcs x17, x17, x14;\n"
        "mul x14, x3, x13;\n"
        "adcs x19, x19, x14;\n"
        "cset x20, hs;\n"
        "umulh x14, x3, x5;\n"
        "adds x23, x23, x14;\n"
        "umulh x14, x3, x6;\n"
        "adcs x24, x24, x14;\n"
        "umulh x14, x3, x7;\n"
        "adcs x1, x1, x14;\n"
        "umulh x14, x3, x8;\n"
        "adcs x0, x0, x14;\n"
        "umulh x14, x3, x9;\n"
        "adcs x15, x15, x14;\n"
        "umulh x14, x3, x10;\n"
        "adcs x16, x16, x14;\n"
        "umulh x14, x3, x11;\n"
        "adcs x17, x17, x14;\n"
        "umulh x14, x3, x12;\n"
        "adcs x19, x19, x14;\n"
        "umulh x14, x3, x13;\n"
        "adc x20, x20, x14;\n"
        "mul x14, x4, x5;\n"
        "adds x23, x23, x14;\n"
        "mul x14, x4, x6;\n"
        "adcs x24, x24, x14;\n"
        "mul x14, x4, x7;\n"
        "adcs x1, x1, x14;\n"
        "mul x14, x4, x8;\n"
        "adcs x0, x0, x14;\n"
        "mul x14, x4, x9;\n"
        "adcs x15, x15, x14;\n"
        "mul x14, x4, x10;\n"
        "adcs x16, x16, x14;\n"
        "mul x14, x4, x11;\n"
        "adcs x17, x17, x14;\n"
        "mul x14, x4, x12;\n"
        "adcs x19, x19, x14;\n"
        "mul x14, x4, x13;\n"
        "adcs x20, x20, x14;\n"
        "cset x21, hs;\n"
        "umulh x14, x4, x5;\n"
        "adds x24, x24, x14;\n"
        "umulh x14, x4, x6;\n"
        "adcs x1, x1, x14;\n"
        "umulh x14, x4, x7;\n"
        "adcs x0, x0, x14;\n"
        "umulh x14, x4, x8;\n"
        "adcs x15, x15, x14;\n"
        "umulh x14, x4, x9;\n"
        "adcs x16, x16, x14;\n"
        "umulh x14, x4, x10;\n"
        "adcs x17, x17, x14;\n"
        "umulh x14, x4, x11;\n"
        "adcs x19, x19, x14;\n"
        "umulh x14, x4, x12;\n"
        "adcs x20, x20, x14;\n"
        "umulh x14, x4, x13;\n"
        "adc x21, x21, x14;\n"
        "stp x22, x23, [" $P0 "+ 48];\n"
        "ldr x3, [" $P1 "+ 64];\n"
        "mul x14, x3, x5;\n"
        "adds x24, x24, x14;\n"
        "mul x14, x3, x6;\n"
        "adcs x1, x1, x14;\n"
        "mul x14, x3, x7;\n"
        "adcs x0, x0, x14;\n"
        "mul x14, x3, x8;\n"
        "adcs x15, x15, x14;\n"
        "mul x14, x3, x9;\n"
        "adcs x16, x16, x14;\n"
        "mul x14, x3, x10;\n"
        "adcs x17, x17, x14;\n"
        "mul x14, x3, x11;\n"
        "adcs x19, x19, x14;\n"
        "mul x14, x3, x12;\n"
        "adcs x20, x20, x14;\n"
        "mul x14, x3, x13;\n"
        "adc x21, x21, x14;\n"
        "umulh x14, x3, x5;\n"
        "adds x1, x1, x14;\n"
        "umulh x14, x3, x6;\n"
        "adcs x0, x0, x14;\n"
        "umulh x14, x3, x7;\n"
        "adcs x15, x15, x14;\n"
        "umulh x14, x3, x8;\n"
        "adcs x16, x16, x14;\n"
        "umulh x14, x3, x9;\n"
        "adcs x17, x17, x14;\n"
        "umulh x14, x3, x10;\n"
        "adcs x19, x19, x14;\n"
        "umulh x14, x3, x11;\n"
        "adcs x20, x20, x14;\n"
        "umulh x14, x3, x12;\n"
        "adc x21, x21, x14;\n"
        "ldp x5, x6, [" $P0 "];\n"
        "extr x14, x1, x24, #9;\n"
        "adds x5, x5, x14;\n"
        "extr x14, x0, x1, #9;\n"
        "adcs x6, x6, x14;\n"
        "ldp x7, x8, [" $P0 "+ 16];\n"
        "extr x14, x15, x0, #9;\n"
        "adcs x7, x7, x14;\n"
        "extr x14, x16, x15, #9;\n"
        "adcs x8, x8, x14;\n"
        "ldp x9, x10, [" $P0 "+ 32];\n"
        "extr x14, x17, x16, #9;\n"
        "adcs x9, x9, x14;\n"
        "extr x14, x19, x17, #9;\n"
        "adcs x10, x10, x14;\n"
        "ldp x11, x12, [" $P0 "+ 48];\n"
        "extr x14, x20, x19, #9;\n"
        "adcs x11, x11, x14;\n"
        "extr x14, x21, x20, #9;\n"
        "adcs x12, x12, x14;\n"
        "and x13, x24, #0x1ff;\n"
        "lsr x14, x21, #9;\n"
        "adc x13, x13, x14;\n"
        "stp x5, x6, [" $P0 "];\n"
        "stp x7, x8, [" $P0 "+ 16];\n"
        "stp x9, x10, [" $P0 "+ 32];\n"
        "stp x11, x12, [" $P0 "+ 48];\n"
        "str x13, [" $P0 "+ 64]"
    )}
}

// P0 = C * P1 - D * P2 == C * P1 + D * (p_521 - P2)

macro_rules! cmsub_p521 {
    ($P0:expr, $C:expr, $P1:expr, $D:expr, $P2:expr) => { Q!(
        "ldp x6, x7, [" $P1 "];\n"
        "mov x1, # (" $C ");\n"
        "mul x3, x1, x6;\n"
        "mul x4, x1, x7;\n"
        "umulh x6, x1, x6;\n"
        "adds x4, x4, x6;\n"
        "umulh x7, x1, x7;\n"
        "ldp x8, x9, [" $P1 "+ 16];\n"
        "mul x5, x1, x8;\n"
        "mul x6, x1, x9;\n"
        "umulh x8, x1, x8;\n"
        "adcs x5, x5, x7;\n"
        "umulh x9, x1, x9;\n"
        "adcs x6, x6, x8;\n"
        "ldp x10, x11, [" $P1 "+ 32];\n"
        "mul x7, x1, x10;\n"
        "mul x8, x1, x11;\n"
        "umulh x10, x1, x10;\n"
        "adcs x7, x7, x9;\n"
        "umulh x11, x1, x11;\n"
        "adcs x8, x8, x10;\n"
        "ldp x12, x13, [" $P1 "+ 48];\n"
        "mul x9, x1, x12;\n"
        "mul x10, x1, x13;\n"
        "umulh x12, x1, x12;\n"
        "adcs x9, x9, x11;\n"
        "umulh x13, x1, x13;\n"
        "adcs x10, x10, x12;\n"
        "ldr x14, [" $P1 "+ 64];\n"
        "mul x11, x1, x14;\n"
        "adc x11, x11, x13;\n"
        "mov x1, # (" $D ");\n"
        "ldp x20, x21, [" $P2 "];\n"
        "mvn x20, x20;\n"
        "mul x0, x1, x20;\n"
        "umulh x20, x1, x20;\n"
        "adds x3, x3, x0;\n"
        "mvn x21, x21;\n"
        "mul x0, x1, x21;\n"
        "umulh x21, x1, x21;\n"
        "adcs x4, x4, x0;\n"
        "ldp x22, x23, [" $P2 "+ 16];\n"
        "mvn x22, x22;\n"
        "mul x0, x1, x22;\n"
        "umulh x22, x1, x22;\n"
        "adcs x5, x5, x0;\n"
        "mvn x23, x23;\n"
        "mul x0, x1, x23;\n"
        "umulh x23, x1, x23;\n"
        "adcs x6, x6, x0;\n"
        "ldp x17, x19, [" $P2 "+ 32];\n"
        "mvn x17, x17;\n"
        "mul x0, x1, x17;\n"
        "umulh x17, x1, x17;\n"
        "adcs x7, x7, x0;\n"
        "mvn x19, x19;\n"
        "mul x0, x1, x19;\n"
        "umulh x19, x1, x19;\n"
        "adcs x8, x8, x0;\n"
        "ldp x2, x16, [" $P2 "+ 48];\n"
        "mvn x2, x2;\n"
        "mul x0, x1, x2;\n"
        "umulh x2, x1, x2;\n"
        "adcs x9, x9, x0;\n"
        "mvn x16, x16;\n"
        "mul x0, x1, x16;\n"
        "umulh x16, x1, x16;\n"
        "adcs x10, x10, x0;\n"
        "ldr x0, [" $P2 "+ 64];\n"
        "eor x0, x0, #0x1ff;\n"
        "mul x0, x1, x0;\n"
        "adc x11, x11, x0;\n"
        "adds x4, x4, x20;\n"
        "adcs x5, x5, x21;\n"
        "and x15, x4, x5;\n"
        "adcs x6, x6, x22;\n"
        "and x15, x15, x6;\n"
        "adcs x7, x7, x23;\n"
        "and x15, x15, x7;\n"
        "adcs x8, x8, x17;\n"
        "and x15, x15, x8;\n"
        "adcs x9, x9, x19;\n"
        "and x15, x15, x9;\n"
        "adcs x10, x10, x2;\n"
        "and x15, x15, x10;\n"
        "adc x11, x11, x16;\n"
        "lsr x12, x11, #9;\n"
        "orr x11, x11, #0xfffffffffffffe00;\n"
        "cmp xzr, xzr;\n"
        "adcs xzr, x3, x12;\n"
        "adcs xzr, x15, xzr;\n"
        "adcs xzr, x11, xzr;\n"
        "adcs x3, x3, x12;\n"
        "adcs x4, x4, xzr;\n"
        "adcs x5, x5, xzr;\n"
        "adcs x6, x6, xzr;\n"
        "adcs x7, x7, xzr;\n"
        "adcs x8, x8, xzr;\n"
        "adcs x9, x9, xzr;\n"
        "adcs x10, x10, xzr;\n"
        "adc x11, x11, xzr;\n"
        "and x11, x11, #0x1ff;\n"
        "stp x3, x4, [" $P0 "];\n"
        "stp x5, x6, [" $P0 "+ 16];\n"
        "stp x7, x8, [" $P0 "+ 32];\n"
        "stp x9, x10, [" $P0 "+ 48];\n"
        "str x11, [" $P0 "+ 64]"
    )}
}

// P0 = 3 * P1 - 8 * P2 == 3 * P1 + 8 * (p_521 - P2)

macro_rules! cmsub38_p521 {
    ($P0:expr, $P1:expr, $P2:expr) => { Q!(
        "ldp x6, x7, [" $P1 "];\n"
        "lsl x3, x6, #1;\n"
        "adds x3, x3, x6;\n"
        "extr x4, x7, x6, #63;\n"
        "adcs x4, x4, x7;\n"
        "ldp x8, x9, [" $P1 "+ 16];\n"
        "extr x5, x8, x7, #63;\n"
        "adcs x5, x5, x8;\n"
        "extr x6, x9, x8, #63;\n"
        "adcs x6, x6, x9;\n"
        "ldp x10, x11, [" $P1 "+ 32];\n"
        "extr x7, x10, x9, #63;\n"
        "adcs x7, x7, x10;\n"
        "extr x8, x11, x10, #63;\n"
        "adcs x8, x8, x11;\n"
        "ldp x12, x13, [" $P1 "+ 48];\n"
        "extr x9, x12, x11, #63;\n"
        "adcs x9, x9, x12;\n"
        "extr x10, x13, x12, #63;\n"
        "adcs x10, x10, x13;\n"
        "ldr x14, [" $P1 "+ 64];\n"
        "extr x11, x14, x13, #63;\n"
        "adc x11, x11, x14;\n"
        "ldp x20, x21, [" $P2 "];\n"
        "mvn x20, x20;\n"
        "lsl x0, x20, #3;\n"
        "adds x3, x3, x0;\n"
        "mvn x21, x21;\n"
        "extr x0, x21, x20, #61;\n"
        "adcs x4, x4, x0;\n"
        "ldp x22, x23, [" $P2 "+ 16];\n"
        "mvn x22, x22;\n"
        "extr x0, x22, x21, #61;\n"
        "adcs x5, x5, x0;\n"
        "and x15, x4, x5;\n"
        "mvn x23, x23;\n"
        "extr x0, x23, x22, #61;\n"
        "adcs x6, x6, x0;\n"
        "and x15, x15, x6;\n"
        "ldp x20, x21, [" $P2 "+ 32];\n"
        "mvn x20, x20;\n"
        "extr x0, x20, x23, #61;\n"
        "adcs x7, x7, x0;\n"
        "and x15, x15, x7;\n"
        "mvn x21, x21;\n"
        "extr x0, x21, x20, #61;\n"
        "adcs x8, x8, x0;\n"
        "and x15, x15, x8;\n"
        "ldp x22, x23, [" $P2 "+ 48];\n"
        "mvn x22, x22;\n"
        "extr x0, x22, x21, #61;\n"
        "adcs x9, x9, x0;\n"
        "and x15, x15, x9;\n"
        "mvn x23, x23;\n"
        "extr x0, x23, x22, #61;\n"
        "adcs x10, x10, x0;\n"
        "and x15, x15, x10;\n"
        "ldr x0, [" $P2 "+ 64];\n"
        "eor x0, x0, #0x1ff;\n"
        "extr x0, x0, x23, #61;\n"
        "adc x11, x11, x0;\n"
        "lsr x12, x11, #9;\n"
        "orr x11, x11, #0xfffffffffffffe00;\n"
        "cmp xzr, xzr;\n"
        "adcs xzr, x3, x12;\n"
        "adcs xzr, x15, xzr;\n"
        "adcs xzr, x11, xzr;\n"
        "adcs x3, x3, x12;\n"
        "adcs x4, x4, xzr;\n"
        "adcs x5, x5, xzr;\n"
        "adcs x6, x6, xzr;\n"
        "adcs x7, x7, xzr;\n"
        "adcs x8, x8, xzr;\n"
        "adcs x9, x9, xzr;\n"
        "adcs x10, x10, xzr;\n"
        "adc x11, x11, xzr;\n"
        "and x11, x11, #0x1ff;\n"
        "stp x3, x4, [" $P0 "];\n"
        "stp x5, x6, [" $P0 "+ 16];\n"
        "stp x7, x8, [" $P0 "+ 32];\n"
        "stp x9, x10, [" $P0 "+ 48];\n"
        "str x11, [" $P0 "+ 64]"
    )}
}

// P0 = 4 * P1 - P2 = 4 * P1 + (p_521 - P2)

macro_rules! cmsub41_p521 {
    ($P0:expr, $P1:expr, $P2:expr) => { Q!(
        "ldp x6, x7, [" $P1 "];\n"
        "lsl x3, x6, #2;\n"
        "extr x4, x7, x6, #62;\n"
        "ldp x8, x9, [" $P1 "+ 16];\n"
        "extr x5, x8, x7, #62;\n"
        "extr x6, x9, x8, #62;\n"
        "ldp x10, x11, [" $P1 "+ 32];\n"
        "extr x7, x10, x9, #62;\n"
        "extr x8, x11, x10, #62;\n"
        "ldp x12, x13, [" $P1 "+ 48];\n"
        "extr x9, x12, x11, #62;\n"
        "extr x10, x13, x12, #62;\n"
        "ldr x14, [" $P1 "+ 64];\n"
        "extr x11, x14, x13, #62;\n"
        "ldp x0, x1, [" $P2 "];\n"
        "mvn x0, x0;\n"
        "adds x3, x3, x0;\n"
        "sbcs x4, x4, x1;\n"
        "ldp x0, x1, [" $P2 "+ 16];\n"
        "sbcs x5, x5, x0;\n"
        "and x15, x4, x5;\n"
        "sbcs x6, x6, x1;\n"
        "and x15, x15, x6;\n"
        "ldp x0, x1, [" $P2 "+ 32];\n"
        "sbcs x7, x7, x0;\n"
        "and x15, x15, x7;\n"
        "sbcs x8, x8, x1;\n"
        "and x15, x15, x8;\n"
        "ldp x0, x1, [" $P2 "+ 48];\n"
        "sbcs x9, x9, x0;\n"
        "and x15, x15, x9;\n"
        "sbcs x10, x10, x1;\n"
        "and x15, x15, x10;\n"
        "ldr x0, [" $P2 "+ 64];\n"
        "eor x0, x0, #0x1ff;\n"
        "adc x11, x11, x0;\n"
        "lsr x12, x11, #9;\n"
        "orr x11, x11, #0xfffffffffffffe00;\n"
        "cmp xzr, xzr;\n"
        "adcs xzr, x3, x12;\n"
        "adcs xzr, x15, xzr;\n"
        "adcs xzr, x11, xzr;\n"
        "adcs x3, x3, x12;\n"
        "adcs x4, x4, xzr;\n"
        "adcs x5, x5, xzr;\n"
        "adcs x6, x6, xzr;\n"
        "adcs x7, x7, xzr;\n"
        "adcs x8, x8, xzr;\n"
        "adcs x9, x9, xzr;\n"
        "adcs x10, x10, xzr;\n"
        "adc x11, x11, xzr;\n"
        "and x11, x11, #0x1ff;\n"
        "stp x3, x4, [" $P0 "];\n"
        "stp x5, x6, [" $P0 "+ 16];\n"
        "stp x7, x8, [" $P0 "+ 32];\n"
        "stp x9, x10, [" $P0 "+ 48];\n"
        "str x11, [" $P0 "+ 64]"
    )}
}

/// Point doubling on NIST curve P-521 in Jacobian coordinates
///
///
/// Does p3 := 2 * p1 where all points are regarded as Jacobian triples.
/// A Jacobian triple (x,y,z) represents affine point (x/z^2,y/z^3).
/// It is assumed that all coordinates of the input point are fully
/// reduced mod p_521 and that the z coordinate is not zero.
pub(crate) fn p521_jdouble(p3: &mut [u64; 27], p1: &[u64; 27]) {
    // SAFETY: inline assembly. see [crate::low::inline_assembly_safety] for safety info.
    unsafe {
        core::arch::asm!(

        // Save regs and make room on stack for temporary variables

        Q!("    stp             " "x19, x20, [sp, #-16] !"),
        Q!("    stp             " "x21, x22, [sp, #-16] !"),
        Q!("    stp             " "x23, x24, [sp, #-16] !"),
        Q!("    stp             " "x25, x26, [sp, #-16] !"),
        Q!("    stp             " "x27, x28, [sp, #-16] !"),
        Q!("    sub             " "sp, sp, # (" NSPACE!() "+ 0)"),

        // Move the input arguments to stable places

        Q!("    mov             " input_z!() ", x0"),
        Q!("    mov             " input_x!() ", x1"),

        // Main code, just a sequence of basic field operations

        // z2 = z^2
        // y2 = y^2

        sqr_p521!(z2!(), z_1!()),
        sqr_p521!(y2!(), y_1!()),

        // x2p = x^2 - z^4 = (x + z^2) * (x - z^2)

        add_p521!(t1!(), x_1!(), z2!()),
        sub_p521!(t2!(), x_1!(), z2!()),
        mul_p521!(x2p!(), t1!(), t2!()),

        // t1 = y + z
        // x4p = x2p^2
        // xy2 = x * y^2

        add_p521!(t1!(), y_1!(), z_1!()),
        sqr_p521!(x4p!(), x2p!()),
        weakmul_p521!(xy2!(), x_1!(), y2!()),

        // t2 = (y + z)^2

        sqr_p521!(t2!(), t1!()),

        // d = 12 * xy2 - 9 * x4p
        // t1 = y^2 + 2 * y * z

        cmsub_p521!(d!(), "12", xy2!(), "9", x4p!()),
        sub_p521!(t1!(), t2!(), z2!()),

        // y4 = y^4

        sqr_p521!(y4!(), y2!()),

        // z_3' = 2 * y * z
        // dx2 = d * x2p

        sub_p521!(z_3!(), t1!(), y2!()),
        weakmul_p521!(dx2!(), d!(), x2p!()),

        // x' = 4 * xy2 - d

        cmsub41_p521!(x_3!(), xy2!(), d!()),

        // y' = 3 * dx2 - 8 * y4

        cmsub38_p521!(y_3!(), dx2!(), y4!()),

        // Restore stack and registers

        Q!("    add             " "sp, sp, # (" NSPACE!() "+ 0)"),

        Q!("    ldp             " "x27, x28, [sp], #16"),
        Q!("    ldp             " "x25, x26, [sp], #16"),
        Q!("    ldp             " "x23, x24, [sp], #16"),
        Q!("    ldp             " "x21, x22, [sp], #16"),
        Q!("    ldp             " "x19, x20, [sp], #16"),

        inout("x0") p3.as_mut_ptr() => _,
        inout("x1") p1.as_ptr() => _,
        // clobbers
        out("x10") _,
        out("x11") _,
        out("x12") _,
        out("x13") _,
        out("x14") _,
        out("x15") _,
        out("x16") _,
        out("x17") _,
        out("x2") _,
        out("x20") _,
        out("x21") _,
        out("x22") _,
        out("x23") _,
        out("x24") _,
        out("x25") _,
        out("x26") _,
        out("x27") _,
        out("x28") _,
        out("x3") _,
        out("x4") _,
        out("x5") _,
        out("x6") _,
        out("x7") _,
        out("x8") _,
        out("x9") _,
            )
    };
}