rind 0.1.12

// generated source. do not edit.
#![allow(non_upper_case_globals, unused_macros, unused_imports)]
use crate::low::macros::*;

// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT-0

// ----------------------------------------------------------------------------
// AES-256-GCM bulk encryption: counter-mode encryption of whole blocks with the GHASH tag update
// Encrypts the input in CTR mode while accumulating the GHASH of the output.
//
//    extern uint64_t aes256_gcm_enc(const uint8_t *in, uint64_t len_bits, uint8_t *out,
//                              uint64_t *tag, uint8_t *ivec,
//                              const s2n_bignum_AES_KEY *key, const uint64_t *htable);
//
// Processes the input "in" of "len_bits" bits (a whole number of 16-byte
// blocks) under AES-256 in GCM mode: each block is encrypted in counter mode
// using the counter block "ivec" and the AES round keys "key", the result is
// written to "out", and the running GHASH authenticator "tag" is updated using
// the precomputed table of H-powers "htable". On return the updated counter is
// written back to "ivec", the updated authenticator to "tag", and the number
// of bytes processed is returned.
//
// The kernel is intended for a whole number of 16-byte blocks, i.e. len_bits a
// multiple of 128; a partial final block is the caller's responsibility, as in
// AWS-LC's GCM layer. For any other length only the floor(len_bits/128)
// complete blocks are processed, while the return value is still len_bits/8.
//
// The table "htable" is the GHASH key schedule used by the PMULL-based GCM
// code in OpenSSL, BoringSSL and AWS-LC: the first 96 bytes of the Htable that
// gcm_init_v8 derives from the subkey H = AES_K(0^128). Its six 16-byte entries
// hold, in order, H, m(H,H^2), H^2, H^3, m(H^3,H^4), H^4, where the powers are
// in the bit-reflected ("twisted") representation that gcm_init_v8 uses and
// m(A,B) packs the Karatsuba middle terms A.lo ^ A.hi (low 64 bits) and
// B.lo ^ B.hi (high 64 bits) of the two neighbouring powers into one entry;
// four powers suffice because GHASH is folded four blocks at a time, and
// nothing beyond the first 96 bytes is read. The reference implementation in
// tests/ref_aes_gcm.c (ref_gcm_init_htable) constructs the full table
// independently.
//
// Standard ARM ABI: X0 = in, X1 = len_bits, X2 = out, X3 = tag, X4 = ivec,
//                   X5 = key, X6 = htable, returns X0
// ----------------------------------------------------------------------------

// Origin: derived from the 'clean' (readability-oriented) AArch64 AES-GCM
// implementation by Hanno Becker, itself derived from the assembly generated by
// aesv8-gcm-armv8.pl (Fangming Fang, OpenSSL) and ultimately from ARM's
// AArch64cryptolib (Samuel Lee).  The main four-block loop is software-pipelined
// and the whole kernel rescheduled with the SLOTHY superoptimizer; the
// surrounding assembler boilerplate follows the s2n-bignum house style.

/// AES-256-GCM bulk encryption: counter-mode encryption of whole blocks with the GHASH tag update
///
/// Encrypts the input in CTR mode while accumulating the GHASH of the output.
///
/// Processes the input "in" of "len_bits" bits (a whole number of 16-byte
/// blocks) under AES-256 in GCM mode: each block is encrypted in counter mode
/// using the counter block "ivec" and the AES round keys "key", the result is
/// written to "out", and the running GHASH authenticator "tag" is updated using
/// the precomputed table of H-powers "htable". On return the updated counter is
/// written back to "ivec", the updated authenticator to "tag", and the number
/// of bytes processed is returned.
///
/// The kernel is intended for a whole number of 16-byte blocks, i.e. len_bits a
/// multiple of 128; a partial final block is the caller's responsibility, as in
/// AWS-LC's GCM layer. For any other length only the floor(len_bits/128)
/// complete blocks are processed, while the return value is still len_bits/8.
///
/// The table "htable" is the GHASH key schedule used by the PMULL-based GCM
/// code in OpenSSL, BoringSSL and AWS-LC: the first 96 bytes of the Htable that
/// gcm_init_v8 derives from the subkey H = AES_K(0^128). Its six 16-byte entries
/// hold, in order, H, m(H,H^2), H^2, H^3, m(H^3,H^4), H^4, where the powers are
/// in the bit-reflected ("twisted") representation that gcm_init_v8 uses and
/// m(A,B) packs the Karatsuba middle terms A.lo ^ A.hi (low 64 bits) and
/// B.lo ^ B.hi (high 64 bits) of the two neighbouring powers into one entry;
/// four powers suffice because GHASH is folded four blocks at a time, and
/// nothing beyond the first 96 bytes is read. The reference implementation in
/// tests/ref_aes_gcm.c (ref_gcm_init_htable) constructs the full table
/// independently.
#[target_feature(enable = "aes")]
pub(crate) unsafe fn aes256_gcm_enc(
    inout: *mut [u8; 16],
    blocks: usize,
    tag: &mut [u8; 16],
    ivec: &mut [u8; 16],
    key: &[core::arch::aarch64::uint8x16_t; 15],
    htable: &[core::arch::aarch64::uint64x2_t; 6],
) {
    // SAFETY: inline assembly. see [crate::low::inline_assembly_safety] for safety info.
    unsafe {
        core::arch::asm!(
        Q!("    sub             " "sp, sp, #0xe0"),

        Q!(Label!("Laes256_gcm_enc_preamble_start", 2) ":"),
        Q!("    stp             " "x19, x20, [sp]"),
        Q!("    stp             " "x21, x22, [sp, #16]"),
        Q!("    stp             " "x23, x24, [sp, #32]"),
        Q!("    stp             " "x25, x26, [sp, #48]"),
        Q!("    stp             " "x27, x28, [sp, #64]"),
        Q!("    stp             " "x29, x30, [sp, #80]"),
        Q!("    stp             " "d8, d9, [sp, #96]"),
        Q!("    stp             " "d10, d11, [sp, #112]"),
        Q!("    stp             " "d12, d13, [sp, #128]"),
        Q!("    stp             " "d14, d15, [sp, #144]"),
        Q!("    lsr             " "x15, x1, #3"),
        Q!("    ldr             " "q18, [x5]"),
        Q!("    ldr             " "q19, [x5, #16]"),
        Q!("    ldr             " "q20, [x5, #32]"),
        Q!("    ldr             " "q21, [x5, #48]"),
        Q!("    ldr             " "q22, [x5, #64]"),
        Q!("    ldr             " "q23, [x5, #80]"),
        Q!("    ldr             " "q24, [x5, #96]"),
        Q!("    ldr             " "q25, [x5, #112]"),
        Q!("    ldr             " "q26, [x5, #128]"),
        Q!("    ldr             " "q27, [x5, #144]"),
        Q!("    ldr             " "q28, [x5, #160]"),
        Q!("    ldr             " "q15, [x5, #176]"),
        Q!("    ldr             " "q16, [x5, #192]"),
        Q!("    ldr             " "q17, [x5, #208]"),
        Q!("    ldp             " "x20, x21, [x5, #224]"),
        Q!("    ldr             " "q30, [x3]"),
        Q!("    rev64           " "v30.16b, v30.16b"),
        Q!("    ldp             " "x11, x12, [x4]"),
        Q!("    lsr             " "x13, x12, #32"),
        Q!("    rev             " "w13, w13"),
        Q!("    orr             " "w12, w12, w12"),
        Q!("    lsr             " "x7, x15, #4"),
        Q!("    lsr             " "x1, x7, #2"),
        Q!("    and             " "x16, x7, #0x3"),
        Q!("    movi            " "v7.8b, #0xc2"),
        Q!("    shl             " "d7, d7, #56"),
        Q!("    ldr             " "q12, [x6]"),
        Q!("    ldr             " "q13, [x6, #32]"),
        Q!("    ldr             " "q14, [x6, #16]"),
        Q!("    ldr             " "q12, [x6, #48]"),
        Q!("    ldr             " "q13, [x6, #80]"),
        Q!("    ldr             " "q14, [x6, #64]"),

        Q!(Label!("Laes256_gcm_enc_preamble_end", 3) ":"),
        Q!("    cbz             " "x1, " Label!("Laes256_gcm_enc_loop_unrolled_end", 4, After)),
        Q!("    cmp             " "x1, #0x1"),
        Q!("    b.eq            " Label!("Laes256_gcm_enc_loop_unrolled_start_iter_1", 5, After)),
        Q!("    ldp             " "x23, x17, [x0, #16]"),
        Q!("    ldp             " "x8, x19, [x0, #32]"),
        Q!("    add             " "w28, w13, #0x1"),
        Q!("    eor             " "x7, x23, x20"),
        Q!("    ldp             " "x27, x30, [x0, #48]"),
        Q!("    add             " "w29, w13, #0x0"),
        Q!("    eor             " "x10, x8, x20"),
        Q!("    rev             " "w24, w29"),
        Q!("    eor             " "x14, x19, x21"),
        Q!("    add             " "w8, w13, #0x2"),
        Q!("    orr             " "x22, x12, x24, lsl #32"),
        Q!("    stp             " "x11, x22, [sp, #160]"),
        Q!("    eor             " "x26, x30, x21"),
        Q!("    rev             " "w29, w8"),
        Q!("    add             " "w8, w13, #0x3"),
        Q!("    add             " "w13, w13, #0x4"),
        Q!("    eor             " "x24, x27, x20"),
        Q!("    rev             " "w25, w8"),
        Q!("    orr             " "x30, x12, x29, lsl #32"),
        Q!("    rev             " "w29, w28"),
        Q!("    orr             " "x8, x12, x25, lsl #32"),
        Q!("    orr             " "x29, x12, x29, lsl #32"),
        Q!("    ldr             " "q3, [sp, #160]"),
        Q!("    stp             " "x11, x30, [sp, #192]"),
        Q!("    stp             " "x11, x8, [sp, #208]"),
        Q!("    stp             " "x11, x29, [sp, #176]"),
        Q!("    aese            " "v3.16b, v18.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    ldr             " "q4, [sp, #192]"),
        Q!("    stp             " "x10, x14, [sp, #192]"),
        Q!("    aese            " "v3.16b, v19.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v4.16b, v18.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    aese            " "v3.16b, v20.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v4.16b, v19.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ldp             " "x25, x10, [x0], #64"),
        Q!("    aese            " "v4.16b, v20.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ldr             " "q31, [sp, #208]"),
        Q!("    aese            " "v3.16b, v21.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v4.16b, v21.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    aese            " "v3.16b, v22.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v31.16b, v18.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "x29, x10, x21"),
        Q!("    ldr             " "q8, [sp, #176]"),
        Q!("    eor             " "x10, x25, x20"),
        Q!("    aese            " "v3.16b, v23.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v31.16b, v19.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    aese            " "v4.16b, v22.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    stp             " "x10, x29, [sp, #160]"),
        Q!("    eor             " "x29, x17, x21"),
        Q!("    aese            " "v8.16b, v18.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v4.16b, v23.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    stp             " "x7, x29, [sp, #176]"),
        Q!("    aese            " "v3.16b, v24.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v4.16b, v24.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ldr             " "q2, [x6, #80]"),
        Q!("    aese            " "v3.16b, v25.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v31.16b, v20.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    ldr             " "q13, [x6]"),
        Q!("    aese            " "v3.16b, v26.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v31.16b, v21.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    aese            " "v3.16b, v27.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v31.16b, v22.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    aese            " "v3.16b, v28.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v31.16b, v23.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    ldr             " "q9, [sp, #160]"),
        Q!("    aese            " "v4.16b, v25.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ldr             " "q12, [x6, #16]"),
        Q!("    aese            " "v3.16b, v15.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    ldr             " "q10, [sp, #192]"),
        Q!("    aese            " "v4.16b, v26.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ldr             " "q6, [x6, #64]"),
        Q!("    aese            " "v3.16b, v16.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v4.16b, v27.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ldp             " "x23, x17, [x0, #16]"),
        Q!("    aese            " "v3.16b, v17.16b"),
        Q!("    aese            " "v31.16b, v24.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    ldp             " "x8, x19, [x0, #32]"),
        Q!("    eor             " "v1.16b, v9.16b, v3.16b"),
        Q!("    aese            " "v8.16b, v19.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    add             " "w28, w13, #0x1"),
        Q!("    aese            " "v31.16b, v25.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    stp             " "x24, x26, [sp, #208]"),
        Q!("    eor             " "x7, x23, x20"),
        Q!("    aese            " "v4.16b, v28.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    rev64           " "v11.16b, v1.16b"),
        Q!("    ldp             " "x27, x30, [x0, #48]"),
        Q!("    add             " "w29, w13, #0x0"),
        Q!("    aese            " "v8.16b, v20.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    eor             " "x10, x8, x20"),
        Q!("    aese            " "v31.16b, v26.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v9.16b, v11.16b, v30.16b"),
        Q!("    rev             " "w24, w29"),
        Q!("    eor             " "x14, x19, x21"),
        Q!("    ldr             " "q30, [x6, #32]"),
        Q!("    aese            " "v8.16b, v21.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    add             " "w8, w13, #0x2"),
        Q!("    orr             " "x22, x12, x24, lsl #32"),
        Q!("    pmull2          " "v11.1q, v9.2d, v2.2d"),
        Q!("    ext             " "v5.16b, v9.16b, v9.16b, #8"),
        Q!("    stp             " "x11, x22, [sp, #160]"),
        Q!("    aese            " "v8.16b, v22.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v31.16b, v27.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    ldr             " "q0, [sp, #176]"),
        Q!("    aese            " "v4.16b, v15.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    eor             " "x26, x30, x21"),
        Q!("    aese            " "v31.16b, v28.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    aese            " "v4.16b, v16.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    rev             " "w29, w8"),
        Q!("    aese            " "v8.16b, v23.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    add             " "w8, w13, #0x3"),
        Q!("    add             " "w13, w13, #0x4"),
        Q!("    aese            " "v4.16b, v17.16b"),
        Q!("    eor             " "x24, x27, x20"),
        Q!("    aese            " "v8.16b, v24.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    rev             " "w25, w8"),
        Q!("    aese            " "v31.16b, v15.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v10.16b, v10.16b, v4.16b"),
        Q!("    orr             " "x30, x12, x29, lsl #32"),
        Q!("    rev             " "w29, w28"),
        Q!("    ldr             " "q4, [sp, #208]"),
        Q!("    aese            " "v8.16b, v25.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    orr             " "x8, x12, x25, lsl #32"),
        Q!("    aese            " "v31.16b, v16.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v5.16b, v5.16b, v9.16b"),
        Q!("    orr             " "x29, x12, x29, lsl #32"),
        Q!("    str             " "q10, [x2, #32]"),
        Q!("    aese            " "v8.16b, v26.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v31.16b, v17.16b"),
        Q!("    ldr             " "q3, [sp, #160]"),
        Q!("    pmull           " "v29.1q, v9.1d, v2.1d"),
        Q!("    rev64           " "v2.16b, v10.16b"),
        Q!("    stp             " "x11, x30, [sp, #192]"),
        Q!("    eor             " "v4.16b, v4.16b, v31.16b"),
        Q!("    aese            " "v8.16b, v27.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    pmull           " "v10.1q, v2.1d, v30.1d"),
        Q!("    ext             " "v14.16b, v2.16b, v2.16b, #8"),
        Q!("    stp             " "x11, x8, [sp, #208]"),
        Q!("    rev64           " "v9.16b, v4.16b"),
        Q!("    aese            " "v8.16b, v28.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    pmull2          " "v31.1q, v2.2d, v30.2d"),
        Q!("    eor             " "v14.16b, v14.16b, v2.16b"),
        Q!("    str             " "q4, [x2, #48]"),
        Q!("    aese            " "v8.16b, v15.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    mov             " "d4, v9.d[1]"),
        Q!("    pmull2          " "v14.1q, v14.2d, v12.2d"),
        Q!("    stp             " "x11, x29, [sp, #176]"),
        Q!("    aese            " "v8.16b, v16.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    eor             " "v30.8b, v4.8b, v9.8b"),
        Q!("    aese            " "v3.16b, v18.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v8.16b, v17.16b"),
        Q!("    ldr             " "q4, [sp, #192]"),
        Q!("    pmull           " "v30.1q, v30.1d, v12.1d"),
        Q!("    stp             " "x10, x14, [sp, #192]"),
        Q!("    pmull2          " "v2.1q, v9.2d, v13.2d"),
        Q!("    eor             " "v12.16b, v0.16b, v8.16b"),
        Q!("    ldr             " "q8, [x6, #48]"),
        Q!("    aese            " "v3.16b, v19.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    str             " "q12, [x2, #16]"),
        Q!("    aese            " "v4.16b, v18.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    str             " "q1, [x2], #64"),
        Q!("    aese            " "v3.16b, v20.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    eor             " "v30.16b, v30.16b, v14.16b"),
        Q!("    aese            " "v4.16b, v19.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ldp             " "x25, x10, [x0], #64"),
        Q!("    rev64           " "v14.16b, v12.16b"),
        Q!("    pmull           " "v12.1q, v9.1d, v13.1d"),
        Q!("    eor             " "v9.16b, v2.16b, v31.16b"),
        Q!("    aese            " "v4.16b, v20.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    pmull           " "v2.1q, v14.1d, v8.1d"),
        Q!("    ldr             " "q31, [sp, #208]"),
        Q!("    eor             " "v13.16b, v12.16b, v10.16b"),
        Q!("    aese            " "v3.16b, v21.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    pmull2          " "v10.1q, v14.2d, v8.2d"),
        Q!("    mov             " "d0, v14.d[1]"),
        Q!("    eor             " "v8.16b, v13.16b, v2.16b"),
        Q!("    aese            " "v4.16b, v21.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    eor             " "v12.8b, v0.8b, v14.8b"),
        Q!("    aese            " "v3.16b, v22.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    eor             " "v0.16b, v9.16b, v10.16b"),
        Q!("    aese            " "v31.16b, v18.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v10.16b, v8.16b, v29.16b"),
        Q!("    pmull2          " "v1.1q, v5.2d, v6.2d"),
        Q!("    eor             " "x29, x10, x21"),
        Q!("    pmull           " "v13.1q, v12.1d, v6.1d"),
        Q!("    ldr             " "q8, [sp, #176]"),
        Q!("    eor             " "x10, x25, x20"),
        Q!("    eor             " "v12.16b, v0.16b, v11.16b"),
        Q!("    aese            " "v3.16b, v23.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v31.16b, v19.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v5.16b, v10.16b, v12.16b"),
        Q!("    pmull           " "v0.1q, v12.1d, v7.1d"),
        Q!("    eor             " "v2.16b, v30.16b, v13.16b"),
        Q!("    aese            " "v4.16b, v22.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    stp             " "x10, x29, [sp, #160]"),
        Q!("    eor             " "x29, x17, x21"),
        Q!("    ext             " "v11.16b, v12.16b, v12.16b, #8"),
        Q!("    aese            " "v8.16b, v18.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    eor             " "v14.16b, v2.16b, v1.16b"),
        Q!("    aese            " "v4.16b, v23.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    stp             " "x7, x29, [sp, #176]"),
        Q!("    eor             " "v0.16b, v11.16b, v0.16b"),
        Q!("    aese            " "v3.16b, v24.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    eor             " "v5.16b, v14.16b, v5.16b"),
        Q!("    aese            " "v4.16b, v24.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ldr             " "q2, [x6, #80]"),
        Q!("    aese            " "v3.16b, v25.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    eor             " "v11.16b, v5.16b, v0.16b"),
        Q!("    aese            " "v31.16b, v20.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    ldr             " "q13, [x6]"),
        Q!("    aese            " "v3.16b, v26.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    pmull           " "v1.1q, v11.1d, v7.1d"),
        Q!("    ext             " "v12.16b, v11.16b, v11.16b, #8"),
        Q!("    aese            " "v31.16b, v21.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v11.16b, v10.16b, v1.16b"),
        Q!("    aese            " "v3.16b, v27.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v31.16b, v22.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v29.16b, v11.16b, v12.16b"),
        Q!("    aese            " "v3.16b, v28.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v31.16b, v23.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    sub             " "x1, x1, #0x2"),
        Q!("    cbz             " "x1, " Label!("Laes256_gcm_enc_loop_unrolled_start_postamble", 6, After)),

        Q!(Label!("Laes256_gcm_enc_loop_unrolled_start", 7) ":"),
        Q!("    ldr             " "q9, [sp, #160]"),
        Q!("    aese            " "v4.16b, v25.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ldr             " "q12, [x6, #16]"),
        Q!("    aese            " "v3.16b, v15.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    ldr             " "q10, [sp, #192]"),
        Q!("    aese            " "v4.16b, v26.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ldr             " "q6, [x6, #64]"),
        Q!("    aese            " "v3.16b, v16.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v4.16b, v27.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ldp             " "x23, x17, [x0, #16]"),
        Q!("    aese            " "v3.16b, v17.16b"),
        Q!("    ext             " "v30.16b, v29.16b, v29.16b, #8"),
        Q!("    aese            " "v31.16b, v24.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    ldp             " "x8, x19, [x0, #32]"),
        Q!("    eor             " "v1.16b, v9.16b, v3.16b"),
        Q!("    aese            " "v8.16b, v19.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    add             " "w28, w13, #0x1"),
        Q!("    aese            " "v31.16b, v25.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    stp             " "x24, x26, [sp, #208]"),
        Q!("    eor             " "x7, x23, x20"),
        Q!("    aese            " "v4.16b, v28.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    rev64           " "v11.16b, v1.16b"),
        Q!("    ldp             " "x27, x30, [x0, #48]"),
        Q!("    add             " "w29, w13, #0x0"),
        Q!("    aese            " "v8.16b, v20.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    eor             " "x10, x8, x20"),
        Q!("    aese            " "v31.16b, v26.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v9.16b, v11.16b, v30.16b"),
        Q!("    rev             " "w24, w29"),
        Q!("    eor             " "x14, x19, x21"),
        Q!("    ldr             " "q30, [x6, #32]"),
        Q!("    aese            " "v8.16b, v21.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    add             " "w8, w13, #0x2"),
        Q!("    orr             " "x22, x12, x24, lsl #32"),
        Q!("    pmull2          " "v11.1q, v9.2d, v2.2d"),
        Q!("    ext             " "v5.16b, v9.16b, v9.16b, #8"),
        Q!("    stp             " "x11, x22, [sp, #160]"),
        Q!("    aese            " "v8.16b, v22.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v31.16b, v27.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    ldr             " "q0, [sp, #176]"),
        Q!("    aese            " "v4.16b, v15.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    eor             " "x26, x30, x21"),
        Q!("    aese            " "v31.16b, v28.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    aese            " "v4.16b, v16.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    rev             " "w29, w8"),
        Q!("    aese            " "v8.16b, v23.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    add             " "w8, w13, #0x3"),
        Q!("    add             " "w13, w13, #0x4"),
        Q!("    aese            " "v4.16b, v17.16b"),
        Q!("    eor             " "x24, x27, x20"),
        Q!("    aese            " "v8.16b, v24.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    rev             " "w25, w8"),
        Q!("    aese            " "v31.16b, v15.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v10.16b, v10.16b, v4.16b"),
        Q!("    orr             " "x30, x12, x29, lsl #32"),
        Q!("    rev             " "w29, w28"),
        Q!("    ldr             " "q4, [sp, #208]"),
        Q!("    aese            " "v8.16b, v25.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    orr             " "x8, x12, x25, lsl #32"),
        Q!("    aese            " "v31.16b, v16.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v5.16b, v5.16b, v9.16b"),
        Q!("    orr             " "x29, x12, x29, lsl #32"),
        Q!("    str             " "q10, [x2, #32]"),
        Q!("    aese            " "v8.16b, v26.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v31.16b, v17.16b"),
        Q!("    ldr             " "q3, [sp, #160]"),
        Q!("    pmull           " "v29.1q, v9.1d, v2.1d"),
        Q!("    rev64           " "v2.16b, v10.16b"),
        Q!("    stp             " "x11, x30, [sp, #192]"),
        Q!("    eor             " "v4.16b, v4.16b, v31.16b"),
        Q!("    aese            " "v8.16b, v27.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    pmull           " "v10.1q, v2.1d, v30.1d"),
        Q!("    ext             " "v14.16b, v2.16b, v2.16b, #8"),
        Q!("    stp             " "x11, x8, [sp, #208]"),
        Q!("    rev64           " "v9.16b, v4.16b"),
        Q!("    aese            " "v8.16b, v28.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    pmull2          " "v31.1q, v2.2d, v30.2d"),
        Q!("    eor             " "v14.16b, v14.16b, v2.16b"),
        Q!("    str             " "q4, [x2, #48]"),
        Q!("    aese            " "v8.16b, v15.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    mov             " "d4, v9.d[1]"),
        Q!("    pmull2          " "v14.1q, v14.2d, v12.2d"),
        Q!("    stp             " "x11, x29, [sp, #176]"),
        Q!("    aese            " "v8.16b, v16.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    eor             " "v30.8b, v4.8b, v9.8b"),
        Q!("    aese            " "v3.16b, v18.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v8.16b, v17.16b"),
        Q!("    ldr             " "q4, [sp, #192]"),
        Q!("    pmull           " "v30.1q, v30.1d, v12.1d"),
        Q!("    stp             " "x10, x14, [sp, #192]"),
        Q!("    pmull2          " "v2.1q, v9.2d, v13.2d"),
        Q!("    eor             " "v12.16b, v0.16b, v8.16b"),
        Q!("    ldr             " "q8, [x6, #48]"),
        Q!("    aese            " "v3.16b, v19.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    str             " "q12, [x2, #16]"),
        Q!("    aese            " "v4.16b, v18.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    str             " "q1, [x2], #64"),
        Q!("    aese            " "v3.16b, v20.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    eor             " "v30.16b, v30.16b, v14.16b"),
        Q!("    aese            " "v4.16b, v19.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ldp             " "x25, x10, [x0], #64"),
        Q!("    rev64           " "v14.16b, v12.16b"),
        Q!("    pmull           " "v12.1q, v9.1d, v13.1d"),
        Q!("    eor             " "v9.16b, v2.16b, v31.16b"),
        Q!("    aese            " "v4.16b, v20.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    pmull           " "v2.1q, v14.1d, v8.1d"),
        Q!("    ldr             " "q31, [sp, #208]"),
        Q!("    eor             " "v13.16b, v12.16b, v10.16b"),
        Q!("    aese            " "v3.16b, v21.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    pmull2          " "v10.1q, v14.2d, v8.2d"),
        Q!("    mov             " "d0, v14.d[1]"),
        Q!("    eor             " "v8.16b, v13.16b, v2.16b"),
        Q!("    aese            " "v4.16b, v21.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    eor             " "v12.8b, v0.8b, v14.8b"),
        Q!("    aese            " "v3.16b, v22.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    eor             " "v0.16b, v9.16b, v10.16b"),
        Q!("    aese            " "v31.16b, v18.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v10.16b, v8.16b, v29.16b"),
        Q!("    pmull2          " "v1.1q, v5.2d, v6.2d"),
        Q!("    eor             " "x29, x10, x21"),
        Q!("    pmull           " "v13.1q, v12.1d, v6.1d"),
        Q!("    ldr             " "q8, [sp, #176]"),
        Q!("    eor             " "x10, x25, x20"),
        Q!("    eor             " "v12.16b, v0.16b, v11.16b"),
        Q!("    aese            " "v3.16b, v23.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v31.16b, v19.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v5.16b, v10.16b, v12.16b"),
        Q!("    pmull           " "v0.1q, v12.1d, v7.1d"),
        Q!("    eor             " "v2.16b, v30.16b, v13.16b"),
        Q!("    aese            " "v4.16b, v22.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    stp             " "x10, x29, [sp, #160]"),
        Q!("    eor             " "x29, x17, x21"),
        Q!("    ext             " "v11.16b, v12.16b, v12.16b, #8"),
        Q!("    aese            " "v8.16b, v18.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    eor             " "v14.16b, v2.16b, v1.16b"),
        Q!("    aese            " "v4.16b, v23.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    stp             " "x7, x29, [sp, #176]"),
        Q!("    eor             " "v0.16b, v11.16b, v0.16b"),
        Q!("    aese            " "v3.16b, v24.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    eor             " "v5.16b, v14.16b, v5.16b"),
        Q!("    aese            " "v4.16b, v24.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ldr             " "q2, [x6, #80]"),
        Q!("    aese            " "v3.16b, v25.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    eor             " "v11.16b, v5.16b, v0.16b"),
        Q!("    aese            " "v31.16b, v20.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    ldr             " "q13, [x6]"),
        Q!("    aese            " "v3.16b, v26.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    pmull           " "v1.1q, v11.1d, v7.1d"),
        Q!("    ext             " "v12.16b, v11.16b, v11.16b, #8"),
        Q!("    aese            " "v31.16b, v21.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v11.16b, v10.16b, v1.16b"),
        Q!("    aese            " "v3.16b, v27.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v31.16b, v22.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v29.16b, v11.16b, v12.16b"),
        Q!("    aese            " "v3.16b, v28.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v31.16b, v23.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    sub             " "x1, x1, #0x1"),
        Q!("    cbnz            " "x1, " Label!("Laes256_gcm_enc_loop_unrolled_start", 7, Before)),

        Q!(Label!("Laes256_gcm_enc_loop_unrolled_start_postamble", 6) ":"),
        Q!("    ldr             " "q9, [sp, #160]"),
        Q!("    aese            " "v4.16b, v25.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ldr             " "q12, [x6, #16]"),
        Q!("    aese            " "v3.16b, v15.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    ldr             " "q10, [sp, #192]"),
        Q!("    aese            " "v4.16b, v26.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ldr             " "q6, [x6, #64]"),
        Q!("    aese            " "v3.16b, v16.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v4.16b, v27.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    aese            " "v3.16b, v17.16b"),
        Q!("    ext             " "v30.16b, v29.16b, v29.16b, #8"),
        Q!("    aese            " "v31.16b, v24.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v1.16b, v9.16b, v3.16b"),
        Q!("    aese            " "v8.16b, v19.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v31.16b, v25.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    stp             " "x24, x26, [sp, #208]"),
        Q!("    aese            " "v4.16b, v28.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    rev64           " "v11.16b, v1.16b"),
        Q!("    aese            " "v8.16b, v20.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v31.16b, v26.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v9.16b, v11.16b, v30.16b"),
        Q!("    ldr             " "q30, [x6, #32]"),
        Q!("    aese            " "v8.16b, v21.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    pmull2          " "v11.1q, v9.2d, v2.2d"),
        Q!("    ext             " "v5.16b, v9.16b, v9.16b, #8"),
        Q!("    aese            " "v8.16b, v22.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v31.16b, v27.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    ldr             " "q0, [sp, #176]"),
        Q!("    aese            " "v4.16b, v15.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    aese            " "v31.16b, v28.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    aese            " "v4.16b, v16.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    aese            " "v8.16b, v23.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v4.16b, v17.16b"),
        Q!("    aese            " "v8.16b, v24.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v31.16b, v15.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v10.16b, v10.16b, v4.16b"),
        Q!("    ldr             " "q4, [sp, #208]"),
        Q!("    aese            " "v8.16b, v25.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v31.16b, v16.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v5.16b, v5.16b, v9.16b"),
        Q!("    str             " "q10, [x2, #32]"),
        Q!("    aese            " "v8.16b, v26.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v31.16b, v17.16b"),
        Q!("    pmull           " "v29.1q, v9.1d, v2.1d"),
        Q!("    rev64           " "v2.16b, v10.16b"),
        Q!("    eor             " "v4.16b, v4.16b, v31.16b"),
        Q!("    aese            " "v8.16b, v27.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    pmull           " "v10.1q, v2.1d, v30.1d"),
        Q!("    ext             " "v14.16b, v2.16b, v2.16b, #8"),
        Q!("    rev64           " "v9.16b, v4.16b"),
        Q!("    aese            " "v8.16b, v28.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    pmull2          " "v31.1q, v2.2d, v30.2d"),
        Q!("    eor             " "v14.16b, v14.16b, v2.16b"),
        Q!("    str             " "q4, [x2, #48]"),
        Q!("    aese            " "v8.16b, v15.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    mov             " "d4, v9.d[1]"),
        Q!("    pmull2          " "v14.1q, v14.2d, v12.2d"),
        Q!("    aese            " "v8.16b, v16.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    eor             " "v30.8b, v4.8b, v9.8b"),
        Q!("    aese            " "v8.16b, v17.16b"),
        Q!("    pmull           " "v30.1q, v30.1d, v12.1d"),
        Q!("    pmull2          " "v2.1q, v9.2d, v13.2d"),
        Q!("    eor             " "v12.16b, v0.16b, v8.16b"),
        Q!("    ldr             " "q8, [x6, #48]"),
        Q!("    str             " "q12, [x2, #16]"),
        Q!("    str             " "q1, [x2], #64"),
        Q!("    eor             " "v30.16b, v30.16b, v14.16b"),
        Q!("    rev64           " "v14.16b, v12.16b"),
        Q!("    pmull           " "v12.1q, v9.1d, v13.1d"),
        Q!("    eor             " "v9.16b, v2.16b, v31.16b"),
        Q!("    pmull           " "v2.1q, v14.1d, v8.1d"),
        Q!("    eor             " "v13.16b, v12.16b, v10.16b"),
        Q!("    pmull2          " "v10.1q, v14.2d, v8.2d"),
        Q!("    mov             " "d0, v14.d[1]"),
        Q!("    eor             " "v8.16b, v13.16b, v2.16b"),
        Q!("    eor             " "v12.8b, v0.8b, v14.8b"),
        Q!("    eor             " "v0.16b, v9.16b, v10.16b"),
        Q!("    eor             " "v10.16b, v8.16b, v29.16b"),
        Q!("    pmull2          " "v1.1q, v5.2d, v6.2d"),
        Q!("    pmull           " "v13.1q, v12.1d, v6.1d"),
        Q!("    eor             " "v12.16b, v0.16b, v11.16b"),
        Q!("    eor             " "v5.16b, v10.16b, v12.16b"),
        Q!("    pmull           " "v0.1q, v12.1d, v7.1d"),
        Q!("    eor             " "v2.16b, v30.16b, v13.16b"),
        Q!("    ext             " "v11.16b, v12.16b, v12.16b, #8"),
        Q!("    eor             " "v14.16b, v2.16b, v1.16b"),
        Q!("    eor             " "v0.16b, v11.16b, v0.16b"),
        Q!("    eor             " "v5.16b, v14.16b, v5.16b"),
        Q!("    eor             " "v11.16b, v5.16b, v0.16b"),
        Q!("    pmull           " "v1.1q, v11.1d, v7.1d"),
        Q!("    ext             " "v12.16b, v11.16b, v11.16b, #8"),
        Q!("    eor             " "v11.16b, v10.16b, v1.16b"),
        Q!("    eor             " "v29.16b, v11.16b, v12.16b"),
        Q!("    ext             " "v30.16b, v29.16b, v29.16b, #8"),
        Q!("    b               " Label!("Laes256_gcm_enc_loop_unrolled_end", 4, After)),

        Q!(Label!("Laes256_gcm_enc_loop_unrolled_start_iter_1", 5) ":"),
        Q!("    ldr             " "q13, [x6, #64]"),
        Q!("    ldp             " "x27, x17, [x0, #48]"),
        Q!("    add             " "w14, w13, #0x2"),
        Q!("    add             " "w19, w13, #0x3"),
        Q!("    ldp             " "x7, x22, [x0, #32]"),
        Q!("    rev             " "w14, w14"),
        Q!("    rev             " "w24, w19"),
        Q!("    orr             " "x10, x12, x24, lsl #32"),
        Q!("    orr             " "x19, x12, x14, lsl #32"),
        Q!("    add             " "w28, w13, #0x0"),
        Q!("    stp             " "x11, x19, [sp, #192]"),
        Q!("    stp             " "x11, x10, [sp, #208]"),
        Q!("    rev             " "w25, w28"),
        Q!("    add             " "w30, w13, #0x1"),
        Q!("    add             " "w13, w13, #0x4"),
        Q!("    orr             " "x19, x12, x25, lsl #32"),
        Q!("    ldr             " "q31, [sp, #208]"),
        Q!("    rev             " "w10, w30"),
        Q!("    eor             " "x23, x27, x20"),
        Q!("    stp             " "x11, x19, [sp, #160]"),
        Q!("    ldr             " "q3, [sp, #192]"),
        Q!("    eor             " "x26, x17, x21"),
        Q!("    eor             " "x29, x7, x20"),
        Q!("    ldr             " "q8, [sp, #160]"),
        Q!("    ldp             " "x17, x8, [x0, #16]"),
        Q!("    orr             " "x19, x12, x10, lsl #32"),
        Q!("    eor             " "x24, x22, x21"),
        Q!("    stp             " "x29, x24, [sp, #192]"),
        Q!("    aese            " "v31.16b, v18.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    ldp             " "x10, x25, [x0], #64"),
        Q!("    ldr             " "q6, [sp, #192]"),
        Q!("    aese            " "v3.16b, v18.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    ldr             " "q14, [x6, #32]"),
        Q!("    aese            " "v8.16b, v18.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    eor             " "x17, x17, x20"),
        Q!("    eor             " "x29, x8, x21"),
        Q!("    aese            " "v31.16b, v19.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    stp             " "x17, x29, [sp, #176]"),
        Q!("    aese            " "v8.16b, v19.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    eor             " "x22, x25, x21"),
        Q!("    eor             " "x10, x10, x20"),
        Q!("    aese            " "v3.16b, v19.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    stp             " "x10, x22, [sp, #160]"),
        Q!("    aese            " "v8.16b, v20.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    ldr             " "q1, [sp, #160]"),
        Q!("    aese            " "v3.16b, v20.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    stp             " "x23, x26, [sp, #208]"),
        Q!("    aese            " "v8.16b, v21.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    ldr             " "q10, [sp, #176]"),
        Q!("    aese            " "v3.16b, v21.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v8.16b, v22.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    stp             " "x11, x19, [sp, #176]"),
        Q!("    ldr             " "q0, [sp, #208]"),
        Q!("    aese            " "v3.16b, v22.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    ldr             " "q4, [sp, #176]"),
        Q!("    aese            " "v8.16b, v23.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    ldr             " "q29, [x6, #48]"),
        Q!("    aese            " "v3.16b, v23.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v8.16b, v24.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v3.16b, v24.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v8.16b, v25.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v3.16b, v25.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v8.16b, v26.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v3.16b, v26.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v8.16b, v27.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v3.16b, v27.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v8.16b, v28.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v3.16b, v28.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v8.16b, v15.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v3.16b, v15.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v8.16b, v16.16b"),
        Q!("    aesmc           " "v8.16b, v8.16b"),
        Q!("    aese            " "v3.16b, v16.16b"),
        Q!("    aesmc           " "v3.16b, v3.16b"),
        Q!("    aese            " "v8.16b, v17.16b"),
        Q!("    aese            " "v3.16b, v17.16b"),
        Q!("    aese            " "v31.16b, v20.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v9.16b, v1.16b, v8.16b"),
        Q!("    eor             " "v6.16b, v6.16b, v3.16b"),
        Q!("    aese            " "v4.16b, v18.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    rev64           " "v5.16b, v9.16b"),
        Q!("    aese            " "v31.16b, v21.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    str             " "q6, [x2, #32]"),
        Q!("    aese            " "v4.16b, v19.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ldr             " "q3, [x6]"),
        Q!("    aese            " "v31.16b, v22.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    aese            " "v4.16b, v20.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    eor             " "v30.16b, v5.16b, v30.16b"),
        Q!("    ldr             " "q8, [x6, #80]"),
        Q!("    aese            " "v31.16b, v23.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    aese            " "v4.16b, v21.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ext             " "v1.16b, v30.16b, v30.16b, #8"),
        Q!("    aese            " "v31.16b, v24.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    rev64           " "v2.16b, v6.16b"),
        Q!("    eor             " "v5.16b, v1.16b, v30.16b"),
        Q!("    aese            " "v4.16b, v22.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    ext             " "v12.16b, v2.16b, v2.16b, #8"),
        Q!("    aese            " "v31.16b, v25.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    aese            " "v4.16b, v23.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    aese            " "v31.16b, v26.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    eor             " "v1.16b, v12.16b, v2.16b"),
        Q!("    aese            " "v4.16b, v24.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    aese            " "v31.16b, v27.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    aese            " "v4.16b, v25.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    aese            " "v31.16b, v28.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    aese            " "v4.16b, v26.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    aese            " "v31.16b, v15.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    aese            " "v4.16b, v27.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    aese            " "v31.16b, v16.16b"),
        Q!("    aesmc           " "v31.16b, v31.16b"),
        Q!("    aese            " "v4.16b, v28.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    aese            " "v31.16b, v17.16b"),
        Q!("    aese            " "v4.16b, v15.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    pmull           " "v12.1q, v2.1d, v14.1d"),
        Q!("    eor             " "v11.16b, v0.16b, v31.16b"),
        Q!("    ldr             " "q31, [x6, #16]"),
        Q!("    aese            " "v4.16b, v16.16b"),
        Q!("    aesmc           " "v4.16b, v4.16b"),
        Q!("    pmull2          " "v6.1q, v2.2d, v14.2d"),
        Q!("    rev64           " "v2.16b, v11.16b"),
        Q!("    str             " "q11, [x2, #48]"),
        Q!("    aese            " "v4.16b, v17.16b"),
        Q!("    pmull2          " "v0.1q, v2.2d, v3.2d"),
        Q!("    mov             " "d14, v2.d[1]"),
        Q!("    eor             " "v11.16b, v10.16b, v4.16b"),
        Q!("    pmull           " "v10.1q, v2.1d, v3.1d"),
        Q!("    eor             " "v4.8b, v14.8b, v2.8b"),
        Q!("    pmull2          " "v2.1q, v1.2d, v31.2d"),
        Q!("    rev64           " "v3.16b, v11.16b"),
        Q!("    eor             " "v14.16b, v10.16b, v12.16b"),
        Q!("    pmull           " "v1.1q, v4.1d, v31.1d"),
        Q!("    eor             " "v10.16b, v0.16b, v6.16b"),
        Q!("    pmull2          " "v0.1q, v3.2d, v29.2d"),
        Q!("    mov             " "d6, v3.d[1]"),
        Q!("    pmull           " "v4.1q, v3.1d, v29.1d"),
        Q!("    eor             " "v12.16b, v1.16b, v2.16b"),
        Q!("    eor             " "v29.8b, v6.8b, v3.8b"),
        Q!("    pmull           " "v31.1q, v30.1d, v8.1d"),
        Q!("    eor             " "v2.16b, v10.16b, v0.16b"),
        Q!("    pmull2          " "v1.1q, v30.2d, v8.2d"),
        Q!("    eor             " "v14.16b, v14.16b, v4.16b"),
        Q!("    pmull           " "v10.1q, v29.1d, v13.1d"),
        Q!("    pmull2          " "v30.1q, v5.2d, v13.2d"),
        Q!("    eor             " "v5.16b, v2.16b, v1.16b"),
        Q!("    eor             " "v1.16b, v14.16b, v31.16b"),
        Q!("    eor             " "v0.16b, v12.16b, v10.16b"),
        Q!("    ext             " "v3.16b, v5.16b, v5.16b, #8"),
        Q!("    pmull           " "v4.1q, v5.1d, v7.1d"),
        Q!("    eor             " "v0.16b, v0.16b, v30.16b"),
        Q!("    eor             " "v30.16b, v1.16b, v5.16b"),
        Q!("    str             " "q11, [x2, #16]"),
        Q!("    str             " "q9, [x2], #64"),
        Q!("    eor             " "v0.16b, v0.16b, v30.16b"),
        Q!("    eor             " "v11.16b, v3.16b, v4.16b"),
        Q!("    eor             " "v11.16b, v0.16b, v11.16b"),
        Q!("    pmull           " "v13.1q, v11.1d, v7.1d"),
        Q!("    ext             " "v11.16b, v11.16b, v11.16b, #8"),
        Q!("    eor             " "v30.16b, v1.16b, v13.16b"),
        Q!("    eor             " "v11.16b, v30.16b, v11.16b"),
        Q!("    ext             " "v30.16b, v11.16b, v11.16b, #8"),

        Q!(Label!("Laes256_gcm_enc_loop_unrolled_end", 4) ":"),
        Q!("    ldr             " "q12, [x6]"),
        Q!("    ldr             " "q13, [x6, #32]"),
        Q!("    ldr             " "q14, [x6, #16]"),
        Q!("    cbz             " "x16, " Label!("Laes256_gcm_enc_loop_1x_end", 8, After)),

        Q!(Label!("Laes256_gcm_enc_loop_1x_start", 9) ":"),
        Q!("    add             " "w30, w13, #0x0"),
        Q!("    rev             " "w28, w30"),
        Q!("    orr             " "x26, x12, x28, lsl #32"),
        Q!("    add             " "w13, w13, #0x1"),
        Q!("    stp             " "x11, x26, [sp, #160]"),
        Q!("    ldr             " "q9, [sp, #160]"),
        Q!("    ldp             " "x28, x10, [x0], #16"),
        Q!("    aese            " "v9.16b, v18.16b"),
        Q!("    aesmc           " "v9.16b, v9.16b"),
        Q!("    eor             " "x7, x28, x20"),
        Q!("    aese            " "v9.16b, v19.16b"),
        Q!("    aesmc           " "v9.16b, v9.16b"),
        Q!("    eor             " "x23, x10, x21"),
        Q!("    stp             " "x7, x23, [sp, #160]"),
        Q!("    aese            " "v9.16b, v20.16b"),
        Q!("    aesmc           " "v9.16b, v9.16b"),
        Q!("    ldr             " "q1, [sp, #160]"),
        Q!("    aese            " "v9.16b, v21.16b"),
        Q!("    aesmc           " "v9.16b, v9.16b"),
        Q!("    aese            " "v9.16b, v22.16b"),
        Q!("    aesmc           " "v9.16b, v9.16b"),
        Q!("    aese            " "v9.16b, v23.16b"),
        Q!("    aesmc           " "v9.16b, v9.16b"),
        Q!("    aese            " "v9.16b, v24.16b"),
        Q!("    aesmc           " "v9.16b, v9.16b"),
        Q!("    aese            " "v9.16b, v25.16b"),
        Q!("    aesmc           " "v9.16b, v9.16b"),
        Q!("    aese            " "v9.16b, v26.16b"),
        Q!("    aesmc           " "v9.16b, v9.16b"),
        Q!("    aese            " "v9.16b, v27.16b"),
        Q!("    aesmc           " "v9.16b, v9.16b"),
        Q!("    aese            " "v9.16b, v28.16b"),
        Q!("    aesmc           " "v9.16b, v9.16b"),
        Q!("    aese            " "v9.16b, v15.16b"),
        Q!("    aesmc           " "v9.16b, v9.16b"),
        Q!("    aese            " "v9.16b, v16.16b"),
        Q!("    aesmc           " "v9.16b, v9.16b"),
        Q!("    aese            " "v9.16b, v17.16b"),
        Q!("    eor             " "v9.16b, v1.16b, v9.16b"),
        Q!("    rev64           " "v3.16b, v9.16b"),
        Q!("    str             " "q9, [x2], #16"),
        Q!("    eor             " "v1.16b, v3.16b, v30.16b"),
        Q!("    mov             " "d0, v1.d[1]"),
        Q!("    pmull2          " "v31.1q, v1.2d, v12.2d"),
        Q!("    pmull           " "v8.1q, v1.1d, v12.1d"),
        Q!("    eor             " "v29.8b, v0.8b, v1.8b"),
        Q!("    pmull           " "v11.1q, v31.1d, v7.1d"),
        Q!("    eor             " "v4.16b, v8.16b, v31.16b"),
        Q!("    ext             " "v31.16b, v31.16b, v31.16b, #8"),
        Q!("    pmull           " "v6.1q, v29.1d, v14.1d"),
        Q!("    eor             " "v31.16b, v31.16b, v11.16b"),
        Q!("    eor             " "v2.16b, v6.16b, v4.16b"),
        Q!("    eor             " "v5.16b, v2.16b, v31.16b"),
        Q!("    pmull           " "v10.1q, v5.1d, v7.1d"),
        Q!("    ext             " "v31.16b, v5.16b, v5.16b, #8"),
        Q!("    eor             " "v4.16b, v8.16b, v10.16b"),
        Q!("    eor             " "v11.16b, v4.16b, v31.16b"),
        Q!("    ext             " "v30.16b, v11.16b, v11.16b, #8"),
        Q!("    sub             " "x16, x16, #0x1"),
        Q!("    cbnz            " "x16, " Label!("Laes256_gcm_enc_loop_1x_start", 9, Before)),

        Q!(Label!("Laes256_gcm_enc_loop_1x_end", 8) ":"),
        Q!("    mov             " "x0, x15"),
        Q!("    rev64           " "v30.16b, v30.16b"),
        Q!("    str             " "q30, [x3]"),
        Q!("    rev             " "w14, w13"),
        Q!("    str             " "w14, [x4, #12]"),
        Q!("    ldp             " "d8, d9, [sp, #96]"),
        Q!("    ldp             " "d10, d11, [sp, #112]"),
        Q!("    ldp             " "d12, d13, [sp, #128]"),
        Q!("    ldp             " "d14, d15, [sp, #144]"),
        Q!("    ldp             " "x19, x20, [sp]"),
        Q!("    ldp             " "x21, x22, [sp, #16]"),
        Q!("    ldp             " "x23, x24, [sp, #32]"),
        Q!("    ldp             " "x25, x26, [sp, #48]"),
        Q!("    ldp             " "x27, x28, [sp, #64]"),
        Q!("    ldp             " "x29, x30, [sp, #80]"),

        Q!(Label!("Laes256_gcm_enc_postamble_end", 12) ":"),
        Q!("    add             " "sp, sp, #0xe0"),
        inout("x0") inout => _,
        inout("x1") blocks * 128 => _,
        inout("x2") inout => _,
        inout("x3") tag.as_mut_ptr() => _,
        inout("x4") ivec.as_mut_ptr() => _,
        inout("x5") key.as_ptr() => _,
        inout("x6") htable.as_ptr() => _,
        // clobbers
        out("v0") _,
        out("v1") _,
        out("v10") _,
        out("v11") _,
        out("v12") _,
        out("v13") _,
        out("v14") _,
        out("v15") _,
        out("v16") _,
        out("v17") _,
        out("v18") _,
        out("v19") _,
        out("v2") _,
        out("v20") _,
        out("v21") _,
        out("v22") _,
        out("v23") _,
        out("v24") _,
        out("v25") _,
        out("v26") _,
        out("v27") _,
        out("v28") _,
        out("v29") _,
        out("v3") _,
        out("v30") _,
        out("v31") _,
        out("v4") _,
        out("v5") _,
        out("v6") _,
        out("v7") _,
        out("v8") _,
        out("v9") _,
        out("x10") _,
        out("x11") _,
        out("x12") _,
        out("x13") _,
        out("x14") _,
        out("x15") _,
        out("x16") _,
        out("x17") _,
        out("x20") _,
        out("x21") _,
        out("x22") _,
        out("x23") _,
        out("x24") _,
        out("x25") _,
        out("x26") _,
        out("x27") _,
        out("x28") _,
        out("x30") _,
        out("x7") _,
        out("x8") _,
            )
    };
}