hardware_rust_crypto/aes_gcm/

mod.rs

//! Candidate AES-256-GCM API for the hardware-only `RustCrypto` fork.
//!
//! The implementation uses vendored hardware-only AES and GHASH paths on
//! supported `x86_64` and `aarch64` targets, with no software AES fallback
//! compiled into the reusable key state. See the repository `NOTICE` file for
//! upstream attribution of the vendored backends.
//!
//! # Constant-time notes
//!
//! All secret-dependent computation happens in the hardware AES and
//! carryless-multiply backends (see the `aes` and `ghash` module docs) or in
//! straight-line XOR/copy loops whose trip counts derive from public
//! lengths. Control flow in this module branches only on public values:
//! input and buffer lengths, CPU feature availability, and the accept/reject
//! outcome of the tag check - which is itself computed in constant time via
//! `subtle` and is public by definition (the caller learns it from the
//! `Result`). Nonces, counter blocks, and lengths are not secret in GCM;
//! keystream and tag material never feed a branch condition or memory index.
//! Data-independent instruction timing is taken as a hardware-vendor
//! guarantee; see `docs/design.md` for the full constant-time basis.

#![allow(unsafe_code)]

mod aes;
mod fork;
mod ghash;
mod nonce;

use core::{
    marker::PhantomData,
    mem::MaybeUninit,
    ptr::{self, NonNull},
};
use subtle::ConstantTimeEq as _;
use zeroize::Zeroize as _;

/// AES-256 key length in bytes.
pub const KEY_SIZE: usize = 32;
/// GCM nonce length in bytes.
pub const NONCE_SIZE: usize = 12;
/// GCM authentication tag length in bytes.
pub const TAG_SIZE: usize = 16;
const AES_BLOCK_SIZE: usize = 16;
const PAR_BLOCKS: usize = aes::PAR_BLOCKS;
const PAR_BYTES: usize = PAR_BLOCKS * AES_BLOCK_SIZE;
const MAX_GCM_DATA_LEN: u64 = ((u32::MAX as u64) - 1) * AES_BLOCK_SIZE as u64;
const MAX_GHASH_INPUT_LEN: u64 = u64::MAX / 8;

/// Opaque key-state layout reported to callers before they allocate storage.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct KeyStateLayout {
    /// Required storage size in bytes.
    pub size: usize,
    /// Required storage alignment in bytes.
    pub align: usize,
}

/// Error type for the candidate AES-256-GCM API.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum Error {
    /// The provided key is not exactly 32 bytes.
    InvalidKeyLength,
    /// The provided nonce is not exactly 12 bytes.
    InvalidNonceLength,
    /// Encryption failed.
    Encrypt,
    /// Decryption or authentication failed.
    Decrypt,
    /// The nonce-appended ciphertext layout is too short to contain tag plus nonce.
    CiphertextTooShort,
    /// Required AES/GHASH hardware support is unavailable.
    UnsupportedCpu,
    /// Caller-provided key-state storage is too small.
    KeyStateStorageTooSmall,
    /// Caller-provided key-state storage does not satisfy alignment.
    KeyStateStorageMisaligned,
    /// Input is too large for AES-GCM's counter or GHASH length limits.
    InputTooLarge,
    /// Caller-provided output buffer is too small for the plaintext.
    OutputTooSmall,
    /// The OS entropy source failed while generating a nonce.
    OsEntropy,
}

impl std::fmt::Display for Error {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Self::InvalidKeyLength => f.write_str("invalid AES-256-GCM key length"),
            Self::InvalidNonceLength => f.write_str("invalid AES-GCM nonce length"),
            Self::Encrypt => f.write_str("AES-256-GCM encryption failed"),
            Self::Decrypt => f.write_str("AES-256-GCM authentication failed"),
            Self::CiphertextTooShort => f.write_str("ciphertext too short"),
            Self::UnsupportedCpu => {
                f.write_str("required AES-256-GCM hardware support is unavailable")
            }
            Self::KeyStateStorageTooSmall => f.write_str("key-state storage too small"),
            Self::KeyStateStorageMisaligned => f.write_str("key-state storage misaligned"),
            Self::InputTooLarge => f.write_str("AES-256-GCM input too large"),
            Self::OutputTooSmall => f.write_str("plaintext output buffer too small"),
            Self::OsEntropy => f.write_str("OS entropy source failed during nonce generation"),
        }
    }
}

impl std::error::Error for Error {}

struct KeyState {
    aes: aes::Aes256,
    ghash: ghash::GHashKey,
}

impl KeyState {
    fn init_in_place(dst: NonNull<Self>, key: &[u8; KEY_SIZE]) -> Result<(), Error> {
        if !HardwareAes256Gcm::hardware_available() {
            return Err(Error::UnsupportedCpu);
        }

        let dst = dst.as_ptr();
        // SAFETY: dst points to valid writable KeyState storage supplied by the
        // caller. The field pointer stays within that allocation.
        let aes_ptr = unsafe { ptr::addr_of_mut!((*dst).aes) };
        aes::Aes256::init_in_place(aes_ptr, key).ok_or(Error::UnsupportedCpu)?;

        let mut hash_subkey = [0_u8; TAG_SIZE];
        // SAFETY: aes_ptr was initialized successfully above and remains live.
        unsafe { (&*aes_ptr).encrypt_block(&mut hash_subkey) };
        // SAFETY: dst points to valid writable KeyState storage supplied by the
        // caller. The field pointer stays within that allocation.
        let ghash_ptr = unsafe { ptr::addr_of_mut!((*dst).ghash) };
        if ghash::GHashKey::init_in_place(ghash_ptr, &mut hash_subkey).is_none() {
            // Unreachable: hardware_available() covered GHASH features above.
            // Defensively wipe the AES state already expanded in caller
            // storage before reporting failure.
            hash_subkey.zeroize();
            // SAFETY: aes_ptr was initialized above and is not used again.
            unsafe { ptr::drop_in_place(aes_ptr) };
            return Err(Error::UnsupportedCpu);
        }
        Ok(())
    }

    fn encrypt(&self, nonce: &[u8], aad: &[u8], plaintext: &[u8]) -> Result<Vec<u8>, Error> {
        let total = plaintext
            .len()
            .checked_add(TAG_SIZE)
            .ok_or(Error::InputTooLarge)?;
        let mut out = vec![0_u8; total];
        self.encrypt_to(nonce, aad, plaintext, &mut out)?;
        Ok(out)
    }

    fn encrypt_nonce_appended(&self, nonce: &[u8], plaintext: &[u8]) -> Result<Vec<u8>, Error> {
        let nonce = nonce_from_slice(nonce)?;
        validate_gcm_lengths(0, plaintext.len())?;
        let total = plaintext
            .len()
            .checked_add(TAG_SIZE + NONCE_SIZE)
            .ok_or(Error::InputTooLarge)?;
        let mut out = Vec::with_capacity(total);
        out.extend_from_slice(plaintext);
        let Some(tag) = self.seal_in_place(&nonce, &[], &mut out) else {
            out.zeroize();
            return Err(Error::Encrypt);
        };
        append_tag_nonce(&mut out, &tag, &nonce);
        Ok(out)
    }

    fn encrypt_nonce_appended_in_place(
        &self,
        nonce: &[u8],
        in_out: &mut Vec<u8>,
    ) -> Result<(), Error> {
        let nonce = nonce_from_slice(nonce)?;
        validate_gcm_lengths(0, in_out.len())?;
        let total = in_out
            .len()
            .checked_add(TAG_SIZE + NONCE_SIZE)
            .ok_or(Error::InputTooLarge)?;
        if in_out.capacity() < total {
            in_out.reserve_exact(total - in_out.len());
        }
        let Some(tag) = self.seal_in_place(&nonce, &[], in_out.as_mut_slice()) else {
            in_out.zeroize();
            return Err(Error::Encrypt);
        };
        append_tag_nonce(in_out, &tag, &nonce);
        Ok(())
    }

    fn encrypt_to(
        &self,
        nonce: &[u8],
        aad: &[u8],
        plaintext: &[u8],
        out: &mut [u8],
    ) -> Result<usize, Error> {
        let nonce = nonce_from_slice(nonce)?;
        validate_gcm_lengths(aad.len(), plaintext.len())?;
        let total = plaintext
            .len()
            .checked_add(TAG_SIZE)
            .ok_or(Error::InputTooLarge)?;
        if out.len() < total {
            return Err(Error::OutputTooSmall);
        }

        let (ciphertext, rest) = out.split_at_mut(plaintext.len());
        let tag = self
            .seal(&nonce, aad, plaintext, ciphertext)
            .ok_or(Error::Encrypt)?;
        rest[..TAG_SIZE].copy_from_slice(&tag);
        Ok(total)
    }

    /// Fused encrypt-and-authenticate: each ciphertext chunk feeds GHASH as
    /// soon as it is produced, so the message is traversed once.
    ///
    /// Constant time with respect to secrets: chunking and tail handling
    /// branch only on the public message length; the keystream is combined
    /// by straight-line XOR.
    fn seal(
        &self,
        nonce: &[u8; NONCE_SIZE],
        aad: &[u8],
        plaintext: &[u8],
        ciphertext: &mut [u8],
    ) -> Option<[u8; TAG_SIZE]> {
        debug_assert_eq!(plaintext.len(), ciphertext.len());
        let mut ghasher = ghash::Ghasher::new(&self.ghash)?;
        ghasher.absorb_padded(aad);

        let mut counter = j0(nonce);
        increment_counter(&mut counter);

        // The bulk region's AES keystream and GHASH run in one fused,
        // software-pipelined loop so the AES and carryless-multiply pipelines
        // overlap; the sub-batch tail is handled serially.
        let bulk = (plaintext.len() / PAR_BYTES) * PAR_BYTES;
        let (plaintext_bulk, plaintext_tail) = plaintext.split_at(bulk);
        let (ciphertext_bulk, ciphertext_tail) = ciphertext.split_at_mut(bulk);
        if !plaintext_bulk.is_empty() {
            ghasher.seal_bulk(
                self.aes.round_keys(),
                &mut counter,
                plaintext_bulk,
                ciphertext_bulk,
            );
        }
        if !plaintext_tail.is_empty() {
            ciphertext_tail.copy_from_slice(plaintext_tail);
            self.apply_ctr_serial(&mut counter, ciphertext_tail);
            ghasher.absorb_padded(ciphertext_tail);
        }

        let mut tag = ghasher.finalize(aad.len(), plaintext.len())?;
        let mut mask = j0(nonce);
        self.aes.encrypt_block(&mut mask);
        for (tag_byte, mask_byte) in tag.iter_mut().zip(mask.iter()) {
            *tag_byte ^= mask_byte;
        }
        Some(tag)
    }

    /// In-place encrypt-and-authenticate. `data` enters as plaintext and exits
    /// as ciphertext, so allocating APIs can avoid zero-filling an output Vec
    /// before immediately overwriting it.
    fn seal_in_place(
        &self,
        nonce: &[u8; NONCE_SIZE],
        aad: &[u8],
        data: &mut [u8],
    ) -> Option<[u8; TAG_SIZE]> {
        let mut ghasher = ghash::Ghasher::new(&self.ghash)?;
        ghasher.absorb_padded(aad);

        let mut counter = j0(nonce);
        increment_counter(&mut counter);

        let bulk = (data.len() / PAR_BYTES) * PAR_BYTES;
        let (data_bulk, data_tail) = data.split_at_mut(bulk);
        if !data_bulk.is_empty() {
            ghasher.seal_in_place_bulk(self.aes.round_keys(), &mut counter, data_bulk);
        }
        if !data_tail.is_empty() {
            let full = (data_tail.len() / AES_BLOCK_SIZE) * AES_BLOCK_SIZE;
            let (data_tail_blocks, data_partial) = data_tail.split_at_mut(full);
            if !data_tail_blocks.is_empty() {
                ghasher.seal_in_place_tail_blocks(
                    self.aes.round_keys(),
                    &mut counter,
                    data_tail_blocks,
                );
            }
            if !data_partial.is_empty() {
                self.apply_ctr_serial(&mut counter, data_partial);
                ghasher.absorb_padded(data_partial);
            }
        }

        let mut tag = ghasher.finalize(aad.len(), data.len())?;
        let mut mask = j0(nonce);
        self.aes.encrypt_block(&mut mask);
        for (tag_byte, mask_byte) in tag.iter_mut().zip(mask.iter()) {
            *tag_byte ^= mask_byte;
        }
        Some(tag)
    }

    /// Single-block in-place CTR for sub-batch tails.
    fn apply_ctr_serial(&self, counter: &mut [u8; AES_BLOCK_SIZE], data: &mut [u8]) {
        let mut keystream = [0_u8; AES_BLOCK_SIZE];
        for block in data.chunks_mut(AES_BLOCK_SIZE) {
            keystream.copy_from_slice(counter);
            self.aes.encrypt_block(&mut keystream);
            for (byte, key_byte) in block.iter_mut().zip(keystream.iter()) {
                *byte ^= key_byte;
            }
            increment_counter(counter);
        }
    }

    fn decrypt(
        &self,
        nonce: &[u8],
        aad: &[u8],
        ciphertext_and_tag: &[u8],
    ) -> Result<Vec<u8>, Error> {
        let nonce = nonce_from_slice(nonce)?;
        if ciphertext_and_tag.len() < TAG_SIZE {
            return Err(Error::Decrypt);
        }

        let tag_pos = ciphertext_and_tag.len() - TAG_SIZE;
        let (ciphertext, tag) = ciphertext_and_tag.split_at(tag_pos);
        validate_gcm_lengths(aad.len(), ciphertext.len())?;
        let mut out = Vec::with_capacity(ciphertext.len());
        out.extend_from_slice(ciphertext);
        let Some(expected) = self.open_in_place(&nonce, aad, &mut out) else {
            out.zeroize();
            return Err(Error::Decrypt);
        };
        if !constant_time_eq(&expected, tag) {
            out.zeroize();
            return Err(Error::Decrypt);
        }
        Ok(out)
    }

    fn decrypt_to(
        &self,
        nonce: &[u8],
        aad: &[u8],
        ciphertext_and_tag: &[u8],
        out: &mut [u8],
    ) -> Result<usize, Error> {
        let nonce = nonce_from_slice(nonce)?;
        if ciphertext_and_tag.len() < TAG_SIZE {
            return Err(Error::Decrypt);
        }

        let tag_pos = ciphertext_and_tag.len() - TAG_SIZE;
        let (ciphertext, tag) = ciphertext_and_tag.split_at(tag_pos);
        validate_gcm_lengths(aad.len(), ciphertext.len())?;
        if out.len() < ciphertext.len() {
            return Err(Error::OutputTooSmall);
        }
        let out = &mut out[..ciphertext.len()];
        let Some(expected) = self.open(&nonce, aad, ciphertext, out) else {
            out.zeroize();
            return Err(Error::Decrypt);
        };
        // The comparison itself is constant time (subtle); branching on its
        // result is sound because accept/reject is public output - the
        // caller observes it through the Result either way. On rejection, wipe
        // the transient plaintext written before authentication completed.
        if !constant_time_eq(&expected, tag) {
            out.zeroize();
            return Err(Error::Decrypt);
        }

        Ok(ciphertext.len())
    }

    /// Fused authenticate-and-decrypt: bulk ciphertext chunks feed GHASH while
    /// their CTR plaintext is written to the caller's buffer. The caller must
    /// zeroize `plaintext` if the returned tag does not match the supplied tag.
    fn open(
        &self,
        nonce: &[u8; NONCE_SIZE],
        aad: &[u8],
        ciphertext: &[u8],
        plaintext: &mut [u8],
    ) -> Option<[u8; TAG_SIZE]> {
        debug_assert_eq!(ciphertext.len(), plaintext.len());
        let mut ghasher = ghash::Ghasher::new(&self.ghash)?;
        ghasher.absorb_padded(aad);

        let mut counter = j0(nonce);
        increment_counter(&mut counter);

        // Decrypt writes plaintext before the final tag check. This matches
        // high-performance AEAD APIs such as ring's in-place open path; the
        // public wrapper wipes the written range on authentication failure.
        let bulk = (ciphertext.len() / PAR_BYTES) * PAR_BYTES;
        let (ciphertext_bulk, ciphertext_tail) = ciphertext.split_at(bulk);
        let (plaintext_bulk, plaintext_tail) = plaintext.split_at_mut(bulk);
        if !ciphertext_bulk.is_empty() {
            ghasher.open_bulk(
                self.aes.round_keys(),
                &mut counter,
                ciphertext_bulk,
                plaintext_bulk,
            );
        }
        if !ciphertext_tail.is_empty() {
            ghasher.absorb_padded(ciphertext_tail);
            plaintext_tail.copy_from_slice(ciphertext_tail);
            self.apply_ctr_serial(&mut counter, plaintext_tail);
        }

        let mut tag = ghasher.finalize(aad.len(), ciphertext.len())?;
        let mut mask = j0(nonce);
        self.aes.encrypt_block(&mut mask);
        for (tag_byte, mask_byte) in tag.iter_mut().zip(mask.iter()) {
            *tag_byte ^= mask_byte;
        }
        Some(tag)
    }

    /// In-place authenticate-and-decrypt. `data` enters as ciphertext and
    /// exits as plaintext. The caller must zeroize `data` if the returned tag
    /// does not match the supplied tag.
    fn open_in_place(
        &self,
        nonce: &[u8; NONCE_SIZE],
        aad: &[u8],
        data: &mut [u8],
    ) -> Option<[u8; TAG_SIZE]> {
        let mut ghasher = ghash::Ghasher::new(&self.ghash)?;
        ghasher.absorb_padded(aad);

        let mut counter = j0(nonce);
        increment_counter(&mut counter);

        let bulk = (data.len() / PAR_BYTES) * PAR_BYTES;
        let (data_bulk, data_tail) = data.split_at_mut(bulk);
        if !data_bulk.is_empty() {
            ghasher.open_in_place_bulk(self.aes.round_keys(), &mut counter, data_bulk);
        }
        if !data_tail.is_empty() {
            ghasher.absorb_padded(data_tail);
            self.apply_ctr_serial(&mut counter, data_tail);
        }

        let mut tag = ghasher.finalize(aad.len(), data.len())?;
        let mut mask = j0(nonce);
        self.aes.encrypt_block(&mut mask);
        for (tag_byte, mask_byte) in tag.iter_mut().zip(mask.iter()) {
            *tag_byte ^= mask_byte;
        }
        Some(tag)
    }
}

/// Caller-owned uninitialized storage for AES-256-GCM key state.
pub struct UninitKeyStateSlot<'a> {
    storage: &'a mut [u8],
}

impl<'a> UninitKeyStateSlot<'a> {
    /// Validates caller-provided storage for key-state initialization.
    ///
    /// # Errors
    ///
    /// Returns [`Error::KeyStateStorageTooSmall`] or
    /// [`Error::KeyStateStorageMisaligned`] before any key material is touched.
    pub fn new(storage: &'a mut [u8]) -> Result<Self, Error> {
        let layout = HardwareAes256Gcm::key_state_layout();
        if storage.len() < layout.size {
            return Err(Error::KeyStateStorageTooSmall);
        }
        if !storage.as_ptr().addr().is_multiple_of(layout.align) {
            return Err(Error::KeyStateStorageMisaligned);
        }
        Ok(Self { storage })
    }
}

impl std::fmt::Debug for UninitKeyStateSlot<'_> {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("UninitKeyStateSlot")
            .field("len", &self.storage.len())
            .finish_non_exhaustive()
    }
}

#[inline]
fn init_key_state_at(key: &[u8], state: NonNull<KeyState>) -> Result<(), Error> {
    if key.len() != KEY_SIZE {
        return Err(Error::InvalidKeyLength);
    }
    let key: &[u8; KEY_SIZE] = key.try_into().map_err(|_| Error::InvalidKeyLength)?;
    KeyState::init_in_place(state, key)
}

fn init_key_state_in_slot(key: &[u8], slot: UninitKeyStateSlot<'_>) -> Result<NonNull<u8>, Error> {
    let UninitKeyStateSlot { storage } = slot;
    // The raw pointer is taken once and the `&mut` slice ends here, so the
    // handle never aliases a live mutable reference.
    // SAFETY: UninitKeyStateSlot validated that storage has non-zero size,
    // sufficient length, and KeyState alignment, so the pointer is non-null.
    let storage = unsafe { NonNull::new_unchecked(storage.as_mut_ptr()) };
    #[allow(clippy::cast_ptr_alignment)]
    let state = storage.cast::<KeyState>();
    init_key_state_at(key, state)?;
    Ok(storage)
}

/// Opaque initialized key-equivalent state in caller-owned storage.
///
/// The handle stores a raw pointer into the caller's storage instead of the
/// caller's `&mut` slice itself, so reads through the handle never coexist
/// with a live mutable reference under strict aliasing models.
pub struct OpaqueKeyState<'a> {
    storage: NonNull<u8>,
    _marker: PhantomData<&'a mut [u8]>,
}

const _: () = {
    const fn assert_send_sync<T: Send + Sync>() {}
    // Compile-time proof that KeyState stays thread-safe; the unsafe Send/Sync
    // impls below rely on it.
    assert_send_sync::<KeyState>();
};

// SAFETY: OpaqueKeyState exclusively owns the KeyState in the caller storage
// for 'a (the storage `&mut` borrow is consumed on construction). KeyState is
// Send + Sync (asserted above), there is no interior mutability, and all
// access outside drop is read-only, so moving the handle across threads is
// sound.
unsafe impl Send for OpaqueKeyState<'_> {}
// SAFETY: see the Send impl; a shared OpaqueKeyState only permits shared reads
// of a Sync pointee.
unsafe impl Sync for OpaqueKeyState<'_> {}

impl OpaqueKeyState<'_> {
    fn state_ptr(&self) -> *mut KeyState {
        // UninitKeyStateSlot validated KeyState alignment before this handle
        // could be constructed.
        #[allow(clippy::cast_ptr_alignment)]
        self.storage.as_ptr().cast::<KeyState>()
    }
}

impl Drop for OpaqueKeyState<'_> {
    fn drop(&mut self) {
        let size = HardwareAes256Gcm::key_state_layout().size;
        // KeyState's field Drop impls (Aes256, GHashKey) exist only to wipe
        // their own bytes and release no heap or handle resources, so they
        // are deliberately not run here: one volatile wipe of the whole
        // storage supersedes them and avoids a redundant second pass. If
        // KeyState ever gains a resource-owning field, this must go back to
        // drop_in_place before the wipe.
        // SAFETY: the caller storage remains borrowed for 'a, holds at least
        // `size` bytes, and is KeyState-aligned (validated at slot
        // construction).
        unsafe { volatile_wipe(self.storage.as_ptr(), size) };
    }
}

impl std::fmt::Debug for OpaqueKeyState<'_> {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("OpaqueKeyState").finish_non_exhaustive()
    }
}

/// AES-256-GCM instance backed by caller-owned key-state storage.
pub struct HardwareAes256GcmIn<'a> {
    state: OpaqueKeyState<'a>,
    /// Lazily initialized on first use of a generated-nonce method, so
    /// caller-supplied-nonce users never draw OS entropy. Not part of the
    /// caller-placed key state, so it does not affect `key_state_layout`.
    nonce_gen: Option<nonce::NonceGen>,
}

impl<'a> HardwareAes256GcmIn<'a> {
    /// Initializes reusable AES-256-GCM state directly in caller-owned storage.
    ///
    /// # Errors
    ///
    /// Returns storage validation errors, [`Error::InvalidKeyLength`], or
    /// [`Error::UnsupportedCpu`].
    pub fn new_in(key: &[u8], slot: UninitKeyStateSlot<'a>) -> Result<Self, Error> {
        let storage = init_key_state_in_slot(key, slot)?;
        Ok(Self {
            state: OpaqueKeyState {
                storage,
                _marker: PhantomData,
            },
            nonce_gen: None,
        })
    }

    /// Encrypts `plaintext` and returns `ciphertext || tag`.
    ///
    /// # Errors
    ///
    /// Returns [`Error::InvalidNonceLength`], [`Error::InputTooLarge`], or
    /// [`Error::Encrypt`].
    pub fn encrypt(&self, nonce: &[u8], aad: &[u8], plaintext: &[u8]) -> Result<Vec<u8>, Error> {
        self.state_ref().encrypt(nonce, aad, plaintext)
    }

    /// Encrypts `plaintext` into a caller-provided buffer as
    /// `ciphertext || tag` and returns the written length.
    ///
    /// No heap allocation is performed.
    ///
    /// # Errors
    ///
    /// Returns [`Error::OutputTooSmall`] if `out` is shorter than
    /// `plaintext.len() + TAG_SIZE`, plus the same errors as [`Self::encrypt`].
    pub fn encrypt_to(
        &self,
        nonce: &[u8],
        aad: &[u8],
        plaintext: &[u8],
        out: &mut [u8],
    ) -> Result<usize, Error> {
        self.state_ref().encrypt_to(nonce, aad, plaintext, out)
    }

    /// Decrypts `ciphertext || tag` and returns plaintext.
    ///
    /// # Errors
    ///
    /// Returns [`Error::InvalidNonceLength`], [`Error::InputTooLarge`], or
    /// [`Error::Decrypt`].
    pub fn decrypt(
        &self,
        nonce: &[u8],
        aad: &[u8],
        ciphertext_and_tag: &[u8],
    ) -> Result<Vec<u8>, Error> {
        self.state_ref().decrypt(nonce, aad, ciphertext_and_tag)
    }

    /// Decrypts `ciphertext || tag` into a caller-provided buffer and returns
    /// the plaintext length.
    ///
    /// Decrypts into `out` before the final tag comparison so CTR and GHASH can
    /// run in one fused pass. If authentication fails, the plaintext-length
    /// prefix of `out` is zeroized before returning [`Error::Decrypt`].
    ///
    /// # Errors
    ///
    /// Returns [`Error::OutputTooSmall`] if `out` is shorter than the
    /// plaintext, plus the same errors as [`Self::decrypt`].
    pub fn decrypt_to(
        &self,
        nonce: &[u8],
        aad: &[u8],
        ciphertext_and_tag: &[u8],
        out: &mut [u8],
    ) -> Result<usize, Error> {
        self.state_ref()
            .decrypt_to(nonce, aad, ciphertext_and_tag, out)
    }

    /// Encrypts and appends the nonce: `ciphertext || tag || nonce`.
    ///
    /// # Errors
    ///
    /// Returns the same errors as [`Self::encrypt`].
    pub fn encrypt_nonce_appended(&self, nonce: &[u8], plaintext: &[u8]) -> Result<Vec<u8>, Error> {
        self.state_ref().encrypt_nonce_appended(nonce, plaintext)
    }

    /// Encrypts the plaintext already in `in_out` in place, then appends the
    /// tag and nonce so the final layout is `ciphertext || tag || nonce`.
    ///
    /// If `in_out` has capacity for `plaintext.len() + TAG_SIZE + NONCE_SIZE`,
    /// this performs no heap allocation. The buffer is zeroized before
    /// returning [`Error::Encrypt`] if encryption fails after mutating it.
    ///
    /// # Errors
    ///
    /// Returns the same errors as [`Self::encrypt`].
    pub fn encrypt_nonce_appended_in_place(
        &self,
        nonce: &[u8],
        in_out: &mut Vec<u8>,
    ) -> Result<(), Error> {
        self.state_ref()
            .encrypt_nonce_appended_in_place(nonce, in_out)
    }

    /// Encrypts into a caller-provided buffer with the nonce appended and
    /// returns the written length. No heap allocation is performed.
    ///
    /// # Errors
    ///
    /// Returns [`Error::OutputTooSmall`] if `out` is shorter than
    /// `plaintext.len() + TAG_SIZE + NONCE_SIZE`, plus the same errors as
    /// [`Self::encrypt`].
    pub fn encrypt_nonce_appended_to(
        &self,
        nonce: &[u8],
        plaintext: &[u8],
        out: &mut [u8],
    ) -> Result<usize, Error> {
        encrypt_nonce_appended_to(self.state_ref(), nonce, plaintext, out)
    }

    /// Decrypts the nonce-appended layout: `ciphertext || tag || nonce`.
    ///
    /// # Errors
    ///
    /// Returns [`Error::CiphertextTooShort`] if the input cannot contain a tag
    /// and nonce. Returns [`Error::InvalidNonceLength`],
    /// [`Error::InputTooLarge`], or [`Error::Decrypt`].
    pub fn decrypt_nonce_appended(&self, data: &[u8]) -> Result<Vec<u8>, Error> {
        if data.len() < TAG_SIZE + NONCE_SIZE {
            return Err(Error::CiphertextTooShort);
        }
        let nonce_pos = data.len() - NONCE_SIZE;
        let (ciphertext_and_tag, nonce) = data.split_at(nonce_pos);
        self.decrypt(nonce, &[], ciphertext_and_tag)
    }

    /// Decrypts the nonce-appended layout into a caller-provided buffer and
    /// returns the plaintext length.
    ///
    /// # Errors
    ///
    /// Returns the same errors as [`Self::decrypt_nonce_appended`] plus
    /// [`Error::OutputTooSmall`].
    pub fn decrypt_nonce_appended_to(&self, data: &[u8], out: &mut [u8]) -> Result<usize, Error> {
        if data.len() < TAG_SIZE + NONCE_SIZE {
            return Err(Error::CiphertextTooShort);
        }
        let nonce_pos = data.len() - NONCE_SIZE;
        let (ciphertext_and_tag, nonce) = data.split_at(nonce_pos);
        self.decrypt_to(nonce, &[], ciphertext_and_tag, out)
    }

    /// Encrypts `plaintext` under a library-generated unique nonce, returning
    /// the nonce alongside `ciphertext || tag`.
    ///
    /// The nonce is drawn from a per-instance sequence (96-bit OS-seeded salt
    /// plus a 64-bit counter, re-salted on fork; see [`crate`]); callers do
    /// not manage nonce uniqueness. The returned nonce must be retained to
    /// decrypt.
    ///
    /// # Errors
    ///
    /// Returns [`Error::OsEntropy`] if seeding the nonce sequence fails, plus
    /// the same errors as [`Self::encrypt`].
    pub fn encrypt_with_generated_nonce(
        &mut self,
        aad: &[u8],
        plaintext: &[u8],
    ) -> Result<([u8; NONCE_SIZE], Vec<u8>), Error> {
        let nonce = self.next_nonce()?;
        let ciphertext = self.state_ref().encrypt(&nonce, aad, plaintext)?;
        Ok((nonce, ciphertext))
    }

    /// Encrypts `plaintext` under a library-generated unique nonce and returns
    /// the self-framed `ciphertext || tag || nonce` layout (empty AAD).
    ///
    /// # Errors
    ///
    /// Returns the same errors as [`Self::encrypt_with_generated_nonce`].
    pub fn encrypt_nonce_appended_generated(&mut self, plaintext: &[u8]) -> Result<Vec<u8>, Error> {
        let nonce = self.next_nonce()?;
        self.state_ref().encrypt_nonce_appended(&nonce, plaintext)
    }

    fn next_nonce(&mut self) -> Result<[u8; NONCE_SIZE], Error> {
        match self.nonce_gen {
            Some(ref mut g) => g.next(),
            None => self.nonce_gen.insert(nonce::NonceGen::new()?).next(),
        }
    }

    fn state_ref(&self) -> &KeyState {
        // SAFETY: OpaqueKeyState owns a live initialized KeyState until drop.
        unsafe { &*self.state.state_ptr() }
    }
}

impl std::fmt::Debug for HardwareAes256GcmIn<'_> {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("HardwareAes256GcmIn")
            .finish_non_exhaustive()
    }
}

#[repr(transparent)]
struct AlignedKeyStateStorage(MaybeUninit<KeyState>);

impl AlignedKeyStateStorage {
    #[inline]
    const fn uninit() -> Self {
        Self(MaybeUninit::uninit())
    }

    #[inline]
    fn state_ptr(&self) -> *const KeyState {
        self.0.as_ptr()
    }

    #[inline]
    fn state_ptr_mut(&mut self) -> NonNull<KeyState> {
        // SAFETY: MaybeUninit<KeyState> is non-null and aligned for KeyState.
        unsafe { NonNull::new_unchecked(self.0.as_mut_ptr()) }
    }

    #[inline]
    fn bytes_ptr_mut(&mut self) -> *mut u8 {
        self.0.as_mut_ptr().cast::<u8>()
    }
}

/// Allocation-free owned reusable AES-256-GCM key state.
///
/// This stores the same opaque key-equivalent bytes reported by
/// [`HardwareAes256Gcm::key_state_layout`] inline in the value, avoiding the
/// heap allocation used by [`HardwareAes256Gcm`] while still wiping the key
/// state on drop.
#[repr(transparent)]
pub struct HardwareAes256GcmKeyState {
    storage: AlignedKeyStateStorage,
}

impl HardwareAes256GcmKeyState {
    /// Creates reusable AES-256-GCM state from a raw 32-byte key without heap
    /// allocation.
    ///
    /// # Errors
    ///
    /// Returns [`Error::InvalidKeyLength`] if `key` is not exactly 32 bytes,
    /// or [`Error::UnsupportedCpu`] if required AES-GCM hardware is absent.
    #[inline]
    pub fn new(key: &[u8]) -> Result<Self, Error> {
        let mut storage = AlignedKeyStateStorage::uninit();
        init_key_state_at(key, storage.state_ptr_mut())?;
        Ok(Self { storage })
    }

    /// Returns the current size of the reusable inline key state.
    #[must_use]
    pub const fn state_size() -> usize {
        std::mem::size_of::<KeyState>()
    }

    /// Encrypts and appends the nonce: `ciphertext || tag || nonce`.
    ///
    /// # Errors
    ///
    /// Returns [`Error::InvalidNonceLength`], [`Error::InputTooLarge`], or
    /// [`Error::Encrypt`].
    #[inline]
    pub fn encrypt_nonce_appended(&self, nonce: &[u8], plaintext: &[u8]) -> Result<Vec<u8>, Error> {
        self.state_ref().encrypt_nonce_appended(nonce, plaintext)
    }

    /// Encrypts the plaintext already in `in_out` in place, then appends the
    /// tag and nonce so the final layout is `ciphertext || tag || nonce`.
    ///
    /// If `in_out` has capacity for `plaintext.len() + TAG_SIZE + NONCE_SIZE`,
    /// this performs no heap allocation. The buffer is zeroized before
    /// returning [`Error::Encrypt`] if encryption fails after mutating it.
    ///
    /// # Errors
    ///
    /// Returns the same errors as [`Self::encrypt_nonce_appended`].
    #[inline]
    pub fn encrypt_nonce_appended_in_place(
        &self,
        nonce: &[u8],
        in_out: &mut Vec<u8>,
    ) -> Result<(), Error> {
        self.state_ref()
            .encrypt_nonce_appended_in_place(nonce, in_out)
    }

    /// Decrypts the nonce-appended layout: `ciphertext || tag || nonce`.
    ///
    /// # Errors
    ///
    /// Returns [`Error::CiphertextTooShort`] if the input cannot contain a tag
    /// and nonce. Returns [`Error::InvalidNonceLength`],
    /// [`Error::InputTooLarge`], or [`Error::Decrypt`].
    #[inline]
    pub fn decrypt_nonce_appended(&self, data: &[u8]) -> Result<Vec<u8>, Error> {
        if data.len() < TAG_SIZE + NONCE_SIZE {
            return Err(Error::CiphertextTooShort);
        }
        let nonce_pos = data.len() - NONCE_SIZE;
        let (ciphertext_and_tag, nonce) = data.split_at(nonce_pos);
        self.state_ref().decrypt(nonce, &[], ciphertext_and_tag)
    }

    fn state_ref(&self) -> &KeyState {
        // SAFETY: HardwareAes256GcmKeyState::new initializes storage before
        // constructing Self, and the storage is never mutated except during
        // Drop after all shared borrows have ended.
        unsafe { &*self.storage.state_ptr() }
    }
}

impl Drop for HardwareAes256GcmKeyState {
    fn drop(&mut self) {
        let size = Self::state_size();
        // KeyState's field Drop impls exist only to wipe their own bytes and
        // release no heap or handle resources, so one volatile wipe of the
        // inline storage supersedes those per-field wipes. If KeyState ever
        // gains a resource-owning field, this must run drop_in_place first.
        // SAFETY: storage is inline in self, writable for `size` bytes, and
        // aligned for KeyState by AlignedKeyStateStorage.
        unsafe { volatile_wipe(self.storage.bytes_ptr_mut(), size) };
    }
}

impl std::fmt::Debug for HardwareAes256GcmKeyState {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("HardwareAes256GcmKeyState")
            .finish_non_exhaustive()
    }
}

/// Owned reusable hardware-only AES-256-GCM key state.
pub struct HardwareAes256Gcm {
    state: Box<KeyState>,
    /// Lazily initialized on first use of a generated-nonce method. Held
    /// outside `KeyState`, so it does not affect `state_size`/the boxed
    /// key-state footprint.
    nonce_gen: Option<nonce::NonceGen>,
}

impl HardwareAes256Gcm {
    /// Creates reusable AES-256-GCM state from a raw 32-byte key.
    ///
    /// # Errors
    ///
    /// Returns [`Error::InvalidKeyLength`] if `key` is not exactly 32 bytes.
    pub fn new(key: &[u8]) -> Result<Self, Error> {
        if key.len() != KEY_SIZE {
            return Err(Error::InvalidKeyLength);
        }
        let mut state = Box::<KeyState>::new_uninit();
        // SAFETY: Box::new_uninit returns a non-null allocation for KeyState.
        let ptr = unsafe { NonNull::new_unchecked(state.as_mut_ptr()) };
        init_key_state_at(key, ptr)?;
        // SAFETY: KeyState::init_in_place initialized the allocation on success.
        let state = unsafe { state.assume_init() };
        Ok(Self {
            state,
            nonce_gen: None,
        })
    }

    /// Returns whether all required AES-GCM hardware features are available.
    #[must_use]
    pub fn hardware_available() -> bool {
        aes::hardware_available() && ghash::hardware_available()
    }

    /// Returns the current size of the reusable key state.
    ///
    /// This is a benchmark/validation hook for the compact hardware-only state.
    #[must_use]
    pub const fn state_size() -> usize {
        std::mem::size_of::<KeyState>()
    }

    /// Returns the current opaque key-state layout.
    ///
    /// Reports the exact layout required for caller-provided opaque storage.
    #[must_use]
    pub const fn key_state_layout() -> KeyStateLayout {
        KeyStateLayout {
            size: std::mem::size_of::<KeyState>(),
            align: std::mem::align_of::<KeyState>(),
        }
    }

    /// Encrypts `plaintext` and returns `ciphertext || tag`.
    ///
    /// # Errors
    ///
    /// Returns [`Error::InvalidNonceLength`] if `nonce` is not exactly 12 bytes.
    /// Returns [`Error::InputTooLarge`] if `plaintext` or `aad` exceed the
    /// AES-GCM limits. Returns [`Error::Encrypt`] if the backend rejects
    /// encryption.
    pub fn encrypt(&self, nonce: &[u8], aad: &[u8], plaintext: &[u8]) -> Result<Vec<u8>, Error> {
        self.state.encrypt(nonce, aad, plaintext)
    }

    /// Encrypts `plaintext` into a caller-provided buffer as
    /// `ciphertext || tag` and returns the written length.
    ///
    /// No heap allocation is performed.
    ///
    /// # Errors
    ///
    /// Returns [`Error::OutputTooSmall`] if `out` is shorter than
    /// `plaintext.len() + TAG_SIZE`, plus the same errors as [`Self::encrypt`].
    pub fn encrypt_to(
        &self,
        nonce: &[u8],
        aad: &[u8],
        plaintext: &[u8],
        out: &mut [u8],
    ) -> Result<usize, Error> {
        self.state.encrypt_to(nonce, aad, plaintext, out)
    }

    /// Decrypts `ciphertext || tag` and returns plaintext.
    ///
    /// # Errors
    ///
    /// Returns [`Error::InvalidNonceLength`] if `nonce` is not exactly 12 bytes.
    /// Returns [`Error::InputTooLarge`] if the ciphertext or `aad` exceed the
    /// AES-GCM limits. Returns [`Error::Decrypt`] if authentication fails.
    pub fn decrypt(
        &self,
        nonce: &[u8],
        aad: &[u8],
        ciphertext_and_tag: &[u8],
    ) -> Result<Vec<u8>, Error> {
        self.state.decrypt(nonce, aad, ciphertext_and_tag)
    }

    /// Decrypts `ciphertext || tag` into a caller-provided buffer and returns
    /// the plaintext length.
    ///
    /// Decrypts into `out` before the final tag comparison so CTR and GHASH can
    /// run in one fused pass. If authentication fails, the plaintext-length
    /// prefix of `out` is zeroized before returning [`Error::Decrypt`].
    ///
    /// # Errors
    ///
    /// Returns [`Error::OutputTooSmall`] if `out` is shorter than the
    /// plaintext, plus the same errors as [`Self::decrypt`].
    pub fn decrypt_to(
        &self,
        nonce: &[u8],
        aad: &[u8],
        ciphertext_and_tag: &[u8],
        out: &mut [u8],
    ) -> Result<usize, Error> {
        self.state.decrypt_to(nonce, aad, ciphertext_and_tag, out)
    }

    /// Encrypts and appends the nonce: `ciphertext || tag || nonce`.
    ///
    /// # Errors
    ///
    /// Returns the same errors as [`Self::encrypt`].
    pub fn encrypt_nonce_appended(&self, nonce: &[u8], plaintext: &[u8]) -> Result<Vec<u8>, Error> {
        self.state.encrypt_nonce_appended(nonce, plaintext)
    }

    /// Encrypts the plaintext already in `in_out` in place, then appends the
    /// tag and nonce so the final layout is `ciphertext || tag || nonce`.
    ///
    /// If `in_out` has capacity for `plaintext.len() + TAG_SIZE + NONCE_SIZE`,
    /// this performs no heap allocation. The buffer is zeroized before
    /// returning [`Error::Encrypt`] if encryption fails after mutating it.
    ///
    /// # Errors
    ///
    /// Returns the same errors as [`Self::encrypt`].
    pub fn encrypt_nonce_appended_in_place(
        &self,
        nonce: &[u8],
        in_out: &mut Vec<u8>,
    ) -> Result<(), Error> {
        self.state.encrypt_nonce_appended_in_place(nonce, in_out)
    }

    /// Encrypts into a caller-provided buffer with the nonce appended and
    /// returns the written length. No heap allocation is performed.
    ///
    /// # Errors
    ///
    /// Returns [`Error::OutputTooSmall`] if `out` is shorter than
    /// `plaintext.len() + TAG_SIZE + NONCE_SIZE`, plus the same errors as
    /// [`Self::encrypt`].
    pub fn encrypt_nonce_appended_to(
        &self,
        nonce: &[u8],
        plaintext: &[u8],
        out: &mut [u8],
    ) -> Result<usize, Error> {
        encrypt_nonce_appended_to(&self.state, nonce, plaintext, out)
    }

    /// Decrypts the nonce-appended layout: `ciphertext || tag || nonce`.
    ///
    /// # Errors
    ///
    /// Returns [`Error::CiphertextTooShort`] if the input cannot contain a tag
    /// and nonce. Returns [`Error::InvalidNonceLength`],
    /// [`Error::InputTooLarge`], or [`Error::Decrypt`] for malformed
    /// nonce/authentication failures.
    pub fn decrypt_nonce_appended(&self, data: &[u8]) -> Result<Vec<u8>, Error> {
        if data.len() < TAG_SIZE + NONCE_SIZE {
            return Err(Error::CiphertextTooShort);
        }
        let nonce_pos = data.len() - NONCE_SIZE;
        let (ciphertext_and_tag, nonce) = data.split_at(nonce_pos);
        self.decrypt(nonce, &[], ciphertext_and_tag)
    }

    /// Decrypts the nonce-appended layout into a caller-provided buffer and
    /// returns the plaintext length.
    ///
    /// # Errors
    ///
    /// Returns the same errors as [`Self::decrypt_nonce_appended`] plus
    /// [`Error::OutputTooSmall`].
    pub fn decrypt_nonce_appended_to(&self, data: &[u8], out: &mut [u8]) -> Result<usize, Error> {
        if data.len() < TAG_SIZE + NONCE_SIZE {
            return Err(Error::CiphertextTooShort);
        }
        let nonce_pos = data.len() - NONCE_SIZE;
        let (ciphertext_and_tag, nonce) = data.split_at(nonce_pos);
        self.decrypt_to(nonce, &[], ciphertext_and_tag, out)
    }

    /// Encrypts `plaintext` under a library-generated unique nonce, returning
    /// the nonce alongside `ciphertext || tag`.
    ///
    /// The nonce is drawn from a per-instance sequence (96-bit OS-seeded salt
    /// plus a 64-bit counter, re-salted on fork; see [`crate`]); callers do
    /// not manage nonce uniqueness. The returned nonce must be retained to
    /// decrypt.
    ///
    /// # Errors
    ///
    /// Returns [`Error::OsEntropy`] if seeding the nonce sequence fails, plus
    /// the same errors as [`Self::encrypt`].
    pub fn encrypt_with_generated_nonce(
        &mut self,
        aad: &[u8],
        plaintext: &[u8],
    ) -> Result<([u8; NONCE_SIZE], Vec<u8>), Error> {
        let nonce = self.next_nonce()?;
        let ciphertext = self.state.encrypt(&nonce, aad, plaintext)?;
        Ok((nonce, ciphertext))
    }

    /// Encrypts `plaintext` under a library-generated unique nonce and returns
    /// the self-framed `ciphertext || tag || nonce` layout (empty AAD).
    ///
    /// # Errors
    ///
    /// Returns the same errors as [`Self::encrypt_with_generated_nonce`].
    pub fn encrypt_nonce_appended_generated(&mut self, plaintext: &[u8]) -> Result<Vec<u8>, Error> {
        let nonce = self.next_nonce()?;
        self.state.encrypt_nonce_appended(&nonce, plaintext)
    }

    fn next_nonce(&mut self) -> Result<[u8; NONCE_SIZE], Error> {
        match self.nonce_gen {
            Some(ref mut g) => g.next(),
            None => self.nonce_gen.insert(nonce::NonceGen::new()?).next(),
        }
    }
}

impl std::fmt::Debug for HardwareAes256Gcm {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("HardwareAes256Gcm").finish_non_exhaustive()
    }
}

fn nonce_from_slice(nonce: &[u8]) -> Result<[u8; NONCE_SIZE], Error> {
    nonce.try_into().map_err(|_| Error::InvalidNonceLength)
}

fn append_tag_nonce(out: &mut Vec<u8>, tag: &[u8; TAG_SIZE], nonce: &[u8; NONCE_SIZE]) {
    debug_assert!(out.capacity() >= out.len() + TAG_SIZE + NONCE_SIZE);

    let start = out.len();
    let trailer = &mut out.spare_capacity_mut()[..TAG_SIZE + NONCE_SIZE];
    for (slot, byte) in trailer[..TAG_SIZE].iter_mut().zip(tag) {
        slot.write(*byte);
    }
    for (slot, byte) in trailer[TAG_SIZE..].iter_mut().zip(nonce) {
        slot.write(*byte);
    }

    // SAFETY: the TAG_SIZE + NONCE_SIZE spare bytes immediately after the
    // original length were initialized above, and capacity was ensured by the
    // caller before encryption mutated the buffer.
    unsafe { out.set_len(start + TAG_SIZE + NONCE_SIZE) };
}

/// Volatile wipe of caller-placed key-state storage using the widest stores
/// the pointer alignment allows.
///
/// # Safety
///
/// `bytes..bytes + len` must be writable.
#[allow(clippy::cast_ptr_alignment)] // Wide stores are alignment-checked at runtime.
unsafe fn volatile_wipe(bytes: *mut u8, len: usize) {
    let mut offset = 0_usize;
    if bytes.addr().is_multiple_of(core::mem::align_of::<u128>()) {
        while offset + core::mem::size_of::<u128>() <= len {
            // SAFETY: in bounds per the caller contract and aligned per the
            // check above.
            unsafe { ptr::write_volatile(bytes.add(offset).cast::<u128>(), 0) };
            offset += core::mem::size_of::<u128>();
        }
    }
    for offset in offset..len {
        // SAFETY: in bounds per the caller contract.
        unsafe { ptr::write_volatile(bytes.add(offset), 0) };
    }
    core::sync::atomic::compiler_fence(core::sync::atomic::Ordering::SeqCst);
}

/// Shared no-alloc nonce-appended encryption: `ciphertext || tag || nonce`.
fn encrypt_nonce_appended_to(
    state: &KeyState,
    nonce: &[u8],
    plaintext: &[u8],
    out: &mut [u8],
) -> Result<usize, Error> {
    let total = plaintext
        .len()
        .checked_add(TAG_SIZE + NONCE_SIZE)
        .ok_or(Error::InputTooLarge)?;
    if out.len() < total {
        return Err(Error::OutputTooSmall);
    }
    let written = state.encrypt_to(nonce, &[], plaintext, out)?;
    // encrypt_to validated the nonce length, so this copy is exactly
    // NONCE_SIZE bytes.
    out[written..total].copy_from_slice(nonce);
    Ok(total)
}

fn validate_gcm_lengths(aad_len: usize, data_len: usize) -> Result<(), Error> {
    if len_exceeds_u64_limit(data_len, MAX_GCM_DATA_LEN)
        || len_exceeds_u64_limit(aad_len, MAX_GHASH_INPUT_LEN)
        || len_exceeds_u64_limit(data_len, MAX_GHASH_INPUT_LEN)
    {
        return Err(Error::InputTooLarge);
    }
    Ok(())
}

fn len_exceeds_u64_limit(len: usize, limit: u64) -> bool {
    u64::try_from(len).map_or(true, |len| len > limit)
}

fn j0(nonce: &[u8; NONCE_SIZE]) -> [u8; 16] {
    let mut out = [0_u8; 16];
    out[..NONCE_SIZE].copy_from_slice(nonce);
    out[15] = 1;
    out
}

/// Operates on the public nonce-derived counter block; constant time anyway
/// (`wrapping_add` carries without branching).
pub(crate) fn increment_counter(counter: &mut [u8; 16]) {
    let mut low_bytes = [0_u8; 4];
    low_bytes.copy_from_slice(&counter[12..]);
    let low = u32::from_be_bytes(low_bytes).wrapping_add(1);
    counter[12..].copy_from_slice(&low.to_be_bytes());
}

fn constant_time_eq(expected: &[u8; TAG_SIZE], actual: &[u8]) -> bool {
    // subtle's slice impl compares lengths first (length is public) and then
    // the contents in constant time behind optimization barriers, so the
    // compiler cannot reintroduce an early-exit byte comparison. This is the
    // one place in the crate where secret-derived values are compared.
    expected.as_slice().ct_eq(actual).into()
}

#[cfg(test)]
mod tests {
    #![allow(clippy::expect_used, clippy::panic)]

    use super::{
        Error, HardwareAes256Gcm, HardwareAes256GcmIn, HardwareAes256GcmKeyState,
        UninitKeyStateSlot, NONCE_SIZE,
    };
    use core::mem::ManuallyDrop;

    #[repr(align(64))]
    struct AlignedStorage<const N: usize>([u8; N]);

    #[test]
    fn rejects_wrong_key_length() {
        let Err(err) = HardwareAes256Gcm::new(&[0_u8; 31]) else {
            panic!("31-byte key should be rejected");
        };
        assert_eq!(err, Error::InvalidKeyLength);
    }

    #[test]
    fn caller_placed_key_state_rejects_wrong_key_length() {
        let layout = HardwareAes256Gcm::key_state_layout();
        let mut storage = AlignedStorage::<512>([0_u8; 512]);
        let slot = UninitKeyStateSlot::new(&mut storage.0[..layout.size]).expect("valid slot");
        let Err(err) = HardwareAes256GcmIn::new_in(&[0_u8; 31], slot) else {
            panic!("31-byte key should be rejected");
        };
        assert_eq!(err, Error::InvalidKeyLength);
    }

    #[test]
    fn inline_owned_key_state_rejects_wrong_key_length() {
        let Err(err) = HardwareAes256GcmKeyState::new(&[0_u8; 31]) else {
            panic!("31-byte key should be rejected");
        };
        assert_eq!(err, Error::InvalidKeyLength);
    }

    #[test]
    fn rejects_ciphertext_shorter_than_tag() {
        let key = HardwareAes256Gcm::new(&[7_u8; 32]).expect("valid test key");
        let Err(err) = key.decrypt(&[0_u8; NONCE_SIZE], &[], &[0_u8; super::TAG_SIZE - 1]) else {
            panic!("input shorter than the tag should be rejected");
        };
        assert_eq!(err, Error::Decrypt);
    }

    #[test]
    fn rejects_nonce_appended_shorter_than_tag_and_nonce() {
        let key = HardwareAes256Gcm::new(&[7_u8; 32]).expect("valid test key");
        let short = [0_u8; super::TAG_SIZE + NONCE_SIZE - 1];
        let Err(err) = key.decrypt_nonce_appended(&short) else {
            panic!("input shorter than tag plus nonce should be rejected");
        };
        assert_eq!(err, Error::CiphertextTooShort);

        let mut out = [0_u8; 64];
        let Err(err) = key.decrypt_nonce_appended_to(&short, &mut out) else {
            panic!("input shorter than tag plus nonce should be rejected");
        };
        assert_eq!(err, Error::CiphertextTooShort);
    }

    #[test]
    fn decrypt_to_writes_plaintext_into_caller_buffer() {
        let key = HardwareAes256Gcm::new(&[7_u8; 32]).expect("valid test key");
        let nonce = [9_u8; NONCE_SIZE];
        let ciphertext = key
            .encrypt(&nonce, b"aad", b"plaintext")
            .expect("encryption should succeed");

        let mut exact = [0_u8; 9];
        let written = key
            .decrypt_to(&nonce, b"aad", &ciphertext, &mut exact)
            .expect("decryption into an exact-size buffer should succeed");
        assert_eq!(written, 9);
        assert_eq!(&exact, b"plaintext");

        let mut oversized = [0xff_u8; 32];
        let written = key
            .decrypt_to(&nonce, b"aad", &ciphertext, &mut oversized)
            .expect("decryption into a larger buffer should succeed");
        assert_eq!(&oversized[..written], b"plaintext");

        let mut short = [0_u8; 8];
        let Err(err) = key.decrypt_to(&nonce, b"aad", &ciphertext, &mut short) else {
            panic!("short output buffer should be rejected");
        };
        assert_eq!(err, Error::OutputTooSmall);
        assert_eq!(short, [0_u8; 8], "no plaintext may leak into short buffers");

        let mut tampered = ciphertext;
        *tampered.last_mut().expect("tag byte") ^= 1;
        let mut invalid_out = [0xff_u8; 9];
        let Err(err) = key.decrypt_to(&nonce, b"aad", &tampered, &mut invalid_out) else {
            panic!("invalid authentication tag should be rejected");
        };
        assert_eq!(err, Error::Decrypt);
        assert_eq!(
            invalid_out, [0_u8; 9],
            "transient plaintext must be wiped on authentication failure"
        );
    }

    #[test]
    fn encrypt_to_matches_allocating_encrypt() {
        let key = HardwareAes256Gcm::new(&[7_u8; 32]).expect("valid test key");
        let nonce = [9_u8; NONCE_SIZE];
        let plaintext = [0x5a_u8; 200];
        let expected = key
            .encrypt(&nonce, b"aad", &plaintext)
            .expect("encryption should succeed");

        let mut exact = [0_u8; 200 + super::TAG_SIZE];
        let written = key
            .encrypt_to(&nonce, b"aad", &plaintext, &mut exact)
            .expect("encryption into an exact-size buffer should succeed");
        assert_eq!(written, exact.len());
        assert_eq!(exact.as_slice(), expected.as_slice());

        let mut oversized = [0_u8; 256];
        let written = key
            .encrypt_to(&nonce, b"aad", &plaintext, &mut oversized)
            .expect("encryption into a larger buffer should succeed");
        assert_eq!(&oversized[..written], expected.as_slice());

        let mut short = [0_u8; 200 + super::TAG_SIZE - 1];
        let Err(err) = key.encrypt_to(&nonce, b"aad", &plaintext, &mut short) else {
            panic!("short output buffer should be rejected");
        };
        assert_eq!(err, Error::OutputTooSmall);
        assert!(
            short.iter().all(|byte| *byte == 0),
            "no ciphertext may be written into short buffers"
        );
    }

    #[test]
    fn encrypt_nonce_appended_to_matches_allocating_variant() {
        let key = HardwareAes256Gcm::new(&[7_u8; 32]).expect("valid test key");
        let nonce = [9_u8; NONCE_SIZE];
        let expected = key
            .encrypt_nonce_appended(&nonce, b"plaintext")
            .expect("encryption should succeed");

        let mut out = [0_u8; 9 + super::TAG_SIZE + NONCE_SIZE];
        let written = key
            .encrypt_nonce_appended_to(&nonce, b"plaintext", &mut out)
            .expect("encryption should succeed");
        assert_eq!(written, out.len());
        assert_eq!(out.as_slice(), expected.as_slice());

        let mut short = [0_u8; 9 + super::TAG_SIZE + NONCE_SIZE - 1];
        let Err(err) = key.encrypt_nonce_appended_to(&nonce, b"plaintext", &mut short) else {
            panic!("short output buffer should be rejected");
        };
        assert_eq!(err, Error::OutputTooSmall);
    }

    #[test]
    fn encrypt_nonce_appended_in_place_uses_existing_vec_capacity() {
        let key = HardwareAes256Gcm::new(&[7_u8; 32]).expect("valid test key");
        let nonce = [9_u8; NONCE_SIZE];
        let expected = key
            .encrypt_nonce_appended(&nonce, b"plaintext")
            .expect("encryption should succeed");

        let mut in_out = Vec::with_capacity(b"plaintext".len() + super::TAG_SIZE + NONCE_SIZE);
        in_out.extend_from_slice(b"plaintext");
        let ptr_before = in_out.as_ptr();
        key.encrypt_nonce_appended_in_place(&nonce, &mut in_out)
            .expect("in-place encryption should succeed");

        assert_eq!(in_out, expected);
        assert_eq!(
            in_out.as_ptr(),
            ptr_before,
            "pre-reserved in-place encryption should not reallocate"
        );
        assert_eq!(
            key.decrypt_nonce_appended(&in_out)
                .expect("decryption should succeed"),
            b"plaintext"
        );
    }

    #[test]
    fn decrypt_nonce_appended_to_round_trips() {
        let key = HardwareAes256Gcm::new(&[7_u8; 32]).expect("valid test key");
        let nonce = [9_u8; NONCE_SIZE];
        let data = key
            .encrypt_nonce_appended(&nonce, b"plaintext")
            .expect("encryption should succeed");

        let mut out = [0_u8; 32];
        let written = key
            .decrypt_nonce_appended_to(&data, &mut out)
            .expect("decryption should succeed");
        assert_eq!(&out[..written], b"plaintext");
    }

    #[test]
    fn generated_nonce_round_trips_and_is_unique() {
        let mut key = HardwareAes256Gcm::new(&[7_u8; 32]).expect("valid test key");

        let (nonce_a, ct_a) = key
            .encrypt_with_generated_nonce(b"aad", b"plaintext")
            .expect("encryption should succeed");
        let (nonce_b, _ct_b) = key
            .encrypt_with_generated_nonce(b"aad", b"plaintext")
            .expect("encryption should succeed");

        // Same key + same plaintext must not reuse a nonce.
        assert_ne!(nonce_a, nonce_b, "generated nonces must be unique");
        // The returned nonce decrypts its ciphertext.
        assert_eq!(
            key.decrypt(&nonce_a, b"aad", &ct_a)
                .expect("decryption should succeed"),
            b"plaintext"
        );
    }

    #[test]
    fn generated_nonce_appended_layout_round_trips() {
        let mut key = HardwareAes256Gcm::new(&[7_u8; 32]).expect("valid test key");
        let blob = key
            .encrypt_nonce_appended_generated(b"plaintext")
            .expect("encryption should succeed");
        assert_eq!(
            blob.len(),
            b"plaintext".len() + super::TAG_SIZE + NONCE_SIZE
        );
        assert_eq!(
            key.decrypt_nonce_appended(&blob)
                .expect("decryption should succeed"),
            b"plaintext"
        );
    }

    #[test]
    fn caller_placed_generated_nonce_round_trips() {
        let layout = HardwareAes256Gcm::key_state_layout();
        let mut storage = AlignedStorage::<512>([0_u8; 512]);
        let slot = UninitKeyStateSlot::new(&mut storage.0[..layout.size]).expect("valid slot");
        let mut key = HardwareAes256GcmIn::new_in(&[7_u8; 32], slot).expect("valid test key");
        let (nonce, ct) = key
            .encrypt_with_generated_nonce(&[], b"plaintext")
            .expect("encryption should succeed");
        assert_eq!(
            key.decrypt(&nonce, &[], &ct)
                .expect("decryption should succeed"),
            b"plaintext"
        );
    }

    #[test]
    fn generated_nonce_does_not_change_key_state_size() {
        // The nonce generator lives on the handle, not the placed key state.
        assert_eq!(HardwareAes256Gcm::key_state_layout().size, 368);
        assert_eq!(HardwareAes256Gcm::state_size(), 368);
    }

    #[test]
    fn inline_owned_key_state_layout_matches_reported_layout() {
        let layout = HardwareAes256Gcm::key_state_layout();
        assert_eq!(
            std::mem::size_of::<HardwareAes256GcmKeyState>(),
            layout.size
        );
        assert_eq!(
            std::mem::align_of::<HardwareAes256GcmKeyState>(),
            layout.align
        );
        assert_eq!(HardwareAes256GcmKeyState::state_size(), layout.size);
    }

    #[test]
    fn inline_owned_key_state_round_trips_and_uses_existing_vec_capacity() {
        let key = HardwareAes256GcmKeyState::new(&[7_u8; 32]).expect("valid test key");
        let nonce = [9_u8; NONCE_SIZE];
        let expected = key
            .encrypt_nonce_appended(&nonce, b"plaintext")
            .expect("encryption should succeed");
        assert_eq!(
            key.decrypt_nonce_appended(&expected)
                .expect("decryption should succeed"),
            b"plaintext"
        );

        let mut in_out = Vec::with_capacity(b"plaintext".len() + super::TAG_SIZE + NONCE_SIZE);
        in_out.extend_from_slice(b"plaintext");
        let ptr_before = in_out.as_ptr();
        key.encrypt_nonce_appended_in_place(&nonce, &mut in_out)
            .expect("in-place encryption should succeed");
        assert_eq!(in_out, expected);
        assert_eq!(
            in_out.as_ptr(),
            ptr_before,
            "pre-reserved in-place encryption should not reallocate"
        );
    }

    #[test]
    fn inline_owned_key_state_wipes_storage_on_drop() {
        let layout = HardwareAes256Gcm::key_state_layout();
        let mut key =
            ManuallyDrop::new(HardwareAes256GcmKeyState::new(&[7_u8; 32]).expect("valid test key"));
        let storage = key.storage.bytes_ptr_mut();

        // SAFETY: key lives in this stack frame. ManuallyDrop lets us run its
        // Drop impl while retaining access to the backing bytes to verify the
        // wipe before the stack slot is reused.
        unsafe {
            ManuallyDrop::drop(&mut key);
            let bytes = std::slice::from_raw_parts(storage.cast_const(), layout.size);
            assert!(bytes.iter().all(|byte| *byte == 0));
        }
    }

    #[test]
    fn caller_placed_key_state_is_usable_across_threads() {
        let layout = HardwareAes256Gcm::key_state_layout();
        let mut storage = AlignedStorage::<512>([0_u8; 512]);
        let slot = UninitKeyStateSlot::new(&mut storage.0[..layout.size]).expect("valid slot");
        let key = HardwareAes256GcmIn::new_in(&[7_u8; 32], slot).expect("valid test key");
        let nonce = [9_u8; NONCE_SIZE];
        let ciphertext = key
            .encrypt(&nonce, &[], b"plaintext")
            .expect("encryption should succeed");

        std::thread::scope(|scope| {
            for _ in 0..4 {
                scope.spawn(|| {
                    assert_eq!(
                        key.decrypt(&nonce, &[], &ciphertext)
                            .expect("shared decryption should succeed"),
                        b"plaintext"
                    );
                });
            }
        });
    }

    #[test]
    fn rejects_wrong_nonce_length() {
        let key = HardwareAes256Gcm::new(&[7_u8; 32]).expect("valid test key");
        let Err(err) = key.encrypt(&[0_u8; NONCE_SIZE - 1], &[], b"data") else {
            panic!("short nonce should be rejected");
        };
        assert_eq!(err, Error::InvalidNonceLength);
    }

    #[test]
    #[cfg(target_pointer_width = "64")]
    fn rejects_inputs_that_exceed_gcm_counter_limit() {
        let max_data_len =
            usize::try_from(super::MAX_GCM_DATA_LEN).expect("64-bit usize holds GCM data limit");
        assert_eq!(
            super::validate_gcm_lengths(0, max_data_len + 1),
            Err(Error::InputTooLarge)
        );
    }

    #[test]
    #[cfg(target_pointer_width = "64")]
    fn accepts_inputs_at_gcm_counter_limit() {
        let max_data_len =
            usize::try_from(super::MAX_GCM_DATA_LEN).expect("64-bit usize holds GCM data limit");
        assert_eq!(super::validate_gcm_lengths(0, max_data_len), Ok(()));
    }

    #[test]
    fn nonce_appended_round_trips() {
        let key = HardwareAes256Gcm::new(&[7_u8; 32]).expect("valid test key");
        let nonce = [9_u8; NONCE_SIZE];
        let ciphertext = key
            .encrypt_nonce_appended(&nonce, b"plaintext")
            .expect("encryption should succeed");
        assert_eq!(
            key.decrypt_nonce_appended(&ciphertext)
                .expect("decryption should succeed"),
            b"plaintext"
        );
    }

    #[test]
    fn reports_nonzero_key_state_layout() {
        let layout = HardwareAes256Gcm::key_state_layout();
        assert!(layout.size >= super::KEY_SIZE);
        assert!(layout.align.is_power_of_two());
        assert!(layout.size <= 384);
        assert_eq!(layout.size, HardwareAes256Gcm::state_size());
    }

    #[test]
    fn caller_placed_key_state_round_trips_and_wipes_storage() {
        let layout = HardwareAes256Gcm::key_state_layout();
        let mut storage = AlignedStorage::<512>([0xa5; 512]);
        let nonce = [9_u8; NONCE_SIZE];

        {
            let slot = UninitKeyStateSlot::new(&mut storage.0[..layout.size]).expect("valid slot");
            let key = HardwareAes256GcmIn::new_in(&[7_u8; 32], slot).expect("valid test key");
            let ciphertext = key
                .encrypt_nonce_appended(&nonce, b"plaintext")
                .expect("encryption should succeed");
            assert_eq!(
                key.decrypt_nonce_appended(&ciphertext)
                    .expect("decryption should succeed"),
                b"plaintext"
            );
        }

        assert!(storage.0[..layout.size].iter().all(|byte| *byte == 0));
    }

    #[test]
    fn caller_placed_key_state_validates_storage_before_key() {
        let layout = HardwareAes256Gcm::key_state_layout();
        let mut short = AlignedStorage::<512>([0_u8; 512]);
        let err = UninitKeyStateSlot::new(&mut short.0[..layout.size - 1])
            .expect_err("short storage should fail");
        assert_eq!(err, Error::KeyStateStorageTooSmall);

        let mut misaligned = AlignedStorage::<512>([0_u8; 512]);
        let start = 1;
        let end = start + layout.size;
        let err = UninitKeyStateSlot::new(&mut misaligned.0[start..end])
            .expect_err("misaligned storage should fail");
        assert_eq!(err, Error::KeyStateStorageMisaligned);
    }
}