syd 3.52.0

//
// Syd: rock-solid application kernel
// src/workers/aes.rs: `syd_aes' encryption thread
//
// Copyright (c) 2024, 2025, 2026 Ali Polatel <alip@chesswob.org>
//
// SPDX-License-Identifier: GPL-3.0

// SAFETY:
// 1. This module has (almost) been liberated from unsafe code.
//    Owner::from_raw_fd is used for crypt_fd which is unsafe.
//    Use deny rather than forbid so we can allow this case.
// 2. This module forbids arithmetic side effects, et al.
#![deny(unsafe_code)]
#![forbid(clippy::arithmetic_side_effects)]
#![forbid(clippy::cast_possible_truncation)]
#![forbid(clippy::cast_possible_wrap)]
#![forbid(clippy::cast_sign_loss)]

use std::{
    collections::hash_map::Entry,
    fmt,
    os::fd::{AsFd, AsRawFd, FromRawFd, RawFd},
    sync::{
        atomic::{AtomicBool, Ordering},
        Arc, Condvar, Mutex,
    },
    thread,
};

use libseccomp::{scmp_cmp, ScmpAction, ScmpFilterContext, ScmpSyscall};
use nix::{
    errno::Errno,
    fcntl::{posix_fadvise, splice, tee, OFlag, PosixFadviseAdvice, SpliceFFlags},
    sched::{unshare, CloneFlags},
    unistd::{lseek64, write, Gid, Uid, Whence},
};
use serde::{Serialize, Serializer};

#[cfg(target_arch = "x86")]
use crate::cookie::CookieIdx::Ftruncate64Arg3;
use crate::{
    alert,
    config::*,
    confine::{
        confine_scmp_accept4, confine_scmp_clone, confine_scmp_clone3, confine_scmp_close,
        confine_scmp_fadvise, confine_scmp_fcntl, confine_scmp_ftruncate, confine_scmp_madvise,
        confine_scmp_open_stat, confine_scmp_pipe2, confine_scmp_prctl, confine_scmp_recvmsg,
        confine_scmp_sendmsg, confine_scmp_setid, confine_scmp_sigaction, confine_scmp_write,
        confine_scmp_wx_syd, secure_getenv, ExportMode,
    },
    cookie::{safe_ftruncate64, safe_pipe2},
    err::{err2no, SydJoinHandle, SydResult},
    error,
    fd::{seal_memfd_all, SafeOwnedFd},
    hash::{
        aes_ctr_enc, aes_ctr_init, hmac_sha256_feed, hmac_sha256_fini, hmac_sha256_init,
        SydHashMap, BLOCK_SIZE, CRYPT_MAGIC_OFFSET, HMAC_TAG_SIZE, IV, IV_SIZE, SYD3_HDR_OFFSET,
        SYD3_HDR_SIZE,
    },
    info,
    landlock::Errata,
    landlock_policy::LandlockPolicy,
    lookup::FileInfo,
    ofd::lock_fd,
    path::{XPath, XPathBuf},
    retry::{retry_on_eintr, retry_on_intr},
    sandbox::Options,
};

#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd)]
pub(crate) enum AesMod {
    Read,
    Append,
    Write,
}

impl fmt::Display for AesMod {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::Read => write!(f, "read"),
            Self::Append => write!(f, "append"),
            Self::Write => write!(f, "write"),
        }
    }
}

impl Serialize for AesMod {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        serializer.serialize_str(&self.to_string())
    }
}

impl From<OFlag> for AesMod {
    fn from(flags: OFlag) -> Self {
        if flags.contains(OFlag::O_WRONLY) || flags.contains(OFlag::O_RDWR) {
            if flags.contains(OFlag::O_APPEND) {
                Self::Append
            } else {
                Self::Write
            }
        } else {
            Self::Read
        }
    }
}

#[derive(Clone)]
pub(crate) struct AesVal {
    pub(crate) crypt_fd: RawFd,
    pub(crate) enc_fd: RawFd,
    pub(crate) iv: Option<IV>, // None means encryption in progress.
    pub(crate) info: FileInfo,
    pub(crate) mode: AesMod,
}

pub(crate) type AesMap = SydHashMap<XPathBuf, AesVal>;
pub(crate) type AesLock = Arc<(Mutex<AesMap>, Condvar)>;

#[derive(Clone)]
pub(crate) struct AesWorker {
    fdalg: (RawFd, RawFd),
    files: AesLock,
    options: Options,
    is_memfd: bool,
    should_exit: Arc<AtomicBool>,
    transit_uids: Vec<(Uid, Uid)>,
    transit_gids: Vec<(Gid, Gid)>,
}

impl AesWorker {
    pub(crate) fn new(
        fdalg: (RawFd, RawFd),
        files: AesLock,
        options: Options,
        is_memfd: bool,
        should_exit: Arc<AtomicBool>,
        transit_uids: &[(Uid, Uid)],
        transit_gids: &[(Gid, Gid)],
    ) -> Self {
        Self {
            fdalg,
            files,
            options,
            is_memfd,
            should_exit,
            transit_uids: transit_uids.to_vec(),
            transit_gids: transit_gids.to_vec(),
        }
    }

    // Confine AES thread.
    #[expect(clippy::cognitive_complexity)]
    pub(crate) fn prepare_confine(
        options: Options,
        transit_uids: &[(Uid, Uid)],
        transit_gids: &[(Gid, Gid)],
        dry_run: bool,
    ) -> SydResult<ScmpFilterContext> {
        if !dry_run {
            // Set up a landlock(7) sandbox to disallow all access.
            let abi = crate::landlock::ABI::new_current();
            let errata = crate::landlock::Errata::query();
            let policy = LandlockPolicy {
                scoped_abs: true,
                scoped_sig: errata.contains(Errata::SCOPED_SIGNAL_SAME_TGID),
                ..Default::default()
            };
            let _ = policy.restrict_self(abi);
        }

        // Create seccomp filter with default action.
        let mut ctx = ScmpFilterContext::new(ScmpAction::KillProcess)?;

        // Enforce the NO_NEW_PRIVS functionality before
        // loading the seccomp filter into the kernel.
        ctx.set_ctl_nnp(true)?;

        // Disable Speculative Store Bypass mitigations
        // with trace/allow_unsafe_exec_speculative:1
        ctx.set_ctl_ssb(options.allow_unsafe_exec_speculative())?;

        // DO NOT synchronize filter to all threads.
        // Main thread will confine itself.
        ctx.set_ctl_tsync(false)?;

        // We kill for bad system call and bad arch.
        ctx.set_act_badarch(ScmpAction::KillProcess)?;

        // Use a binary tree sorted by syscall number if possible.
        let _ = ctx.set_ctl_optimize(2);

        // Syscall argument cookies may be disabled
        // at startup with trace/allow_unsafe_nocookie:1.
        let restrict_cookie = !options.allow_unsafe_nocookie();

        // Do NOT add supported architectures to the filter.
        // This ensures Syd can never run a non-native system call,
        // which we do not need at all.
        // seccomp_add_architectures(&mut ctx)?;

        // Deny open and stat family with ENOSYS rather than KillProcess.
        confine_scmp_open_stat(&mut ctx, true /* openat2 */)?;

        // Allow reads up to MAX(HMAC | IV) bytes.
        // See the note in sync_file().
        //
        // IMPORTANT: Because of this rule, log invocations after
        // sandboxing must NOT include the `pid` key which will cause an
        // immediate Syd crash with an audit log entry.
        let rw_max: u64 = HMAC_TAG_SIZE
            .max(IV_SIZE)
            .try_into()
            .or(Err(Errno::EOVERFLOW))?;
        match ScmpSyscall::from_name("read") {
            Ok(syscall) => {
                ctx.add_rule_conditional(
                    ScmpAction::Allow,
                    syscall,
                    &[scmp_cmp!($arg2 <= rw_max)],
                )?;
            }
            Err(_) => {
                info!("ctx": "confine", "op": "allow_aes_syscall",
                    "msg": "invalid or unsupported syscall read");
            }
        }

        // Prevent executable memory.
        confine_scmp_wx_syd(&mut ctx)?;

        // Allow writes to log-fd OR up to MAX(HMAC | IV) bytes.
        // No proc_pid_mem(5) access required here.
        confine_scmp_write(&mut ctx, Some(rw_max), false)?;

        // Allow clones without namespace flags.
        confine_scmp_clone(&mut ctx)?;
        // Deny clone3 with ENOSYS for compatibility.
        confine_scmp_clone3(&mut ctx)?;

        // Allow fadvise family syscalls.
        confine_scmp_fadvise(&mut ctx)?;

        // Allow safe madvise(2) advice.
        confine_scmp_madvise(&mut ctx)?;

        // ftruncate{,64}(2) may be used only with syscall argument cookies.
        confine_scmp_ftruncate(&mut ctx, restrict_cookie)?;

        // accept4(2) may be used only with syscall argument cookies.
        confine_scmp_accept4(&mut ctx, restrict_cookie)?;

        // recvmsg(2)/ sendmsg(2) may be used only with syscall argument cookies.
        confine_scmp_recvmsg(&mut ctx, restrict_cookie)?;
        confine_scmp_sendmsg(&mut ctx, restrict_cookie)?;

        // close(2) may be used only with syscall argument cookies.
        confine_scmp_close(&mut ctx, restrict_cookie)?;

        // Restrict pipe2(2) flags, and use syscall argument cookies.
        confine_scmp_pipe2(&mut ctx, restrict_cookie, OFlag::O_CLOEXEC)?;

        // Allow safe fcntl(2) utility calls.
        confine_scmp_fcntl(&mut ctx, AES_FCNTL_OPS)?;

        // Allow safe prctl(2) operations.
        confine_scmp_prctl(&mut ctx, AES_PRCTL_OPS)?;

        // Deny installing new signal handlers for {rt_,}sigaction(2).
        confine_scmp_sigaction(&mut ctx)?;

        // Allow safe system calls.
        //
        // KCOV_SYSCALLS is empty in case `kcov` feature is disabled.
        for sysname in AES_SYSCALLS
            .iter()
            .chain(ALLOC_SYSCALLS)
            .chain(FUTEX_SYSCALLS)
            .chain(GETID_SYSCALLS)
            .chain(KCOV_SYSCALLS)
            .chain(VDSO_SYSCALLS)
        {
            match ScmpSyscall::from_name(sysname) {
                Ok(syscall) => {
                    ctx.add_rule(ScmpAction::Allow, syscall)?;
                }
                Err(_) => {
                    info!("ctx": "confine", "op": "allow_aes_syscall",
                        "msg": format!("invalid or unsupported syscall {sysname}"));
                }
            }
        }

        // Allow UID/GID changing system calls as necessary.
        let safe_setuid = options.allow_safe_setuid();
        let safe_setgid = options.allow_safe_setgid();
        if safe_setuid || safe_setgid {
            confine_scmp_setid(
                "aes",
                &mut ctx,
                safe_setuid,
                safe_setgid,
                transit_uids,
                transit_gids,
            )?;
        }

        Ok(ctx)
    }

    #[expect(clippy::cognitive_complexity)]
    pub(crate) fn try_spawn(self) -> Result<SydJoinHandle<()>, Errno> {
        thread::Builder::new()
            .name("syd_aes".to_string())
            .stack_size(AES_STACK_SIZE)
            .spawn(move || {
                // We use exit_group(2) here to bail, because this
                // unsharing is a critical safety feature. CLONE_FILES
                // can't be unshared because this thread must share file
                // descriptors with the emulator threads.
                if let Err(errno) = unshare(CloneFlags::CLONE_FS | CloneFlags::CLONE_SYSVSEM) {
                    alert!("ctx": "boot", "op": "unshare_aes_thread",
                        "msg": format!("failed to unshare(CLONE_FS|CLONE_SYSVSEM): {errno}"),
                        "err": errno as i32);
                    std::process::exit(101);
                }

                // Honour dry-run when exporting.
                let dry_run =
                    secure_getenv(ENV_SKIP_SCMP).is_some() || ExportMode::from_env().is_some();

                if !dry_run {
                    // Load the filter immediately.
                    // Logging is permitted as long as the `pid` key is unused.
                    // See prepare_confine for more information.
                    let ctx = Self::prepare_confine(
                        self.options,
                        &self.transit_uids,
                        &self.transit_gids,
                        false,
                    )?;
                    ctx.load()?;

                    let safe_setid = self.options.intersects(
                        Options::OPT_ALLOW_SAFE_SETUID | Options::OPT_ALLOW_SAFE_SETGID,
                    );
                    info!("ctx": "confine", "op": "confine_aes_thread",
                        "msg": format!("AES thread confined with{} SROP mitigation",
                            if safe_setid { "out" } else { "" }));
                } else {
                    error!("ctx": "confine", "op": "confine_aes_thread",
                        "msg": "AES threads are running unconfined in debug mode");
                }

                // Enter main loop.
                Self::main(self.fdalg, self.files, self.is_memfd, self.should_exit)
            })
            .map_err(|err| err2no(&err))
    }

    fn main(
        fdalg: (RawFd, RawFd),
        files: AesLock,
        is_memfd: bool,
        should_exit: Arc<AtomicBool>,
    ) -> SydResult<()> {
        let (aes_map, cvar) = &*files;
        let mut batches = Vec::new();
        let mut threads: Vec<Option<SydJoinHandle<()>>> = Vec::with_capacity(*NPROC);
        loop {
            // Check if there're any pending encryption requests.
            let mut aes_map = aes_map.lock().unwrap_or_else(|e| e.into_inner());
            aes_map = cvar
                .wait_while(aes_map, |map| {
                    map.is_empty() && !should_exit.load(Ordering::Acquire)
                })
                .unwrap_or_else(|e| e.into_inner());

            // Exit only when there's no pending work.
            if aes_map.is_empty() && should_exit.load(Ordering::Acquire) {
                break;
            }

            for (crypt_path, crypt_data) in aes_map.iter_mut() {
                if let Some(iv) = crypt_data.iv.take() {
                    let crypt_data = AesVal {
                        iv: Some(iv),
                        ..*crypt_data
                    };
                    batches.push((crypt_path.clone(), crypt_data));
                }
            }
            drop(aes_map); // Release the lock.

            // Join finished threads.
            threads.retain_mut(|thread| {
                if thread.as_ref().is_some_and(|t| t.is_finished()) {
                    if let Some(thread) = thread.take() {
                        let _ = thread.join();
                    }
                    false // remove
                } else {
                    true // retain
                }
            });

            // Spawn threads to handle pending encryption requests.
            for (crypt_path, crypt_data) in batches.drain(..) {
                threads.push(Some(Self::spawn(
                    fdalg,
                    &files,
                    &crypt_path,
                    crypt_data,
                    is_memfd,
                )?));
            }
        }

        // Wait for the ongoing encryption operations before exiting.
        for thread in threads.into_iter().flatten() {
            let _ = thread.join();
        }

        Ok(())
    }

    fn spawn(
        fdalg: (RawFd, RawFd),
        files: &AesLock,
        crypt_path: &XPath,
        crypt_data: AesVal,
        memfd: bool,
    ) -> SydResult<SydJoinHandle<()>> {
        let handle = retry_on_intr(|| {
            let files = Arc::clone(files);
            let crypt_data = crypt_data.clone();
            let crypt_path = crypt_path.to_owned();

            thread::Builder::new()
                .name("syd_aes".into())
                .stack_size(AES_STACK_SIZE)
                .spawn(move || {
                    // SAFETY: crypt_map keys are valid FDs.
                    #[expect(unsafe_code)]
                    let crypt_fd = unsafe { SafeOwnedFd::from_raw_fd(crypt_data.crypt_fd) };

                    // Wait until we take a write lock on the encrypted fd.
                    // This will succeed once all fds owned by the sandbox
                    // process are closed.
                    retry_on_eintr(|| lock_fd(&crypt_fd, true, true))?;

                    // Sync contents to disk.
                    let result = Self::sync(fdalg, &crypt_fd, crypt_data, memfd);

                    // Safe to remove file entry now if entry is still ours.
                    {
                        let (aes_map, _cvar) = &*files;
                        let mut aes_map = aes_map.lock().unwrap_or_else(|e| e.into_inner());
                        if let Entry::Occupied(entry) = aes_map.entry(crypt_path) {
                            let iv = entry.get().iv.as_ref();
                            let fd = entry.get().crypt_fd;
                            if iv.is_none() && fd == crypt_fd.as_raw_fd() {
                                entry.remove();
                            }
                        }
                    } // Lock is released here.

                    // Close the encrypted FD.
                    drop(crypt_fd);

                    result
                })
                .map_err(|err| err2no(&err))
        })?;

        Ok(handle)
    }

    fn sync<Fd: AsFd>(
        fdalg: (RawFd, RawFd),
        crypt_fd: Fd,
        crypt_data: AesVal,
        memfd: bool,
    ) -> SydResult<()> {
        if memfd {
            // Seal memfd to ensure no further writes happen.
            seal_memfd_all(&crypt_fd)?;
        }

        let (aes_fd, mac_fd) = fdalg;
        let file_mode = crypt_data.mode;

        // syd_aes thread steals the IV, therefore it is always Some.
        #[expect(clippy::disallowed_methods)]
        let mut iv = crypt_data.iv.unwrap();

        // SAFETY: crypt_data.enc_fd is a valid FD.
        #[expect(unsafe_code)]
        let enc_fd = unsafe { SafeOwnedFd::from_raw_fd(crypt_data.enc_fd) };

        // Nothing to do if file was readonly.
        let mut is_append = match file_mode {
            AesMod::Read => return Ok(()),
            AesMod::Append => true,
            _ => false,
        };

        // Handle truncation quickly.
        let data_size: u64 = lseek64(&crypt_fd, 0, Whence::SeekEnd)?
            .try_into()
            .or(Err(Errno::EOVERFLOW))?;
        if data_size == 0 {
            retry_on_eintr(|| safe_ftruncate64(&enc_fd, 0))?;
            return Ok(());
        }

        // Handle opened for append but encrypted file is new.
        let mut file_size: u64 = lseek64(&enc_fd, 0, Whence::SeekEnd)?
            .try_into()
            .or(Err(Errno::EOVERFLOW))?;
        if is_append && file_size == 0 {
            is_append = false;
        }

        // Handle opened for append but appended nothing quickly.
        if is_append && data_size <= file_size.saturating_sub(SYD3_HDR_SIZE) {
            return Ok(());
        }

        // We handled quick cases, before possibly truncating the
        // encrypted file, let's ensure we open the connections as
        // expected, and use posix_fadvise(2) to hint the kernel about
        // I/O access patterns.

        // Initialize HMAC socket and feed magic header and IV.
        let sock_mac = hmac_sha256_init(&mac_fd, false)?;
        hmac_sha256_feed(&sock_mac, CRYPT_MAGIC, true)?;
        hmac_sha256_feed(&sock_mac, iv.as_ref(), true)?;
        let (pipe_rd_mac, pipe_wr_mac) = safe_pipe2(OFlag::O_CLOEXEC)?;

        // Hint the kernel about I/O access patterns.
        Self::advise_io(&crypt_fd, &enc_fd)?;

        // Handle last block re-encryption for append.
        if is_append {
            // Adjust file_size to exclude the header.
            file_size = file_size
                .checked_sub(SYD3_HDR_SIZE)
                .ok_or(Errno::EOVERFLOW)?;

            // Get offset of the last full block.
            let last_block_offset = Self::get_last_block_offset(file_size)?;

            // Adjust the IV counter based on the last full block offset.
            iv.add_counter(last_block_offset);

            // Position crypt_fd offset for append.
            Self::seek2append(&crypt_fd, last_block_offset, file_size)?;

            // Ensure no stale bytes from the last partial block survive.
            assert!(
                data_size >= file_size,
                "BUG: stale bytes in last partial block, report a bug!"
            );

            // Feed kept ciphertext prefix into HMAC.
            Self::prefeed_hmac(
                &enc_fd,
                &sock_mac,
                &pipe_rd_mac,
                &pipe_wr_mac,
                last_block_offset,
            )?;
        } else {
            // Non-append mode: overwrite the file.

            // Reset crypt_fd to the beginning.
            lseek64(&crypt_fd, 0, Whence::SeekSet)?;

            if file_size > 0 {
                // Remove previous content, wipe IV to avoid reuse.
                retry_on_eintr(|| safe_ftruncate64(&enc_fd, 0))?;
                lseek64(&enc_fd, 0, Whence::SeekSet)?;
            }

            // Write encrypted file header with the given IV.
            Self::write_header(&enc_fd, &iv)?;
        }

        // Initialize encryption socket, and set IV.
        let sock_enc = aes_ctr_init(&aes_fd, false)?;
        aes_ctr_enc(&sock_enc, &[], Some(&iv), true)?;

        // IV is no longer needed (zeroized on Drop).
        drop(iv);

        let (pipe_rd_enc, pipe_wr_enc) = safe_pipe2(OFlag::O_CLOEXEC)?;

        // Feed plaintext into AES & HMAC algorithm sockets.
        Self::aes_feed(
            (&crypt_fd, &enc_fd),
            (&sock_enc, &sock_mac),
            (&pipe_rd_enc, &pipe_wr_enc, &pipe_rd_mac, &pipe_wr_mac),
        )?;

        // Write HMAC tag to the encrypted file.
        Self::write_hmac(&enc_fd, &sock_mac)?;

        Ok(())
    }

    // Hint the kernel about I/O access patterns.
    fn advise_io<Fd1: AsFd, Fd2: AsFd>(crypt_fd: Fd1, enc_fd: Fd2) -> Result<(), Errno> {
        // Mark enc_fd as SEQUENTIAL before writes to encourage
        // clustered I/O and reduce random writeback patterns.
        posix_fadvise(&enc_fd, 0, 0, PosixFadviseAdvice::POSIX_FADV_SEQUENTIAL)?;

        // Set crypt_fd SEQUENTIAL|WILLNEED before the main pump to
        // prime readahead and avoid small read bursts starving the
        // AF_ALG pipeline.
        posix_fadvise(&crypt_fd, 0, 0, PosixFadviseAdvice::POSIX_FADV_SEQUENTIAL)?;
        posix_fadvise(&crypt_fd, 0, 0, PosixFadviseAdvice::POSIX_FADV_WILLNEED)?;

        Ok(())
    }

    // Returns the offset of the last full block.
    //
    // File size doesn't include the header size.
    fn get_last_block_offset(file_size: u64) -> Result<u64, Errno> {
        let remainder = file_size
            .checked_rem(BLOCK_SIZE as u64)
            .ok_or(Errno::EOVERFLOW)?;
        file_size.checked_sub(remainder).ok_or(Errno::EOVERFLOW)
    }

    // Positions the plaintext fd offset for append.
    //
    // If there is a partial block at the end, we need to re-encrypt it.
    // Last block offset is the offset of the last full block.
    // File size doesn't include the header size.
    fn seek2append<Fd: AsFd>(
        crypt_fd: Fd,
        last_block_offset: u64,
        file_size: u64,
    ) -> Result<(), Errno> {
        if last_block_offset < file_size {
            // Adjust crypt_fd to read from the last full block offset.
            let off: i64 = last_block_offset.try_into().or(Err(Errno::EOVERFLOW))?;
            lseek64(crypt_fd, off, Whence::SeekSet)
        } else {
            // No partial block, start reading from the current file size.
            let off: i64 = file_size.try_into().or(Err(Errno::EOVERFLOW))?;
            lseek64(crypt_fd, off, Whence::SeekSet)
        }
        .map(drop)
    }

    // Read from the encrypted file starting after the header.
    fn prefeed_hmac<Fd1: AsFd, Fd2: AsFd, Fd3: AsFd, Fd4: AsFd>(
        enc_fd: Fd1,
        sock_mac: Fd2,
        pipe_rd_mac: Fd3,
        pipe_wr_mac: Fd4,
        last_block_offset: u64,
    ) -> Result<(), Errno> {
        lseek64(&enc_fd, SYD3_HDR_OFFSET, Whence::SeekSet)?;

        let mut remain: usize = last_block_offset.try_into().or(Err(Errno::EOVERFLOW))?;
        while remain > 0 {
            let n = retry_on_eintr(|| {
                splice(
                    &enc_fd,
                    None,
                    &pipe_wr_mac,
                    None,
                    remain.min(PIPE_BUF_ALG),
                    SpliceFFlags::empty(),
                )
            })?;
            if n == 0 {
                // Unexpected EOF, concurrent shrink/truncation?
                return Err(Errno::EIO);
            }

            let mut ncopy = n;
            while ncopy > 0 {
                let n = retry_on_eintr(|| {
                    splice(
                        &pipe_rd_mac,
                        None,
                        &sock_mac,
                        None,
                        ncopy,
                        SpliceFFlags::SPLICE_F_MORE,
                    )
                })?;
                if n == 0 {
                    return Err(Errno::EBADMSG);
                }
                ncopy = ncopy.checked_sub(n).ok_or(Errno::EOVERFLOW)?;
            }

            remain = remain.checked_sub(n).ok_or(Errno::EOVERFLOW)?;
        }

        Ok(())
    }

    // Write encrypted file header with the given IV.
    //
    // write(2) is allowed up to 32 bytes by seccomp(2).
    fn write_header<Fd: AsFd>(enc_fd: Fd, iv: &IV) -> Result<(), Errno> {
        Self::write_all(&enc_fd, CRYPT_MAGIC)?;
        Self::write_all(&enc_fd, &[0u8; HMAC_TAG_SIZE])?;
        Self::write_all(&enc_fd, iv.as_ref())?;
        Ok(())
    }

    // Feed plaintext into AES & HMAC algorithm sockets.
    fn aes_feed<
        Fd1: AsFd,
        Fd2: AsFd,
        Fd3: AsFd,
        Fd4: AsFd,
        Fd5: AsFd,
        Fd6: AsFd,
        Fd7: AsFd,
        Fd8: AsFd,
    >(
        crypt_fds: (Fd1, Fd2),
        sock_fds: (Fd3, Fd4),
        pipe_fds: (Fd5, Fd6, Fd7, Fd8),
    ) -> Result<(), Errno> {
        let (crypt_fd, enc_fd) = crypt_fds;
        let (sock_enc, sock_mac) = sock_fds;
        let (pipe_rd_enc, pipe_wr_enc, pipe_rd_mac, pipe_wr_mac) = pipe_fds;

        // Feed plaintext via zero-copy into the kernel socket.
        let mut nflush = 0usize;
        loop {
            let nfeed = retry_on_eintr(|| {
                splice(
                    &crypt_fd,
                    None,
                    &pipe_wr_enc,
                    None,
                    PIPE_BUF_ALG,
                    SpliceFFlags::empty(),
                )
            })?;
            if nfeed == 0 {
                break;
            }

            // splice(2) plaintext into AES socket.
            Self::splice_all(&pipe_rd_enc, &sock_enc, nfeed, SpliceFFlags::SPLICE_F_MORE)?;

            nflush = nflush.checked_add(nfeed).ok_or(Errno::EOVERFLOW)?;
            while nflush >= BLOCK_SIZE {
                let rem = nflush.checked_rem(BLOCK_SIZE).ok_or(Errno::EOVERFLOW)?;
                let len = nflush.checked_sub(rem).ok_or(Errno::EOVERFLOW)?;

                // splice(2) len bytes of ciphertext from AES socket into enc pipe.
                let n = Self::splice_nonzero(
                    &sock_enc,
                    &pipe_wr_enc,
                    len,
                    SpliceFFlags::SPLICE_F_MORE,
                )?;

                // Duplicate data from encryption pipe to the MAC pipe using tee(2).
                Self::tee_all(&pipe_rd_enc, &pipe_wr_mac, n)?;

                // Splice encrypted data to output file.
                Self::splice_all(&pipe_rd_enc, &enc_fd, n, SpliceFFlags::empty())?;
                nflush = nflush.checked_sub(n).ok_or(Errno::EOVERFLOW)?;

                // Splice duplicated data to HMAC socket.
                Self::splice_all(&pipe_rd_mac, &sock_mac, n, SpliceFFlags::SPLICE_F_MORE)?;
            }
        }

        // Flush the final batch.
        while nflush > 0 {
            // Finalize encryption with `false`.
            //
            // Some kernel versions may incorrectly return EINVAL here.
            // Gracefully handle this errno and move on.
            match aes_ctr_enc(&sock_enc, &[], None, false) {
                Ok(_) | Err(Errno::EINVAL) => {}
                Err(errno) => return Err(errno),
            }

            let len = nflush.min(PIPE_BUF_ALG);
            let n = Self::splice_nonzero(&sock_enc, &pipe_wr_enc, len, SpliceFFlags::empty())?;

            // Duplicate data from encryption pipe to the MAC pipe using tee(2).
            Self::tee_all(&pipe_rd_enc, &pipe_wr_mac, n)?;

            // Splice encrypted data to output file.
            Self::splice_all(&pipe_rd_enc, &enc_fd, n, SpliceFFlags::empty())?;
            nflush = nflush.checked_sub(n).ok_or(Errno::EOVERFLOW)?;

            // Splice duplicated data to HMAC socket.
            Self::splice_all(&pipe_rd_mac, &sock_mac, n, SpliceFFlags::SPLICE_F_MORE)?;
        }

        Ok(())
    }

    // Finalize HMAC computation, retrieve the tag and write to the encrypted file.
    //
    // read(2) is allowed up to 32 bytes by seccomp(2).
    fn write_hmac<Fd1: AsFd, Fd2: AsFd>(enc_fd: Fd1, sock_mac: Fd2) -> Result<(), Errno> {
        let tag = hmac_sha256_fini(&sock_mac)?;
        lseek64(&enc_fd, CRYPT_MAGIC_OFFSET, Whence::SeekSet)?;
        Self::write_all(&enc_fd, tag.as_slice())
    }

    // Drain exactly N bytes with checked subtraction.
    fn splice_all<Fd1: AsFd, Fd2: AsFd>(
        src: Fd1,
        dst: Fd2,
        mut len: usize,
        flags: SpliceFFlags,
    ) -> Result<(), Errno> {
        while len > 0 {
            let n = Self::splice_nonzero(&src, &dst, len, flags)?;
            len = len.checked_sub(n).ok_or(Errno::EOVERFLOW)?;
        }
        Ok(())
    }

    // Duplicate exactly N bytes in PIPE_BUF_ALG-bounded chunks.
    fn tee_all<Fd1: AsFd, Fd2: AsFd>(src: Fd1, dst: Fd2, mut len: usize) -> Result<(), Errno> {
        while len > 0 {
            let n = Self::tee_nonzero(&src, &dst, len)?;
            len = len.checked_sub(n).ok_or(Errno::EOVERFLOW)?;
        }
        Ok(())
    }

    // splice(2) that must move >0 or EBADMSG.
    fn splice_nonzero<Fd1: AsFd, Fd2: AsFd>(
        src: Fd1,
        dst: Fd2,
        len: usize,
        flags: SpliceFFlags,
    ) -> Result<usize, Errno> {
        let n = retry_on_eintr(|| splice(&src, None, &dst, None, len, flags))?;
        if n > 0 {
            Ok(n)
        } else {
            Err(Errno::EBADMSG)
        }
    }

    // tee(2) that must move >0 or EBADMSG.
    fn tee_nonzero<Fd1: AsFd, Fd2: AsFd>(src: Fd1, dst: Fd2, len: usize) -> Result<usize, Errno> {
        let n = retry_on_eintr(|| tee(&src, &dst, len, SpliceFFlags::empty()))?;
        if n > 0 {
            Ok(n)
        } else {
            Err(Errno::EBADMSG)
        }
    }

    // write(2) that must write exactly given bytes or EINVAL.
    fn write_all<Fd: AsFd>(fd: Fd, buf: &[u8]) -> Result<(), Errno> {
        let mut nwrite = 0;
        while nwrite < buf.len() {
            match retry_on_eintr(|| write(&fd, &buf[nwrite..]))? {
                0 => return Err(Errno::EINVAL),
                n => nwrite = nwrite.checked_add(n).ok_or(Errno::EOVERFLOW)?,
            }
        }
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::hash::BLOCK_SIZE;

    #[test]
    fn test_get_last_block_offset_1() {
        let result = AesWorker::get_last_block_offset(0);
        assert_eq!(result, Ok(0));
    }

    #[test]
    fn test_get_last_block_offset_2() {
        let result = AesWorker::get_last_block_offset(BLOCK_SIZE as u64);
        assert_eq!(result, Ok(BLOCK_SIZE as u64));
    }

    #[test]
    fn test_get_last_block_offset_3() {
        let result = AesWorker::get_last_block_offset(BLOCK_SIZE as u64 + 1);
        assert_eq!(result, Ok(BLOCK_SIZE as u64));
    }

    #[test]
    fn test_get_last_block_offset_4() {
        let result = AesWorker::get_last_block_offset(2 * BLOCK_SIZE as u64);
        assert_eq!(result, Ok(2 * BLOCK_SIZE as u64));
    }

    #[test]
    fn test_get_last_block_offset_5() {
        let result = AesWorker::get_last_block_offset(1);
        assert_eq!(result, Ok(0));
    }

    #[test]
    fn test_get_last_block_offset_6() {
        let result = AesWorker::get_last_block_offset(BLOCK_SIZE as u64 - 1);
        assert_eq!(result, Ok(0));
    }
}