convert-rs 2.0.0

Extensible file conversion library with graph routing and optional native tools
Documentation
//! Out-of-process execution for native conversion tools.
//!
//! This module is the only place in the crate that spawns a child process. It
//! never routes input through a shell: the program name and every argument are
//! passed as discrete values, so file paths and format strings cannot be
//! parsed as commands. On Unix, children are additionally isolated: the
//! environment is scrubbed to `PATH`, `HOME`, and `TMPDIR`, POSIX resource
//! limits cap address space, CPU time, and file size, and each child runs in
//! its own session so a timeout kills its whole process group. Stdout and stderr are bounded, and a deadline kills and reaps the child.
//! These guards reduce blast radius but are not a security sandbox: native
//! tools still have this process's filesystem access, so external conversions
//! stay behind [`crate::Policy::allow_external`] and trusted inputs only.

use std::path::Path;
use std::process::Stdio;
use std::time::Duration;

use tokio::io::{AsyncRead, AsyncReadExt};

use crate::{Error, Limits, Result};

/// Hard ceiling on captured stderr. Tool chatter beyond this is dropped.
const STDERR_CAP: u64 = 256 * 1024;
/// How many trailing stderr bytes to surface in a failure message.
const ERROR_TAIL: usize = 4096;
/// A short, self-contained deadline for a presence probe.
const PROBE_TIMEOUT: Duration = Duration::from_secs(10);
/// Read size while draining a tool pipe.
const CHUNK: usize = 8 * 1024;

/// Run `program` with `args`, capturing bounded output, honoring `limits`.
///
/// The child's stdin is closed, its stdout and stderr are drained concurrently
/// to avoid pipe deadlock, and the whole run is wrapped in `limits.timeout`. On
/// timeout, output overflow, or a nonzero exit the child is killed (via
/// `kill_on_drop`) and a descriptive [`Error`] is returned. A missing binary
/// surfaces as [`Error::Io`].
pub async fn run(
    backend: &str,
    program: &str,
    args: &[String],
    cwd: Option<&Path>,
    limits: &Limits,
) -> Result<()> {
    let mut command = tokio::process::Command::new(program);
    command
        .args(args)
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .kill_on_drop(true);
    if let Some(directory) = cwd {
        command.current_dir(directory);
    }
    sandbox(&mut command, limits);
    let mut child = command.spawn().map_err(io_start_error(program))?;
    run_child(&mut child, backend, program, limits).await?;
    Ok(())
}

/// Run `program` with `args` and return its captured stdout on success.
///
/// Same guarantees as [`run`]: no shell, bounded pipes, scrubbed child
/// environment, and `limits.timeout` covering the whole run. Intended for
/// probe tools such as `ffprobe` whose answer lives on stdout.
pub async fn run_capture(
    backend: &str,
    program: &str,
    args: &[String],
    limits: &Limits,
) -> Result<Vec<u8>> {
    let mut command = tokio::process::Command::new(program);
    command
        .args(args)
        .stdin(Stdio::null())
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .kill_on_drop(true);
    sandbox(&mut command, limits);
    let mut child = command.spawn().map_err(io_start_error(program))?;
    run_child(&mut child, backend, program, limits).await
}

fn io_start_error(program: &str) -> impl Fn(std::io::Error) -> Error + '_ {
    move |error| {
        Error::Io(std::io::Error::new(
            error.kind(),
            format!("cannot start {program}: {error}"),
        ))
    }
}

/// Drain both pipes and reap the child within `limits.timeout`, returning stdout.
///
/// Dropping the child on any early return kills it because `kill_on_drop` is
/// set; the timeout path additionally kills the whole process group on Unix.
async fn run_child(
    child: &mut tokio::process::Child,
    backend: &str,
    program: &str,
    limits: &Limits,
) -> Result<Vec<u8>> {
    let stdout_stream = child
        .stdout
        .take()
        .ok_or_else(|| Error::Conversion(format!("{program} stdout was not captured")))?;
    let stderr_stream = child
        .stderr
        .take()
        .ok_or_else(|| Error::Conversion(format!("{program} stderr was not captured")))?;

    let job = async {
        let (stdout, stderr, status) = tokio::join!(
            read_bounded(stdout_stream, limits.max_output_bytes),
            read_bounded(stderr_stream, STDERR_CAP),
            child.wait(),
        );
        Ok::<_, Error>((stdout?, stderr?, status.map_err(Error::Io)?))
    };
    let (stdout, stderr, status) = match tokio::time::timeout(limits.timeout, job).await {
        Ok(result) => result?,
        Err(_elapsed) => {
            let pid = child.id();
            let _ = child.start_kill();
            #[cfg(unix)]
            unsafe_scope::kill_group(pid);
            let _ = child.wait().await;
            return Err(Error::Timeout {
                backend: backend.to_owned(),
            });
        }
    };

    let code = status.code();
    if !status.success() {
        return Err(Error::Conversion(format!(
            "{program} ({backend}) exited with status {code:?}: {}",
            tail(&stderr)
        )));
    }
    Ok(stdout)
}

/// Whether `program` exists on `PATH` and executes quickly enough to probe.
///
/// Presence is inferred from a successful spawn, not the exit code: tools print
/// usage and exit nonzero when called with an unknown flag. Any spawn error
/// (missing, not executable) reports false. A probe that hangs is killed.
pub async fn available(program: &str) -> bool {
    spawn_flag(program, "--version").await || spawn_flag(program, "-version").await
}

async fn spawn_flag(program: &str, flag: &str) -> bool {
    let mut command = tokio::process::Command::new(program);
    command
        .arg(flag)
        .stdin(Stdio::null())
        .stdout(Stdio::null())
        .stderr(Stdio::null())
        .kill_on_drop(true);
    let Ok(mut child) = command.spawn() else {
        return false;
    };
    // Whether it exits cleanly or is killed for stalling, a successful spawn
    // only happens when the binary is really present and runnable.
    let _ = tokio::time::timeout(PROBE_TIMEOUT, child.wait()).await;
    true
}

async fn read_bounded<S>(stream: S, cap: u64) -> Result<Vec<u8>>
where
    S: AsyncRead + Unpin,
{
    let mut collected = Vec::new();
    let mut buffer = [0u8; CHUNK];
    let mut reader = stream;
    loop {
        let count = reader
            .read(&mut buffer)
            .await
            .map_err(|error| Error::Conversion(format!("failed reading tool output: {error}")))?;
        if count == 0 {
            break;
        }
        if (collected.len() + count) as u64 > cap {
            let room = cap.saturating_sub(collected.len() as u64);
            let room = usize::try_from(room).unwrap_or(usize::MAX).min(count);
            collected.extend_from_slice(&buffer[..room]);
            return Err(Error::Limit("external tool output exceeded its cap".into()));
        }
        collected.extend_from_slice(&buffer[..count]);
    }
    Ok(collected)
}

fn tail(bytes: &[u8]) -> String {
    let start = bytes.len().saturating_sub(ERROR_TAIL);
    String::from_utf8_lossy(&bytes[start..]).trim().to_owned()
}

/// Apply per-child environment scrubbing and, on Unix, POSIX resource limits
/// and a new session so timeouts can kill the whole process group.
fn sandbox(command: &mut tokio::process::Command, limits: &Limits) {
    command.env_clear();
    for key in ["PATH", "HOME", "TMPDIR"] {
        if let Some(value) = std::env::var_os(key) {
            command.env(key, value);
        }
    }
    #[cfg(unix)]
    unsafe_scope::pre_exec(command, limits);
}

#[cfg(unix)]
mod unsafe_scope {
    //! Narrow unsafe surface for POSIX sandboxing of native tool children.
    //!
    //! Every `unsafe` block lives here, behind one module-level `allow`, and is
    //! documented with its contract. Nothing else in the crate uses `unsafe`.

    #![allow(unsafe_code)]

    use crate::Limits;

    /// CPU seconds a tool may burn before the kernel kills it.
    const CPU_LIMIT_SLACK_SECS: u64 = 5;
    /// Soft cap on child address space, aligned with [`Limits::max_memory_bytes`]
    /// but floored below: native tools reserve far more virtual memory than they
    /// touch (ffmpeg and OpenBLAS map gigabytes of untouched thread stacks), so a
    /// cap that tracks the adapter budget too closely kills working processes.
    const AS_LIMIT_SLACK: u64 = 256 * 1024 * 1024;
    const AS_LIMIT_FLOOR: u64 = 2 * 1024 * 1024 * 1024;
    /// File size a tool may grow on disk before SIGXFSZ.
    const FILE_LIMIT_SLACK: u64 = 512 * 1024 * 1024;

    /// Install `pre_exec` hooks: `setsid` plus `RLIMIT_*` values from `limits`.
    ///
    /// SAFETY: `pre_exec` code must be async-signal-safe. `setsid`, `getrlimit`,
    /// and `setrlimit` are listed as async-signal-safe by POSIX. The closures
    /// capture only plain integers, borrow no `Path` or allocator state, and
    /// report failures as `io::Error` so the spawn surfaces them. The cost is
    /// per-child and single-threaded at fork time (tokio forks from a dedicated
    /// thread precisely for this hook).
    pub fn pre_exec(command: &mut tokio::process::Command, limits: &Limits) {
        let cpu_secs = limits.timeout.as_secs() + CPU_LIMIT_SLACK_SECS;
        let address_space = limits
            .max_memory_bytes
            .saturating_add(AS_LIMIT_SLACK)
            .max(AS_LIMIT_FLOOR);
        let file_size = limits.max_output_bytes.saturating_add(FILE_LIMIT_SLACK);
        let cpu = u64_to_rlim(cpu_secs);
        let aspace = u64_to_rlim(address_space);
        let fsize = u64_to_rlim(file_size);
        unsafe {
            command.pre_exec(move || {
                if libc::setsid() == -1 {
                    return Err(std::io::Error::last_os_error());
                }
                set_limit(libc::RLIMIT_CPU as libc::c_int, cpu)?;
                set_limit(libc::RLIMIT_AS as libc::c_int, aspace)?;
                set_limit(libc::RLIMIT_FSIZE as libc::c_int, fsize)?;
                Ok(())
            });
        }
    }

    fn set_limit(resource: libc::c_int, value: libc::rlim_t) -> std::io::Result<()> {
        let mut current = libc::rlimit {
            rlim_cur: 0,
            rlim_max: 0,
        };
        // SAFETY: `current` is a valid, initialized rlimit; getrlimit writes at
        // most `size_of::<rlimit>()` bytes into it on success.
        let rc = unsafe { libc::getrlimit(resource as _, &mut current) };
        if rc == -1 {
            return Err(std::io::Error::last_os_error());
        }
        // Tighten, never loosen: keep any hard cap the OS already imposes.
        let rlim_max = if current.rlim_max == libc::RLIM_INFINITY {
            libc::RLIM_INFINITY
        } else {
            current.rlim_max
        };
        let desired = libc::rlimit {
            rlim_cur: if current.rlim_max == libc::RLIM_INFINITY {
                value
            } else {
                value.min(current.rlim_max)
            },
            rlim_max,
        };
        // SAFETY: `desired` is fully initialized and `resource` is a valid
        // RLIMIT constant; failures are reported through the return value.
        let rc = unsafe { libc::setrlimit(resource as _, &desired) };
        if rc == -1 {
            return Err(std::io::Error::last_os_error());
        }
        Ok(())
    }

    fn u64_to_rlim(value: u64) -> libc::rlim_t {
        libc::rlim_t::try_from(value).unwrap_or(libc::RLIM_INFINITY)
    }

    /// SIGKILL the whole process group of `pid`, catching tool thread/fork trees.
    ///
    /// Best effort: if the child already exited or `setsid` failed, the group
    /// kill is a harmless no-op or targets the child alone.
    pub fn kill_group(pid: Option<u32>) {
        let Some(pid) = pid else { return };
        let Ok(pid) = i32::try_from(pid) else { return };
        // SAFETY: kill with a positive pid and SIGKILL cannot dereference
        // memory; an ESRCH/EPERM result simply does nothing.
        unsafe {
            libc::killpg(pid, libc::SIGKILL);
        }
    }
}

#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
    use super::*;

    fn limits() -> Limits {
        Limits::default()
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn run_captures_success() {
        let output = run("test-probe", "true", &[], None, &limits()).await;
        assert!(output.is_ok());
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn run_maps_nonzero_exit() {
        let error = run("test-probe", "false", &[], None, &limits())
            .await
            .unwrap_err();
        assert!(matches!(error, Error::Conversion(_)), "{error:?}");
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn run_enforces_timeout() {
        let tight = Limits {
            timeout: Duration::from_millis(50),
            ..Limits::default()
        };
        let error = run("test-probe", "sleep", &["5".into()], None, &tight)
            .await
            .unwrap_err();
        assert!(matches!(error, Error::Timeout { .. }), "{error:?}");
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn run_bounds_collected_output() {
        let cap = Limits {
            max_output_bytes: 16,
            ..Limits::default()
        };
        let error = run(
            "test-probe",
            "head",
            &["-c".into(), "100000".into(), "/dev/zero".into()],
            None,
            &cap,
        )
        .await
        .unwrap_err();
        assert!(matches!(error, Error::Limit(_)), "{error:?}");
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn run_reports_missing_program_as_io() {
        let error = run(
            "test-probe",
            "definitely-not-installed-convert-rs-xyz",
            &[],
            None,
            &limits(),
        )
        .await
        .unwrap_err();
        assert!(matches!(error, Error::Io(_)), "{error:?}");
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn available_detects_present_and_absent() {
        assert!(available("true").await);
        assert!(!available("definitely-not-installed-convert-rs-xyz").await);
    }
}