alien-bindings 3.3.24

//! Sandbox binding providers.
//!
//! Per-cloud backends land with their controllers. Local is here because it speaks the same
//! authenticated transport the cloud backends do, so it exercises the real path rather than a
//! shortcut.

#[cfg(any(feature = "aws", feature = "kubernetes"))]
pub mod agent_protocol;

#[cfg(feature = "aws")]
pub mod aws;

#[cfg(feature = "azure")]
pub mod azure;

#[cfg(feature = "gcp")]
pub mod gcp_agent_platform;

#[cfg(feature = "kubernetes")]
pub mod kubernetes;

#[cfg(feature = "local")]
pub mod local;

#[cfg(feature = "aws")]
mod refusal;

#[cfg(all(
    test,
    feature = "aws",
    feature = "azure",
    feature = "gcp",
    feature = "kubernetes",
    feature = "local"
))]
#[path = "jobs_capability_tests.rs"]
mod jobs_capability_tests;

#[cfg(all(
    test,
    feature = "aws",
    feature = "azure",
    feature = "gcp",
    feature = "kubernetes",
    feature = "local"
))]
#[path = "lifetime_capability_tests.rs"]
mod lifetime_capability_tests;

/// The lifetime a backend is asked for, from the milliseconds a caller asked for.
///
/// Every backend's lifetime primitive is whole seconds, so this rounds up: a sandbox reaped
/// before the time the caller asked for is a wrong answer, and up to a second of extra life is
/// not. A declared ceiling bounds the result, so a request can only ever shorten a sandbox.
///
/// Both ends are refused rather than adjusted, as [`guard_for`] refuses both ends of a command
/// timeout. A zero raised to a second buys a sandbox the backend reaps while `create` is still
/// waiting for it to serve, and that wait fails as unreachable — a retryable answer to a request
/// that can never succeed. Past what whole seconds can hold there is no lifetime to send, and a
/// ceiling does not stand in for one: the ordinary deployment declares none.
#[cfg(any(feature = "aws", feature = "gcp"))]
pub(crate) fn requested_lifetime_seconds(
    timeout_ms: u64,
    declared_ceiling: Option<u32>,
    operation: &str,
) -> crate::error::Result<u32> {
    let refuse = |details: String| {
        alien_error::AlienError::new(crate::error::ErrorData::InvalidInput {
            operation_context: operation.to_string(),
            details,
            field_name: Some("timeoutMs".to_string()),
        })
    };

    if timeout_ms == 0 {
        return Err(refuse(
            "a sandbox lifetime must be at least one millisecond".to_string(),
        ));
    }

    let asked = u32::try_from(timeout_ms.div_ceil(1_000)).map_err(|_| {
        refuse(format!(
            "a sandbox lifetime must be at most {} seconds",
            u32::MAX
        ))
    })?;

    Ok(match declared_ceiling {
        Some(ceiling) => asked.min(ceiling),
        None => asked,
    })
}

/// The longest command timeout these backends accept.
///
/// A ceiling rather than a guard: a timer takes a point in time, and a duration near
/// `Duration::MAX` has no representable one, so accepting it would panic the caller's task.
/// Nothing legitimate reaches this — a sandbox outlives its commands and no cloud lets one run
/// a day — so the far end is refused as the invalid request it is, and the arithmetic below
/// cannot overflow.
#[cfg(any(feature = "azure", feature = "local"))]
pub(crate) const MAX_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60);

/// How long past a command's timeout a provider without a supervising agent waits for the
/// in-sandbox `timeout` to report back before it ends the sandbox itself. Generous against the
/// transport's own latency and short against a caller's patience.
#[cfg(any(feature = "azure", feature = "local"))]
pub(crate) const TIMEOUT_GRACE: std::time::Duration = std::time::Duration::from_secs(10);

/// Bounds one command inside a sandbox, and reports its own kill.
///
/// Neither the exit status nor the clock can say whether a command was killed at its timeout:
/// `timeout -s KILL` exits 137 on GNU and BusyBox alike, `-k` exits 124 on one and 143 on the
/// other, BusyBox reports 0 for a command that traps `TERM`, and what a provider measures is its
/// own round trip rather than how long the command ran. So the wrapper does the killing itself
/// and says so — measured on both shells rather than read off a manual page.
///
/// The report is a nonce the **sandbox** draws, announced on the first line of stderr and
/// repeated only if the killer's signal landed. Drawn there rather than passed in because the
/// command can read its parent's `/proc/<pid>/cmdline` and `environ`: a nonce that travelled in
/// either could be echoed back, and untrusted code would be able to claim its own timeout. A
/// shell variable is in neither, and the command cannot read what has already been written to
/// the stream it inherits. `unset` first, because an inherited *exported* variable of the same
/// name keeps its export attribute across re-assignment and would carry the nonce straight back
/// into the command's own environment.
///
/// Nothing but `sh` and `/dev/urandom` is required, which every sandbox image has.
#[cfg(any(feature = "azure", feature = "local"))]
pub(crate) struct TimeoutReport;

/// Hex digits in the nonce the wrapper draws: `od -N16` reads 16 bytes.
///
/// Checked exactly, so a single stray hex character on a line of its own cannot be read as an
/// announcement and turn the rest of the stream into its own repeat.
#[cfg(any(feature = "azure", feature = "local"))]
const NONCE_HEXITS: usize = 32;

#[cfg(any(feature = "azure", feature = "local"))]
impl TimeoutReport {
    /// The shell program that runs a command under this timeout.
    ///
    /// The command arrives as `"$@"`, so nothing re-parses its text. It is started in a sandbox
    /// of its own so the kill reaches its process group rather than one pid: a command that
    /// spawned children would otherwise leave them running while the caller is told the timeout
    /// contained it. A child that starts a sandbox of its own leaves that group and outlives the
    /// kill — measured — so this covers what the command left behind, not what it moved away. An
    /// image that cannot start a sandbox runs nothing: a timeout that cannot be enforced at all
    /// is refused rather than approximated.
    ///
    /// The killer repeats the nonce when its signal was delivered, which the status has to confirm:
    /// a command already exited and awaiting reaping takes the signal too. Once the command is
    /// reaped the killer is told to stop and then
    /// awaited: asleep, it reaps its own sleeper and leaves at once, so a command that ended early —
    /// or was killed by something other than the timeout — is not held for the rest of the timeout;
    /// past its sleep it ignores the stop, so its report lands before it leaves rather than being
    /// cut off mid-write. It stays a subshell so the nonce reaches it as a variable and never through
    /// argv, which the command could read.
    pub(crate) fn bounded_program(timeout: std::time::Duration) -> String {
        format!(
            "unset nonce command_pid killer_pid sleeper status; \
             command -v setsid >/dev/null 2>&1 || exit {unboundable}; \
             nonce=$(od -An -N16 -tx1 /dev/urandom | tr -d ' \\n') || exit {unboundable}; \
             printf '%s\\n' \"$nonce\" >&2; \
             setsid \"$@\" & command_pid=$!; \
             ( sleep {timeout} & sleeper=$!; trap 'kill \"$sleeper\" 2>/dev/null; exit' TERM; \
               wait \"$sleeper\"; \
               trap '' TERM; kill -KILL -\"$command_pid\" 2>/dev/null && printf %s \"$nonce\" >&2 ) & killer_pid=$!; \
             wait \"$command_pid\"; status=$?; \
             kill \"$killer_pid\" 2>/dev/null; wait \"$killer_pid\"; \
             exit \"$status\"",
            unboundable = Self::UNBOUNDABLE_EXIT_CODE,
            timeout = timeout_seconds(timeout)
        )
    }

    /// What the wrapper exits with when the sandbox cannot bound a command at all.
    ///
    /// Distinct from anything the command could return, because the command never ran: the
    /// wrapper exits before starting it.
    const UNBOUNDABLE_EXIT_CODE: i32 = 126;

    /// What a shell reports for a command `SIGKILL` ended, which is how the timeout ends one.
    const KILLED_EXIT_CODE: i32 = 137;

    /// What became of a command the wrapper was asked to bound.
    ///
    /// The announcement is removed either way; a repeat of it after that is the killer's signal,
    /// because only the sandbox knows the value. Whether that signal ended the command is the
    /// status's to say.
    pub(crate) fn read(exit_code: Option<i32>, stderr: &str) -> Bounded {
        // The first line that is a nonce and nothing else, rather than the first line: a shell
        // asked to trace itself writes its own lines before this one, and they displace an
        // announcement that has to be found for the report to mean anything. Unforgeable either
        // way — the sandbox writes it before the command starts, and a traced line carries the
        // shell's prefix, so nothing the command chose can be read as the announcement.
        let announced = stderr.split('\n').enumerate().find_map(|(index, line)| {
            // A carriage return would make the announcement 33 bytes and invisible, and the first
            // line the command chose would be adopted in its place. No transport here delivers
            // one today; the cost of not depending on that is one trim.
            let line = line.strip_suffix('\r').unwrap_or(line);
            let is_nonce =
                line.len() == NONCE_HEXITS && line.chars().all(|c| c.is_ascii_hexdigit());
            is_nonce.then(|| {
                let after = stderr
                    .split('\n')
                    .skip(index + 1)
                    .collect::<Vec<_>>()
                    .join("\n");
                (line, after)
            })
        });

        let Some((nonce, rest)) = announced.as_ref().map(|(n, r)| (*n, r.as_str())) else {
            // No announcement means the wrapper exited before starting anything, so nothing of
            // the caller's ran and nothing about a timeout can be claimed.
            return Bounded::NotRun {
                reason: if exit_code == Some(Self::UNBOUNDABLE_EXIT_CODE) {
                    "the sandbox image cannot hold a command to a timeout: `setsid` and \
                     `/dev/urandom` are required"
                        .to_string()
                } else {
                    format!(
                        "the sandbox could not start a bounded command: {}",
                        stderr.trim()
                    )
                },
            };
        };

        match rest.find(nonce) {
            Some(at) => Bounded::Ran {
                // The repeat says the killer fired, not that it ended anything: `kill` succeeds on
                // a command that has exited and is not yet reaped. Only the status separates the
                // two, so a command that finished at its timeout keeps its own result.
                killed: exit_code == Some(Self::KILLED_EXIT_CODE),
                stderr: format!("{}{}", &rest[..at], &rest[at + nonce.len()..]),
            },
            None => Bounded::Ran {
                killed: false,
                stderr: rest.to_string(),
            },
        }
    }
}

/// What became of a command the wrapper was asked to bound.
#[cfg(any(feature = "azure", feature = "local"))]
pub(crate) enum Bounded {
    /// The wrapper never started it, so there is nothing to report but why.
    NotRun { reason: String },
    /// It ran; `killed` is the sandbox's own report of ending it at the timeout.
    Ran { killed: bool, stderr: String },
}

/// The timeout as `sleep` takes it, to the millisecond — both shells accept a fraction, so a
/// sub-second timeout is not rounded up to the next second.
#[cfg(any(feature = "azure", feature = "local"))]
fn timeout_seconds(timeout: std::time::Duration) -> String {
    let millis = timeout.as_millis().max(1);
    if millis % 1000 == 0 {
        (millis / 1000).to_string()
    } else {
        format!("{}.{:03}", millis / 1000, millis % 1000)
    }
}

/// Refuses a timeout these backends cannot honour, and returns the guard the caller waits on.
///
/// Both ends are refused rather than quietly adjusted, as the agent-supervised backends refuse a
/// timeout that floors to zero milliseconds. Below a millisecond the in-sandbox bound cannot
/// express it. At the other end the guard is a point in time, not a length: a duration near
/// `Duration::MAX` has no representable instant, and asking a timer for one panics the caller's
/// task instead of failing its request.
#[cfg(any(feature = "azure", feature = "local"))]
pub(crate) fn guard_for(timeout: std::time::Duration) -> crate::error::Result<std::time::Duration> {
    let refuse = |reason: &str| {
        alien_error::AlienError::new(crate::error::ErrorData::SandboxCommandFailed {
            failure: "invalidRequest".to_string(),
            reason: reason.to_string(),
        })
    };

    if timeout < std::time::Duration::from_millis(1) {
        return Err(refuse("a command timeout must be at least one millisecond"));
    }

    if timeout > MAX_TIMEOUT {
        return Err(refuse(&format!(
            "a command timeout must be at most {} hours",
            MAX_TIMEOUT.as_secs() / 3600
        )));
    }

    Ok(timeout + TIMEOUT_GRACE)
}

#[cfg(all(test, any(feature = "aws", feature = "gcp")))]
mod lifetime_tests {
    use super::requested_lifetime_seconds;

    /// Pins both decisions `requested_lifetime_seconds` makes, because getting either wrong is
    /// silent: rounding down would reap a sandbox before the time asked for, and letting a
    /// request exceed the declared ceiling would let a caller outlive the deployment's own limit.
    #[test]
    fn a_requested_lifetime_rounds_up_and_never_raises_the_declared_ceiling() {
        let seconds = |timeout_ms, ceiling| {
            requested_lifetime_seconds(timeout_ms, ceiling, "sandbox.create")
                .expect("a lifetime the backends can serve")
        };

        assert_eq!(seconds(60_000, None), 60);
        assert_eq!(seconds(1_500, None), 2);
        assert_eq!(seconds(1, None), 1);
        assert_eq!(seconds(600_000, Some(1_800)), 600);
        assert_eq!(seconds(7_200_000, Some(1_800)), 1_800);
    }

    /// A lifetime no backend can serve is refused as the invalid request it is, rather than
    /// raised to a second or saturated at the widest one seconds can hold. Either adjustment
    /// builds a sandbox nobody asked for; raising a zero builds one the backend reaps inside
    /// `create`'s own readiness wait, which reports a permanently invalid request as retryable.
    #[test]
    fn a_lifetime_no_backend_can_serve_is_refused_rather_than_adjusted() {
        for (what, timeout_ms, ceiling) in [
            ("zero", 0_u64, None),
            ("zero under a ceiling", 0, Some(1_800)),
            ("wider than seconds hold", u64::MAX, None),
            (
                "wider than seconds hold, under a ceiling",
                u64::MAX,
                Some(1_800),
            ),
        ] {
            let error = requested_lifetime_seconds(timeout_ms, ceiling, "sandbox.create")
                .expect_err("a lifetime no backend can serve");
            assert!(
                matches!(
                    &error.error,
                    Some(crate::error::ErrorData::InvalidInput { field_name, .. })
                        if field_name.as_deref() == Some("timeoutMs")
                ),
                "{what} has to name the field the caller sent, which is what a client turns into \
                 a field-level message: {error:?}"
            );
            assert_eq!(
                error.code, "INVALID_INPUT",
                "{what} has to be refused as the caller's, not reported as a backend failure it \
                 could retry: {error}"
            );
            assert!(
                error.to_string().contains("sandbox lifetime"),
                "{what} has to say which field is wrong: {error}"
            );
        }
    }
}

#[cfg(all(test, any(feature = "azure", feature = "local")))]
mod tests {
    use super::*;

    #[test]
    fn the_timeout_reaches_the_shell_to_the_millisecond() {
        assert_eq!(timeout_seconds(std::time::Duration::from_secs(30)), "30");
        assert_eq!(
            timeout_seconds(std::time::Duration::from_millis(1500)),
            "1.500"
        );
        assert_eq!(
            timeout_seconds(std::time::Duration::from_millis(500)),
            "0.500"
        );
    }

    /// A command cannot claim the timeout: the nonce is drawn inside the sandbox, and the
    /// command can read neither the parent's command line nor its environment for it.
    #[test]
    fn only_the_sandbox_can_report_a_timeout() {
        // The shell writes its own notice after the signal, so the repeat is not always last.
        let killed = match TimeoutReport::read(
            Some(137),
            "a1b2c3d4a1b2c3d4a1b2c3d4a1b2c3d4\nboom\na1b2c3d4a1b2c3d4a1b2c3d4a1b2c3d4Killed\n",
        ) {
            Bounded::Ran { killed, stderr } => {
                assert_eq!(stderr, "boom\nKilled\n");
                killed
            }
            Bounded::NotRun { reason } => panic!("the command ran: {reason}"),
        };
        assert!(killed);

        // A command echoing something nonce-shaped repeats nothing the sandbox announced.
        assert!(matches!(
            TimeoutReport::read(
                Some(0),
                "a1b2c3d4a1b2c3d4a1b2c3d4a1b2c3d4\nboom\ndeadbeef\n"
            ),
            Bounded::Ran { killed: false, .. }
        ));
    }

    /// An image that cannot hold a command to its timeout runs nothing, and says so: reporting
    /// an ordinary exit there would be a timeout the resource never enforced.
    #[test]
    fn a_sandbox_that_cannot_bound_a_command_runs_nothing() {
        let Bounded::NotRun { reason } = TimeoutReport::read(Some(126), "") else {
            panic!("an unboundable sandbox must not look like a command that ran");
        };
        assert!(reason.contains("setsid"), "{reason}");

        // Any other silence is the sandbox failing to start the wrapper at all.
        let Bounded::NotRun { reason } = TimeoutReport::read(Some(127), "sh: not found\n") else {
            panic!("stderr with no announcement is not a command that ran");
        };
        assert!(reason.contains("could not start"), "{reason}");
    }

    /// The announcement survives a shell that writes before it, and a short hex line is not one.
    ///
    /// A `sh` that is really bash turns on tracing from `SHELLOPTS` in its environment and writes
    /// its own lines first. Reading only line 1 lost the announcement there and reported every
    /// command — including the ones that succeeded — as never bounded. The width is checked
    /// exactly, so a stray hex fragment on a line of its own cannot stand in for it.
    #[test]
    fn the_announcement_is_found_by_shape_rather_than_by_position() {
        let traced = format!("+ unset nonce command_pid\n+ printf\na1b2c3d4a1b2c3d4a1b2c3d4a1b2c3d4\nboom\na1b2c3d4a1b2c3d4a1b2c3d4a1b2c3d4Killed\n");
        let Bounded::Ran { killed, stderr } = TimeoutReport::read(Some(137), &traced) else {
            panic!("the trace must not hide the announcement");
        };
        assert!(killed, "the killer's repeat still reports the kill");
        assert_eq!(
            stderr, "boom\nKilled\n",
            "what precedes the announcement was written before the command started, so it is the \
             sandbox's own noise rather than the command's — and one of those lines is the trace \
             of the announcement itself"
        );

        // A carriage return does not hide the announcement, which would otherwise let the first
        // line the command chose stand in for it.
        let crlf = format!("a1b2c3d4a1b2c3d4a1b2c3d4a1b2c3d4\r\nboom\r\na1b2c3d4a1b2c3d4a1b2c3d4a1b2c3d4Killed\r\n");
        assert!(
            matches!(
                TimeoutReport::read(Some(137), &crlf),
                Bounded::Ran { killed: true, .. }
            ),
            "a carriage return is not part of the nonce"
        );

        // Short of the width the sandbox draws, so not an announcement — and the rest of the
        // stream is not its repeat.
        assert!(
            matches!(
                TimeoutReport::read(Some(137), "ab\nboom\nabc\n"),
                Bounded::NotRun { .. }
            ),
            "a hex fragment is not a nonce"
        );
    }

    /// A command that finished as the killer fired keeps its own result. `kill` succeeds on a
    /// process that has exited and is not yet reaped, so the repeat alone would turn a command
    /// that beat its timeout into a timeout failure and throw away what it returned.
    #[test]
    fn a_command_that_finished_as_the_killer_fired_keeps_its_result() {
        let Bounded::Ran { killed, stderr } = TimeoutReport::read(
            Some(0),
            "a1b2c3d4a1b2c3d4a1b2c3d4a1b2c3d4\nboom\na1b2c3d4a1b2c3d4a1b2c3d4a1b2c3d4",
        ) else {
            panic!("the command ran");
        };
        assert!(
            !killed,
            "a command that exited 0 was not ended by the timeout, whatever the signal reached"
        );
        assert_eq!(stderr, "boom\n", "the announcement is removed either way");
    }

    /// The command reaches the shell as arguments, the nonce is drawn in the sandbox, the kill
    /// reaches the whole process group, and the killer only claims a kill it made.
    #[test]
    fn the_bounded_program_kills_and_reports_only_what_it_killed() {
        let program = TimeoutReport::bounded_program(std::time::Duration::from_millis(1500));
        assert!(program.contains("/dev/urandom"), "{program}");
        assert!(program.contains("command -v setsid"), "{program}");
        assert!(
            program.contains("setsid \"$@\" & command_pid=$!"),
            "{program}"
        );
        assert!(program.contains("sleep 1.500"), "{program}");
        assert!(
            program.contains("kill -KILL -\"$command_pid\""),
            "the kill reaches the command's process group, not one pid: {program}"
        );
        assert!(
            program.contains("2>/dev/null && printf %s \"$nonce\""),
            "the repeat follows a signal that was delivered: {program}"
        );
        assert!(
            program.contains("kill \"$killer_pid\" 2>/dev/null; wait \"$killer_pid\""),
            "the killer is stopped and then awaited, whatever the command's exit: {program}"
        );
        assert!(
            program.contains(r#"wait "$sleeper"; trap '' TERM; kill -KILL"#),
            "past its sleep the killer ignores the stop, so its report is never cut off, and the \
             pid is quoted so an inherited IFS cannot split it into words that are not children: \
             {program}"
        );
        assert!(
            program.contains(r#"trap 'kill "$sleeper" 2>/dev/null; exit' TERM"#),
            "a stopped killer reaps its own sleeper, so none outlives the command: {program}"
        );
        assert!(
            !program.contains("\"$nonce\" &") && program.contains("( sleep"),
            "the killer is a subshell: the nonce reaches it as a variable, never as an argument \
             the command could read from /proc: {program}"
        );
    }

    /// An inherited variable of the wrapper's own name never reaches the command.
    ///
    /// Run against a real `sh`: an exported variable keeps its export attribute across
    /// re-assignment, so `nonce=$(…)` would hand the command the sandbox's own nonce. A
    /// stand-in `setsid` is supplied because macOS ships none.
    #[test]
    #[cfg(unix)]
    fn the_wrapper_never_hands_its_nonce_to_the_command() {
        use std::os::unix::fs::PermissionsExt;

        let bin = std::env::temp_dir().join(format!("alien-sandbox-{}", std::process::id()));
        std::fs::create_dir_all(&bin).expect("a directory for the stand-in");
        let setsid = bin.join("setsid");
        std::fs::write(&setsid, "#!/bin/sh\nexec \"$@\"\n").expect("the stand-in is written");
        std::fs::set_permissions(&setsid, std::fs::Permissions::from_mode(0o755))
            .expect("the stand-in is executable");

        let path = format!(
            "{}:{}",
            bin.display(),
            std::env::var("PATH").unwrap_or_default()
        );
        let run = std::process::Command::new("/bin/sh")
            .arg("-c")
            .arg(TimeoutReport::bounded_program(
                std::time::Duration::from_secs(5),
            ))
            .arg("sh")
            .arg("printenv")
            .arg("nonce")
            .env("PATH", path)
            .env("nonce", "inherited-from-the-sandbox")
            .output()
            .expect("a shell runs");
        std::fs::remove_dir_all(&bin).ok();

        let announced = String::from_utf8_lossy(&run.stderr);
        let announced = announced.lines().next().unwrap_or_default().to_string();
        assert!(
            announced.len() == 32 && announced.chars().all(|c| c.is_ascii_hexdigit()),
            "the sandbox has to reach the point of drawing a nonce, or this proves nothing: \
             stderr {:?}",
            String::from_utf8_lossy(&run.stderr)
        );

        let seen = String::from_utf8_lossy(&run.stdout);
        assert!(
            seen.trim().is_empty(),
            "the command must inherit no `nonce` at all, and it saw {seen:?}"
        );
    }

    /// A timeout neither end can honour is refused, not stretched or waited on.
    #[tokio::test]
    async fn a_timeout_outside_what_the_backends_can_honour_is_refused() {
        let too_fine = guard_for(std::time::Duration::from_micros(500))
            .expect_err("half a millisecond cannot be bounded");
        assert!(
            too_fine.to_string().contains("invalidRequest"),
            "{too_fine}"
        );

        let too_long = guard_for(std::time::Duration::MAX)
            .expect_err("a timeout with no representable instant cannot be waited out");
        assert!(
            too_long.to_string().contains("invalidRequest"),
            "{too_long}"
        );

        assert_eq!(
            guard_for(std::time::Duration::from_secs(30)).expect("an ordinary timeout"),
            std::time::Duration::from_secs(30) + TIMEOUT_GRACE,
            "the guard is the timeout plus the grace"
        );
    }
}