aion-worker 0.27.1

//! Failures both declared-command executors answer with, stated once.
//!
//! The activity executor ([`super::action`]) and the declared-body executor
//! ([`super::declared`]) fail over the same two things — a process that could
//! not be run to completion, and a process whose ending this host cannot read —
//! and a guarantee that is written twice is a guarantee free to drift. Both
//! live here, and the only difference between the two executors' renderings is
//! WHERE the failure happened, carried as an optional [`BodySite`].

use super::exit::Ending;
use crate::activity::ActivityFailure;
use crate::process::ProcessGroupError;

/// Where in a declared command's BODY something happened.
///
/// A string-body action has one process and needs none of this; a declared
/// body has many, and an operator reading a failure needs to know how much of
/// the body ran, because the part that ran has already changed whatever it
/// changes.
#[derive(Debug, Clone, Copy)]
pub(super) struct BodySite<'a> {
    /// The command's declared name.
    pub(super) command: &'a str,
    /// The failing line's ONE-BASED position in the body.
    pub(super) line: usize,
    /// How many lines the body states.
    pub(super) total: usize,
}

impl BodySite<'_> {
    /// The subject of a sentence about this site: which command, which line of
    /// how many, and the program that line names.
    fn subject(self, program: &str) -> String {
        let Self {
            command,
            line,
            total,
        } = self;
        format!("command `{command}`'s line {line} of {total} (`{program}`)")
    }

    /// The subject of a sentence about `program` at `site`, or about the
    /// program alone when there is no body to place it in.
    fn subject_of(site: Option<Self>, program: &str) -> String {
        site.map_or_else(
            || format!("the declared command `{program}`"),
            |site| site.subject(program),
        )
    }
}

/// Classify a spawn or observation failure — the ONE classifier, read by the
/// worker executors and by the CLI's shell worker, so one OS condition cannot
/// receive opposite verdicts from two surfaces.
///
/// It used to be terminal in every arm here, on the reasoning that a program
/// absent or unrunnable now will be exactly as absent next time. That is true
/// of the conditions it was written for and false of the ones it also caught:
/// [`ProcessGroupError::Spawn`] carries the kernel's refusal to fork, so a
/// process table under pressure (`EAGAIN`), an exhausted file-descriptor table
/// (`EMFILE`/`ENFILE`) or a momentary `ENOMEM` **permanently failed a durable
/// workflow step over a condition that clears by itself**. `os error 24` is not
/// hypothetical here: it voided a battery in this repository's own gate logs.
/// Meanwhile the CLI answered the same `Spawn` variant with *retryable*, so the
/// verdict depended on which surface you reached.
///
/// The split is by **errno**, and the direction is structural rather than a
/// list of conditions someone remembered:
///
/// **A spawn failure means nothing executed.** `execve` did not happen, so no
/// body line ran and no effect was produced — which is why retrying a spawn
/// failure can never double-run a body. The only spawn errnos worth failing
/// permanently are the ones saying **the program itself is wrong**: `ENOENT`,
/// `EACCES`, `EPERM`, `ENOEXEC`, `EISDIR`. Those are the same on the next
/// attempt. **Everything else is the world being transient** — `EAGAIN` (the
/// process table), `EMFILE`/`ENFILE` (descriptors), `ENOMEM`, `ETXTBSY`,
/// `EINTR`, `EBUSY`, `EIO`, `ENOSPC` — and is **retryable**, including an errno
/// no one here has enumerated. The default falls that way *because* the safe
/// direction is knowable: a wrongly-retried absent program fails again at once
/// and visibly, bounded by the retry policy, while a wrongly-terminal transient
/// destroys a workflow that would have succeeded.
///
/// `spawn_permits_retry_by_errno` is written as the TERMINAL set with a
/// retryable default for exactly that reason. An earlier version wrote it the
/// other way, as an enumerated retryable set with a terminal default, and the
/// doc above it claimed the default was retryable — so the stated safety
/// argument and the implemented behaviour disagreed, and `EINTR` permanently
/// failed durable steps. The tests below pin the default in both directions so
/// it cannot silently invert again.
///
/// **A `Read` or `Reap` failure is TERMINAL, and this is not the same
/// question.** By then the child HAS been spawned. It may still be running.
/// This host failed to observe it, which is not the same as it not having
/// happened, and a retry re-enters the body loop at its FIRST line
/// (`super::declared`), so retrying would re-run lines that already ran — for
/// `Read`, against a process that may still be alive. That is concurrent
/// double execution, not repetition. Retrying a command whose termination could
/// NOT be confirmed risks running it twice concurrently, which is why these
/// arms are not shared with `Spawn` however similar the errno looks; it is the
/// same reasoning [`ending_permits_retry`] applies from the other end of the
/// process's life.
pub(super) fn spawn_failure(
    program: &str,
    site: Option<BodySite<'_>>,
    error: &ProcessGroupError,
) -> ActivityFailure {
    let sentence = format!(
        "{subject} could not be run to completion: {error}",
        subject = BodySite::subject_of(site, program)
    );
    if spawn_failure_permits_retry(error) {
        ActivityFailure::retryable(sentence)
    } else {
        ActivityFailure::terminal(sentence)
    }
}

/// Whether `error` describes a condition a second attempt can clear.
///
/// Public so the CLI's shell worker classifies through this function rather
/// than by matching the variant itself, which is how the two surfaces came to
/// disagree.
#[must_use]
pub fn spawn_failure_permits_retry(error: &ProcessGroupError) -> bool {
    match error {
        ProcessGroupError::Spawn { source } => spawn_permits_retry_by_errno(source),
        // Read and Reap are NOT shared with Spawn: by then a child exists, and
        // for Read it may still be running. See this module's doc.
        _ => false,
    }
}

/// Whether a SPAWN error leaves retrying safe and useful.
///
/// Written as the TERMINAL set with a retryable default, because nothing ran:
/// the errnos that say the program itself is wrong are knowable and finite,
/// while the transient world is not. An errno absent from this list retries.
///
/// The numbers are the ones IDENTICAL on both Unixes this crate builds for
/// (macOS and Linux): `EPERM` 1, `ENOENT` 2, `ENOEXEC` 8, `EACCES` 13,
/// `EISDIR` 21. Where the standard library already maintains a portable
/// mapping it is used instead of a number — and `EAGAIN` is precisely why that
/// matters: it is 35 on macOS and 11 on Linux, so a hand-written table would
/// have been wrong on the platform nobody tested. It needs no entry here in any
/// case, since it is not the program being wrong.
fn spawn_permits_retry_by_errno(error: &std::io::Error) -> bool {
    const EPERM: i32 = 1;
    const ENOENT: i32 = 2;
    const ENOEXEC: i32 = 8;
    const EACCES: i32 = 13;
    const EISDIR: i32 = 21;

    if matches!(
        error.kind(),
        std::io::ErrorKind::NotFound | std::io::ErrorKind::PermissionDenied
    ) {
        return false;
    }
    !matches!(
        error.raw_os_error(),
        Some(EPERM | ENOENT | ENOEXEC | EACCES | EISDIR)
    )
}

/// Whether a failure over `ending` may be retried — the ONE classification of
/// an ending that named neither an exit code nor a signal, read by both
/// executors.
///
/// An exit or a signal is the command's own verdict, and a failing verdict is
/// the retry family's territory: a busy resource, an unreachable host, a
/// transient permission state are all endings a second attempt clears.
///
/// An ending that stated NEITHER is not a verdict at all — it is this host
/// failing to read one, over a process that has already been waited on and
/// reaped. Nothing on this side can say whether that process did its work, and
/// re-running work that may already have happened is the one thing a durable
/// engine must not decide on its own: it is the same uncertainty
/// [`spawn_failure`] answers terminally, arriving from the other end of the
/// process's life. So an unreadable ending is TERMINAL at both executors, the
/// failure says so in words, and an operator — who can look at what the
/// command touched — decides whether it runs again.
pub(super) const fn ending_permits_retry(ending: Ending) -> bool {
    ending.reported_code().is_some()
}

/// The clause a failure adds when the ending could not be read at all.
///
/// Empty for an ending that named an exit code or a signal: there is nothing
/// to warn about, and a sentence that warns anyway teaches an operator to skip
/// the warning.
pub(super) fn unreadable_ending_clause(ending: Ending) -> &'static str {
    if ending_permits_retry(ending) {
        ""
    } else {
        ", so this host cannot say whether it did its work and will not run it again on its own"
    }
}

#[cfg(test)]
mod tests {
    use super::{
        BodySite, ending_permits_retry, spawn_failure, spawn_failure_permits_retry,
        unreadable_ending_clause,
    };
    use crate::activity::Classification;
    use crate::process::ProcessGroupError;
    use crate::shell::exit::Ending;

    fn observation_error() -> ProcessGroupError {
        ProcessGroupError::Spawn {
            source: std::io::Error::from(std::io::ErrorKind::NotFound),
        }
    }

    /// A string-body action's spawn failure names the program and nothing it
    /// does not have: there is no body to place it in.
    #[test]
    fn a_spawn_failure_without_a_body_names_the_program() {
        let failure = spawn_failure("nope", None, &observation_error());
        assert_eq!(failure.classification(), &Classification::Terminal);
        assert!(
            failure
                .message()
                .starts_with("the declared command `nope` could not be run to completion:"),
            "{}",
            failure.message()
        );
    }

    /// A declared body's spawn failure names the command, the line, and the
    /// program — an operator reading it knows how much of the body ran.
    #[test]
    fn a_spawn_failure_inside_a_body_names_the_command_and_which_line() {
        let failure = spawn_failure(
            "nope",
            Some(BodySite {
                command: "release",
                line: 2,
                total: 5,
            }),
            &observation_error(),
        );
        assert_eq!(failure.classification(), &Classification::Terminal);
        assert!(
            failure.message().starts_with(
                "command `release`'s line 2 of 5 (`nope`) could not be run to completion:"
            ),
            "{}",
            failure.message()
        );
    }

    /// ONE classification, both executors: a read ending may be retried, an
    /// unreadable one may not, and the unreadable one says why.
    #[test]
    fn only_an_ending_this_host_can_read_permits_a_retry() {
        assert!(ending_permits_retry(Ending::Exited(3)));
        assert!(ending_permits_retry(Ending::Signalled(9)));
        assert!(!ending_permits_retry(Ending::Unstated));

        assert_eq!(unreadable_ending_clause(Ending::Exited(3)), "");
        assert_eq!(unreadable_ending_clause(Ending::Signalled(9)), "");
        assert!(
            unreadable_ending_clause(Ending::Unstated)
                .contains("cannot say whether it did its work")
        );
    }

    /// `EMFILE` is the errno that voided a battery in this repository's own
    /// gate logs. Under the old rule it permanently failed the workflow step.
    #[test]
    fn an_exhausted_descriptor_table_is_retryable() {
        let error = spawn(24);
        assert!(spawn_failure_permits_retry(&error));
        assert!(
            matches!(
                spawn_failure("prog", None, &error).classification(),
                Classification::Retryable
            ),
            "resource pressure must not permanently fail a durable step"
        );
    }

    /// The terminal set: the errnos that say the PROGRAM is wrong. These are
    /// the same on the next attempt, so retrying only burns the budget.
    #[test]
    fn an_absent_or_unrunnable_program_stays_terminal() {
        for errno in [1, 2, 8, 13, 21] {
            let error = spawn(errno);
            assert!(
                !spawn_failure_permits_retry(&error),
                "errno {errno} names the program itself as wrong and must stay terminal"
            );
            assert!(!matches!(
                spawn_failure("prog", None, &error).classification(),
                Classification::Retryable
            ));
        }
    }

    /// THE DEFAULT, pinned in the direction it actually falls.
    ///
    /// A spawn failure means `execve` never happened, so nothing ran and a
    /// retry cannot double-run a body. Every errno outside the terminal set is
    /// therefore retryable — INCLUDING ones nobody enumerated. An earlier
    /// version documented this default and implemented its opposite, so
    /// `EINTR` and `EBUSY` permanently failed durable steps; no test covered an
    /// unclassified errno, which is exactly why that survived. This is that
    /// test.
    #[test]
    fn an_unenumerated_spawn_errno_retries() {
        // EINTR 4, EIO 5, EBUSY 16, ENOTDIR 20, ENOSPC 28 — all identical on
        // macOS and Linux, none of them the program being wrong.
        for errno in [4, 5, 16, 20, 28] {
            let error = spawn(errno);
            assert!(
                spawn_failure_permits_retry(&error),
                "errno {errno} is the world being transient, not the program being wrong"
            );
        }
    }

    /// `EAGAIN` is 35 on macOS and 11 on Linux, so it is never hand-written.
    /// It retries by falling through the terminal set, on whichever platform
    /// this suite runs.
    #[test]
    fn a_fork_refused_under_pressure_is_retryable_on_this_platform() {
        let eagain = if cfg!(target_os = "macos") { 35 } else { 11 };
        let error = spawn(eagain);
        assert_eq!(error_kind_of(&error), std::io::ErrorKind::WouldBlock);
        assert!(spawn_failure_permits_retry(&error));
    }

    /// A child that HAS been spawned may still be running. This host failing to
    /// read its output is not the command not having happened, and a retry
    /// re-enters the body at line 1 — so retrying would run it twice, for
    /// `Read` possibly concurrently. Terminal regardless of how transient the
    /// errno looks: the SAME errno that retries on `Spawn` must not retry here.
    #[test]
    fn a_spawned_child_this_host_could_not_observe_stays_terminal() {
        for error in [
            ProcessGroupError::Read {
                stream: "stdout",
                source: std::io::Error::from_raw_os_error(24),
            },
            ProcessGroupError::Reap {
                source: std::io::Error::from_raw_os_error(24),
            },
        ] {
            assert!(
                !spawn_failure_permits_retry(&error),
                "a spawned process must not be re-run on this host's own initiative"
            );
        }
        // The discriminating pair: errno 24 retries on Spawn and does not here.
        assert!(spawn_failure_permits_retry(&spawn(24)));
    }

    fn spawn(errno: i32) -> ProcessGroupError {
        ProcessGroupError::Spawn {
            source: std::io::Error::from_raw_os_error(errno),
        }
    }

    fn error_kind_of(error: &ProcessGroupError) -> std::io::ErrorKind {
        match error {
            ProcessGroupError::Spawn { source } | ProcessGroupError::Read { source, .. } => {
                source.kind()
            }
            _ => std::io::ErrorKind::Other,
        }
    }
}