start-command 0.25.4

//! Exit reason hints for finished executions.
//!
//! A numeric exit code alone is frequently misleading: a Bun/Node process that
//! exhausts its heap aborts itself, so the container reports `exitCode 139`
//! with `OOMKilled false` even though the run died of memory exhaustion (issue
//! #162, related to #144, #148 and #151). `start` already reads the tail of the
//! log to find the terminal footer, so the same tail is scanned for well-known
//! fatal markers and the finding is surfaced as an extra `exitReason` field.
//!
//! The same scan answers a narrower question for consumers that key off
//! `oomKilled`: was this run killed by memory exhaustion at all? That answer is
//! surfaced as `memoryExhausted` plus `memoryExhaustedReason`, the log line that
//! carried the evidence (issue #165).
//!
//! Both fields are hints, never verdicts: they never change `status`,
//! `exitCode` or `oomKilled`.

use regex::Regex;

/// Fatal markers, most specific first. The first matching entry wins.
const EXIT_REASON_MARKERS: [(&str, &str); 5] = [
    (
        "memory-exhaustion (v8-heap-limit)",
        r"(?i)FATAL ERROR:[^\r\n]*(?:Reached heap limit|JavaScript heap out of memory)",
    ),
    (
        "memory-exhaustion (v8-heap-limit)",
        r"(?i)JavaScript heap out of memory",
    ),
    (
        "memory-exhaustion (kernel-oom-killer)",
        r"(?i)Out of memory: Kill(?:ed)? process|oom-kill(?:er)?[: ]|Killed process \d+",
    ),
    (
        "memory-exhaustion (go-runtime)",
        r"(?i)fatal error: runtime: out of memory",
    ),
    (
        "memory-exhaustion (allocation-failure)",
        r"(?i)std::bad_alloc|Cannot allocate memory|memory allocation of \d+ bytes failed|Allocation failed - process out of memory|Array buffer allocation failed",
    ),
];

/// Exit codes above 128 encode the signal that killed the process.
/// Only the signals that actually show up in command logs are named.
///
/// Public because the detached completion watcher generates a POSIX `case`
/// from this very table, so the signal the watcher writes into the log and the
/// one `--status` reports can never disagree (issue #171.4).
pub const SIGNAL_NAMES: [(i32, &str); 10] = [
    (1, "SIGHUP"),
    (2, "SIGINT"),
    (3, "SIGQUIT"),
    (4, "SIGILL"),
    (6, "SIGABRT"),
    (8, "SIGFPE"),
    (9, "SIGKILL"),
    (11, "SIGSEGV"),
    (13, "SIGPIPE"),
    (15, "SIGTERM"),
];

/// Reasons carrying this prefix report memory exhaustion, whatever the
/// mechanism (runtime self-abort, kernel OOM killer, failed allocation).
const MEMORY_EXHAUSTION_PREFIX: &str = "memory-exhaustion";

/// A matched marker line longer than this is truncated: the evidence travels in
/// `--status` output, and some runtimes print very long single-line traces.
const MAX_MARKER_LINE_LENGTH: usize = 300;

/// A fatal marker found in the log: its category and the line that carried it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ExitReasonMarker {
    pub reason: String,
    pub line: String,
}

/// The memory-exhaustion observation derived from a log tail (issue #165).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MemoryExhaustion {
    pub memory_exhausted: bool,
    pub memory_exhausted_reason: String,
}

/// Extract the whole (trimmed, length-bounded) line containing byte `index`.
fn extract_line_at(text: &str, index: usize) -> String {
    let start = text[..index].rfind('\n').map(|i| i + 1).unwrap_or(0);
    let end = text[index..]
        .find('\n')
        .map(|i| index + i)
        .unwrap_or(text.len());
    let line = text[start..end].trim();
    if line.chars().count() <= MAX_MARKER_LINE_LENGTH {
        return line.to_string();
    }
    let truncated: String = line.chars().take(MAX_MARKER_LINE_LENGTH).collect();
    format!("{}...", truncated)
}

/// Scan log text for the first accepted fatal marker, most specific first.
pub fn find_exit_reason_marker(
    text: Option<&str>,
    accept: impl Fn(&str) -> bool,
) -> Option<ExitReasonMarker> {
    let text = text?;
    if text.is_empty() {
        return None;
    }
    for (reason, pattern) in EXIT_REASON_MARKERS {
        if !accept(reason) {
            continue;
        }
        if let Ok(re) = Regex::new(pattern) {
            if let Some(found) = re.find(text) {
                return Some(ExitReasonMarker {
                    reason: reason.to_string(),
                    line: extract_line_at(text, found.start()),
                });
            }
        }
    }
    None
}

/// Scan log text for a known fatal marker.
pub fn detect_exit_reason(text: Option<&str>) -> Option<String> {
    find_exit_reason_marker(text, |_| true).map(|marker| marker.reason)
}

/// Scan log text for a marker that reports memory exhaustion specifically.
pub fn detect_memory_marker(text: Option<&str>) -> Option<ExitReasonMarker> {
    find_exit_reason_marker(text, |reason| reason.starts_with(MEMORY_EXHAUSTION_PREFIX))
}

/// Decide whether an execution died of memory exhaustion, and say why.
///
/// Only abnormal exits are explained: a command that merely printed a fatal
/// marker (an `rg` dump, a quoted incident log) and then succeeded is not a
/// memory failure. An observation, never a verdict (issue #151).
///
/// The container-wide OOM signals (`State.OOMKilled`, the cgroup `oom_kill`
/// counter) only count when the exit fits an OOM kill of the command itself:
/// a child OOM-killed under a command that later exits 1 is not the reason the
/// command ended (issue #180).
pub fn resolve_memory_exhaustion(
    exit_code: Option<i32>,
    log_tail: Option<&str>,
    oom_killed: Option<bool>,
    oom_kills: Option<u64>,
) -> Option<MemoryExhaustion> {
    match exit_code {
        Some(code) if code != 0 => {}
        _ => return None,
    }
    let (main_oom, daemon_restart) = crate::exit_evidence::from_log(log_tail);
    if daemon_restart {
        return None;
    }
    if let Some(marker) = detect_memory_marker(log_tail)
        .filter(|m| !m.reason.contains("kernel-oom-killer") || main_oom)
    {
        return Some(MemoryExhaustion {
            memory_exhausted: true,
            memory_exhausted_reason: marker.line,
        });
    }
    let main_oom = main_oom && exit_code.is_some_and(|code| code == 137 || code < 0);
    if oom_killed == Some(true) && main_oom {
        return Some(MemoryExhaustion {
            memory_exhausted: true,
            memory_exhausted_reason: "Docker reported State.OOMKilled=true".to_string(),
        });
    }
    let kills = oom_kills.unwrap_or(0);
    if kills > 0 && main_oom {
        return Some(MemoryExhaustion {
            memory_exhausted: true,
            memory_exhausted_reason: format!("cgroup memory.events reported oom_kill={}", kills),
        });
    }
    if main_oom {
        return Some(MemoryExhaustion {
            memory_exhausted: true,
            memory_exhausted_reason: "Exit-time evidence reported a main-process OOM kill".into(),
        });
    }
    None
}

/// Reason reported when the container-wide OOM observation explains the exit.
pub const CGROUP_OOM_EXIT_REASON: &str = "memory-exhaustion (cgroup-oom-killer)";

/// Can a container-wide OOM observation be blamed on the command itself?
///
/// Docker's `State.OOMKilled` and the cgroup `oom_kill` counter are
/// container-wide and sticky (moby/moby#43564): they turn on as soon as *any*
/// process in the container is OOM-killed, e.g. one `rustc` child under
/// `cargo test`, while the command keeps running and later exits on its own
/// (issue #180). The OOM killer always sends SIGKILL, so the observation only
/// explains the command's exit when that exit is 137 (`128 + SIGKILL`) or there
/// is no usable exit code at all (none, or the `-1` of an unknown exit). This
/// is the rule `is_killed_exit` already applies to `--on-kill-resume` (#178).
pub fn is_oom_kill_of_command(
    exit_code: Option<i32>,
    oom_killed: Option<bool>,
    oom_kills: Option<u64>,
) -> bool {
    // Without attempt-scoped evidence this legacy signature cannot attribute OOM.
    let _ = (exit_code, oom_killed, oom_kills);
    false
}

/// Text used wherever a fact could not be observed at all.
pub const UNKNOWN_EXIT_CODE: &str = "unknown";

/// A decoded exit code: the numeric value, the signal it encodes (if any), and
/// the human-readable rendering both the container post-mortem and `--status`
/// print.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ExitCodeDescription {
    pub code: Option<i32>,
    pub signal: Option<String>,
    pub text: String,
}

/// Decode an exit code once, for every caller (issue #171.4).
///
/// `128 + n` was previously decoded in the status formatter and described in
/// prose by the watcher, which is how the same `137` could be reported as
/// `SIGKILL` in one place and as a bare number in the other.
pub fn describe_exit_code(exit_code: Option<i32>) -> ExitCodeDescription {
    let code = match exit_code {
        Some(code) => code,
        None => {
            return ExitCodeDescription {
                code: None,
                signal: None,
                text: UNKNOWN_EXIT_CODE.to_string(),
            }
        }
    };
    let signal = signal_name_for_exit_code(Some(code));
    let text = match &signal {
        Some(name) => format!("{} ({} - 128+{})", code, name, code - 128),
        None => code.to_string(),
    };
    ExitCodeDescription {
        code: Some(code),
        signal,
        text,
    }
}

/// Decode an exit code that arrived as text (e.g. from the watcher's shell).
pub fn describe_exit_code_str(value: &str) -> ExitCodeDescription {
    describe_exit_code(value.trim().parse::<i32>().ok())
}

/// Map a shell exit code to the signal name it encodes.
pub fn signal_name_for_exit_code(exit_code: Option<i32>) -> Option<String> {
    let code = exit_code?;
    if code <= 128 || code > 128 + 64 {
        return None;
    }
    SIGNAL_NAMES
        .iter()
        .find(|(signal, _)| *signal == code - 128)
        .map(|(_, name)| name.to_string())
}

/// Resolve the best available hint for why an execution ended.
///
/// Precedence: the log marker (evidence written by the command itself), then
/// the cgroup OOM observation when the exit fits an OOM kill of the command
/// (issue #180), then the signal encoded in the exit code.
pub fn resolve_exit_reason(
    exit_code: Option<i32>,
    log_tail: Option<&str>,
    oom_killed: Option<bool>,
    oom_kills: Option<u64>,
) -> Option<String> {
    let (main_oom, daemon_restart) = crate::exit_evidence::from_log(log_tail);
    if daemon_restart {
        return Some("killed (docker daemon restart)".into());
    }
    if let Some(reason) =
        detect_exit_reason(log_tail).filter(|r| !r.contains("kernel-oom-killer") || main_oom)
    {
        return Some(reason);
    }

    let _ = (oom_killed, oom_kills);
    if main_oom && exit_code.is_none_or(|code| code < 0 || code == 137) {
        return Some(CGROUP_OOM_EXIT_REASON.to_string());
    }

    signal_name_for_exit_code(exit_code).map(|name| {
        format!(
            "signal ({}{})",
            name,
            if name == "SIGKILL" {
                "; cause unknown"
            } else {
                ""
            }
        )
    })
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn detects_v8_heap_limit_from_the_incident_log() {
        let tail =
            "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory";
        assert_eq!(
            detect_exit_reason(Some(tail)),
            Some("memory-exhaustion (v8-heap-limit)".to_string())
        );
    }

    #[test]
    fn detects_kernel_oom_killer() {
        assert_eq!(
            detect_exit_reason(Some("Out of memory: Killed process 1234 (bun)")),
            Some("memory-exhaustion (kernel-oom-killer)".to_string())
        );
    }

    #[test]
    fn detects_other_runtimes_reporting_their_own_exhaustion() {
        assert_eq!(
            detect_exit_reason(Some("fatal error: runtime: out of memory")),
            Some("memory-exhaustion (go-runtime)".to_string())
        );
        assert_eq!(
            detect_exit_reason(Some("Array buffer allocation failed")),
            Some("memory-exhaustion (allocation-failure)".to_string())
        );
    }

    #[test]
    fn returns_none_for_ordinary_output() {
        assert_eq!(detect_exit_reason(Some("all tests passed")), None);
        assert_eq!(detect_exit_reason(None), None);
    }

    #[test]
    fn maps_signal_exit_codes() {
        assert_eq!(signal_name_for_exit_code(Some(137)), Some("SIGKILL".into()));
        assert_eq!(signal_name_for_exit_code(Some(139)), Some("SIGSEGV".into()));
        assert_eq!(signal_name_for_exit_code(Some(0)), None);
        assert_eq!(signal_name_for_exit_code(Some(1)), None);
        assert_eq!(signal_name_for_exit_code(None), None);
    }

    #[test]
    fn log_marker_wins_over_signal_and_cgroup_flag() {
        let tail =
            "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory";
        assert_eq!(
            resolve_exit_reason(Some(139), Some(tail), Some(false), None),
            Some("memory-exhaustion (v8-heap-limit)".to_string())
        );
    }

    #[test]
    fn memory_marker_returns_the_whole_matched_line() {
        let tail = "building...\nFATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory\n----- Native stack trace -----\n";
        let marker = detect_memory_marker(Some(tail)).expect("marker");
        assert_eq!(marker.reason, "memory-exhaustion (v8-heap-limit)");
        assert_eq!(
            marker.line,
            "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory"
        );
    }

    #[test]
    fn memory_marker_line_is_length_bounded() {
        let tail = format!(
            "{}JavaScript heap out of memory{}",
            "x".repeat(400),
            "y".repeat(400)
        );
        let marker = detect_memory_marker(Some(&tail)).expect("marker");
        assert!(marker.line.chars().count() <= MAX_MARKER_LINE_LENGTH + 3);
        assert!(marker.line.ends_with("..."));
    }

    #[test]
    fn memory_exhaustion_explains_only_abnormal_exits() {
        let tail =
            "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory";
        let observed =
            resolve_memory_exhaustion(Some(139), Some(tail), Some(false), None).expect("observed");
        assert!(observed.memory_exhausted);
        assert_eq!(observed.memory_exhausted_reason, tail);
        // The same marker with a successful exit is just output, not a failure.
        assert_eq!(
            resolve_memory_exhaustion(Some(0), Some(tail), Some(false), None),
            None
        );
        assert_eq!(
            resolve_memory_exhaustion(None, Some(tail), None, None),
            None
        );
    }

    #[test]
    fn memory_exhaustion_falls_back_to_the_container_flag() {
        let observed = resolve_memory_exhaustion(
            Some(137),
            Some(crate::exit_evidence::MAIN_OOM),
            Some(true),
            None,
        )
        .expect("observed");
        assert_eq!(
            observed.memory_exhausted_reason,
            "Docker reported State.OOMKilled=true"
        );
        assert_eq!(
            resolve_memory_exhaustion(Some(1), Some("no marker"), Some(false), None),
            None
        );
    }

    #[test]
    fn falls_back_to_cgroup_observation_then_signal() {
        assert_eq!(
            resolve_exit_reason(
                Some(137),
                Some(crate::exit_evidence::MAIN_OOM),
                Some(true),
                None
            ),
            Some("memory-exhaustion (cgroup-oom-killer)".to_string())
        );
        assert_eq!(
            resolve_exit_reason(Some(139), Some("no marker here"), Some(false), None),
            Some("signal (SIGSEGV)".to_string())
        );
        assert_eq!(resolve_exit_reason(Some(0), Some("fine"), None, None), None);
    }
}