start_command/
exit_reason.rs

1//! Exit reason hints for finished executions.
2//!
3//! A numeric exit code alone is frequently misleading: a Bun/Node process that
4//! exhausts its heap aborts itself, so the container reports `exitCode 139`
5//! with `OOMKilled false` even though the run died of memory exhaustion (issue
6//! #162, related to #144, #148 and #151). `start` already reads the tail of the
7//! log to find the terminal footer, so the same tail is scanned for well-known
8//! fatal markers and the finding is surfaced as an extra `exitReason` field.
9//!
10//! The same scan answers a narrower question for consumers that key off
11//! `oomKilled`: was this run killed by memory exhaustion at all? That answer is
12//! surfaced as `memoryExhausted` plus `memoryExhaustedReason`, the log line that
13//! carried the evidence (issue #165).
14//!
15//! Both fields are hints, never verdicts: they never change `status`,
16//! `exitCode` or `oomKilled`.
17
18use regex::Regex;
19
20/// Fatal markers, most specific first. The first matching entry wins.
21const EXIT_REASON_MARKERS: [(&str, &str); 5] = [
22    (
23        "memory-exhaustion (v8-heap-limit)",
24        r"(?i)FATAL ERROR:[^\r\n]*(?:Reached heap limit|JavaScript heap out of memory)",
25    ),
26    (
27        "memory-exhaustion (v8-heap-limit)",
28        r"(?i)JavaScript heap out of memory",
29    ),
30    (
31        "memory-exhaustion (kernel-oom-killer)",
32        r"(?i)Out of memory: Kill(?:ed)? process|oom-kill(?:er)?[: ]|Killed process \d+",
33    ),
34    (
35        "memory-exhaustion (go-runtime)",
36        r"(?i)fatal error: runtime: out of memory",
37    ),
38    (
39        "memory-exhaustion (allocation-failure)",
40        r"(?i)std::bad_alloc|Cannot allocate memory|memory allocation of \d+ bytes failed|Allocation failed - process out of memory|Array buffer allocation failed",
41    ),
42];
43
44/// Exit codes above 128 encode the signal that killed the process.
45/// Only the signals that actually show up in command logs are named.
46///
47/// Public because the detached completion watcher generates a POSIX `case`
48/// from this very table, so the signal the watcher writes into the log and the
49/// one `--status` reports can never disagree (issue #171.4).
50pub const SIGNAL_NAMES: [(i32, &str); 10] = [
51    (1, "SIGHUP"),
52    (2, "SIGINT"),
53    (3, "SIGQUIT"),
54    (4, "SIGILL"),
55    (6, "SIGABRT"),
56    (8, "SIGFPE"),
57    (9, "SIGKILL"),
58    (11, "SIGSEGV"),
59    (13, "SIGPIPE"),
60    (15, "SIGTERM"),
61];
62
63/// Reasons carrying this prefix report memory exhaustion, whatever the
64/// mechanism (runtime self-abort, kernel OOM killer, failed allocation).
65const MEMORY_EXHAUSTION_PREFIX: &str = "memory-exhaustion";
66
67/// A matched marker line longer than this is truncated: the evidence travels in
68/// `--status` output, and some runtimes print very long single-line traces.
69const MAX_MARKER_LINE_LENGTH: usize = 300;
70
71/// A fatal marker found in the log: its category and the line that carried it.
72#[derive(Debug, Clone, PartialEq, Eq)]
73pub struct ExitReasonMarker {
74    pub reason: String,
75    pub line: String,
76}
77
78/// The memory-exhaustion observation derived from a log tail (issue #165).
79#[derive(Debug, Clone, PartialEq, Eq)]
80pub struct MemoryExhaustion {
81    pub memory_exhausted: bool,
82    pub memory_exhausted_reason: String,
83}
84
85/// Extract the whole (trimmed, length-bounded) line containing byte `index`.
86fn extract_line_at(text: &str, index: usize) -> String {
87    let start = text[..index].rfind('\n').map(|i| i + 1).unwrap_or(0);
88    let end = text[index..]
89        .find('\n')
90        .map(|i| index + i)
91        .unwrap_or(text.len());
92    let line = text[start..end].trim();
93    if line.chars().count() <= MAX_MARKER_LINE_LENGTH {
94        return line.to_string();
95    }
96    let truncated: String = line.chars().take(MAX_MARKER_LINE_LENGTH).collect();
97    format!("{}...", truncated)
98}
99
100/// Scan log text for the first accepted fatal marker, most specific first.
101pub fn find_exit_reason_marker(
102    text: Option<&str>,
103    accept: impl Fn(&str) -> bool,
104) -> Option<ExitReasonMarker> {
105    let text = text?;
106    if text.is_empty() {
107        return None;
108    }
109    for (reason, pattern) in EXIT_REASON_MARKERS {
110        if !accept(reason) {
111            continue;
112        }
113        if let Ok(re) = Regex::new(pattern) {
114            if let Some(found) = re.find(text) {
115                return Some(ExitReasonMarker {
116                    reason: reason.to_string(),
117                    line: extract_line_at(text, found.start()),
118                });
119            }
120        }
121    }
122    None
123}
124
125/// Scan log text for a known fatal marker.
126pub fn detect_exit_reason(text: Option<&str>) -> Option<String> {
127    find_exit_reason_marker(text, |_| true).map(|marker| marker.reason)
128}
129
130/// Scan log text for a marker that reports memory exhaustion specifically.
131pub fn detect_memory_marker(text: Option<&str>) -> Option<ExitReasonMarker> {
132    find_exit_reason_marker(text, |reason| reason.starts_with(MEMORY_EXHAUSTION_PREFIX))
133}
134
135/// Decide whether an execution died of memory exhaustion, and say why.
136///
137/// Only abnormal exits are explained: a command that merely printed a fatal
138/// marker (an `rg` dump, a quoted incident log) and then succeeded is not a
139/// memory failure. An observation, never a verdict (issue #151).
140///
141/// The container-wide OOM signals (`State.OOMKilled`, the cgroup `oom_kill`
142/// counter) only count when the exit fits an OOM kill of the command itself:
143/// a child OOM-killed under a command that later exits 1 is not the reason the
144/// command ended (issue #180).
145pub fn resolve_memory_exhaustion(
146    exit_code: Option<i32>,
147    log_tail: Option<&str>,
148    oom_killed: Option<bool>,
149    oom_kills: Option<u64>,
150) -> Option<MemoryExhaustion> {
151    match exit_code {
152        Some(code) if code != 0 => {}
153        _ => return None,
154    }
155    if let Some(marker) = detect_memory_marker(log_tail) {
156        return Some(MemoryExhaustion {
157            memory_exhausted: true,
158            memory_exhausted_reason: marker.line,
159        });
160    }
161    if oom_killed == Some(true) && is_oom_kill_of_command(exit_code, oom_killed, None) {
162        return Some(MemoryExhaustion {
163            memory_exhausted: true,
164            memory_exhausted_reason: "Docker reported State.OOMKilled=true".to_string(),
165        });
166    }
167    let kills = oom_kills.unwrap_or(0);
168    if kills > 0 && is_oom_kill_of_command(exit_code, None, oom_kills) {
169        return Some(MemoryExhaustion {
170            memory_exhausted: true,
171            memory_exhausted_reason: format!("cgroup memory.events reported oom_kill={}", kills),
172        });
173    }
174    None
175}
176
177/// Reason reported when the container-wide OOM observation explains the exit.
178pub const CGROUP_OOM_EXIT_REASON: &str = "memory-exhaustion (cgroup-oom-killer)";
179
180/// Can a container-wide OOM observation be blamed on the command itself?
181///
182/// Docker's `State.OOMKilled` and the cgroup `oom_kill` counter are
183/// container-wide and sticky (moby/moby#43564): they turn on as soon as *any*
184/// process in the container is OOM-killed, e.g. one `rustc` child under
185/// `cargo test`, while the command keeps running and later exits on its own
186/// (issue #180). The OOM killer always sends SIGKILL, so the observation only
187/// explains the command's exit when that exit is 137 (`128 + SIGKILL`) or there
188/// is no usable exit code at all (none, or the `-1` of an unknown exit). This
189/// is the rule `is_killed_exit` already applies to `--on-kill-resume` (#178).
190pub fn is_oom_kill_of_command(
191    exit_code: Option<i32>,
192    oom_killed: Option<bool>,
193    oom_kills: Option<u64>,
194) -> bool {
195    let observed = oom_killed == Some(true) || oom_kills.unwrap_or(0) > 0;
196    if !observed {
197        return false;
198    }
199    match exit_code {
200        None => true,
201        Some(code) => {
202            code < 0 || signal_name_for_exit_code(Some(code)).as_deref() == Some("SIGKILL")
203        }
204    }
205}
206
207/// Text used wherever a fact could not be observed at all.
208pub const UNKNOWN_EXIT_CODE: &str = "unknown";
209
210/// A decoded exit code: the numeric value, the signal it encodes (if any), and
211/// the human-readable rendering both the container post-mortem and `--status`
212/// print.
213#[derive(Debug, Clone, PartialEq, Eq)]
214pub struct ExitCodeDescription {
215    pub code: Option<i32>,
216    pub signal: Option<String>,
217    pub text: String,
218}
219
220/// Decode an exit code once, for every caller (issue #171.4).
221///
222/// `128 + n` was previously decoded in the status formatter and described in
223/// prose by the watcher, which is how the same `137` could be reported as
224/// `SIGKILL` in one place and as a bare number in the other.
225pub fn describe_exit_code(exit_code: Option<i32>) -> ExitCodeDescription {
226    let code = match exit_code {
227        Some(code) => code,
228        None => {
229            return ExitCodeDescription {
230                code: None,
231                signal: None,
232                text: UNKNOWN_EXIT_CODE.to_string(),
233            }
234        }
235    };
236    let signal = signal_name_for_exit_code(Some(code));
237    let text = match &signal {
238        Some(name) => format!("{} ({} - 128+{})", code, name, code - 128),
239        None => code.to_string(),
240    };
241    ExitCodeDescription {
242        code: Some(code),
243        signal,
244        text,
245    }
246}
247
248/// Decode an exit code that arrived as text (e.g. from the watcher's shell).
249pub fn describe_exit_code_str(value: &str) -> ExitCodeDescription {
250    describe_exit_code(value.trim().parse::<i32>().ok())
251}
252
253/// Map a shell exit code to the signal name it encodes.
254pub fn signal_name_for_exit_code(exit_code: Option<i32>) -> Option<String> {
255    let code = exit_code?;
256    if code <= 128 || code > 128 + 64 {
257        return None;
258    }
259    SIGNAL_NAMES
260        .iter()
261        .find(|(signal, _)| *signal == code - 128)
262        .map(|(_, name)| name.to_string())
263}
264
265/// Resolve the best available hint for why an execution ended.
266///
267/// Precedence: the log marker (evidence written by the command itself), then
268/// the cgroup OOM observation when the exit fits an OOM kill of the command
269/// (issue #180), then the signal encoded in the exit code.
270pub fn resolve_exit_reason(
271    exit_code: Option<i32>,
272    log_tail: Option<&str>,
273    oom_killed: Option<bool>,
274    oom_kills: Option<u64>,
275) -> Option<String> {
276    if let Some(reason) = detect_exit_reason(log_tail) {
277        return Some(reason);
278    }
279
280    if is_oom_kill_of_command(exit_code, oom_killed, oom_kills) {
281        return Some(CGROUP_OOM_EXIT_REASON.to_string());
282    }
283
284    signal_name_for_exit_code(exit_code).map(|name| format!("signal ({})", name))
285}
286
287#[cfg(test)]
288mod tests {
289    use super::*;
290
291    #[test]
292    fn detects_v8_heap_limit_from_the_incident_log() {
293        let tail =
294            "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory";
295        assert_eq!(
296            detect_exit_reason(Some(tail)),
297            Some("memory-exhaustion (v8-heap-limit)".to_string())
298        );
299    }
300
301    #[test]
302    fn detects_kernel_oom_killer() {
303        assert_eq!(
304            detect_exit_reason(Some("Out of memory: Killed process 1234 (bun)")),
305            Some("memory-exhaustion (kernel-oom-killer)".to_string())
306        );
307    }
308
309    #[test]
310    fn detects_other_runtimes_reporting_their_own_exhaustion() {
311        assert_eq!(
312            detect_exit_reason(Some("fatal error: runtime: out of memory")),
313            Some("memory-exhaustion (go-runtime)".to_string())
314        );
315        assert_eq!(
316            detect_exit_reason(Some("Array buffer allocation failed")),
317            Some("memory-exhaustion (allocation-failure)".to_string())
318        );
319    }
320
321    #[test]
322    fn returns_none_for_ordinary_output() {
323        assert_eq!(detect_exit_reason(Some("all tests passed")), None);
324        assert_eq!(detect_exit_reason(None), None);
325    }
326
327    #[test]
328    fn maps_signal_exit_codes() {
329        assert_eq!(signal_name_for_exit_code(Some(137)), Some("SIGKILL".into()));
330        assert_eq!(signal_name_for_exit_code(Some(139)), Some("SIGSEGV".into()));
331        assert_eq!(signal_name_for_exit_code(Some(0)), None);
332        assert_eq!(signal_name_for_exit_code(Some(1)), None);
333        assert_eq!(signal_name_for_exit_code(None), None);
334    }
335
336    #[test]
337    fn log_marker_wins_over_signal_and_cgroup_flag() {
338        let tail =
339            "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory";
340        assert_eq!(
341            resolve_exit_reason(Some(139), Some(tail), Some(false), None),
342            Some("memory-exhaustion (v8-heap-limit)".to_string())
343        );
344    }
345
346    #[test]
347    fn memory_marker_returns_the_whole_matched_line() {
348        let tail = "building...\nFATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory\n----- Native stack trace -----\n";
349        let marker = detect_memory_marker(Some(tail)).expect("marker");
350        assert_eq!(marker.reason, "memory-exhaustion (v8-heap-limit)");
351        assert_eq!(
352            marker.line,
353            "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory"
354        );
355    }
356
357    #[test]
358    fn memory_marker_line_is_length_bounded() {
359        let tail = format!(
360            "{}JavaScript heap out of memory{}",
361            "x".repeat(400),
362            "y".repeat(400)
363        );
364        let marker = detect_memory_marker(Some(&tail)).expect("marker");
365        assert!(marker.line.chars().count() <= MAX_MARKER_LINE_LENGTH + 3);
366        assert!(marker.line.ends_with("..."));
367    }
368
369    #[test]
370    fn memory_exhaustion_explains_only_abnormal_exits() {
371        let tail =
372            "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory";
373        let observed =
374            resolve_memory_exhaustion(Some(139), Some(tail), Some(false), None).expect("observed");
375        assert!(observed.memory_exhausted);
376        assert_eq!(observed.memory_exhausted_reason, tail);
377        // The same marker with a successful exit is just output, not a failure.
378        assert_eq!(
379            resolve_memory_exhaustion(Some(0), Some(tail), Some(false), None),
380            None
381        );
382        assert_eq!(
383            resolve_memory_exhaustion(None, Some(tail), None, None),
384            None
385        );
386    }
387
388    #[test]
389    fn memory_exhaustion_falls_back_to_the_container_flag() {
390        let observed = resolve_memory_exhaustion(Some(137), Some("no marker"), Some(true), None)
391            .expect("observed");
392        assert_eq!(
393            observed.memory_exhausted_reason,
394            "Docker reported State.OOMKilled=true"
395        );
396        assert_eq!(
397            resolve_memory_exhaustion(Some(1), Some("no marker"), Some(false), None),
398            None
399        );
400    }
401
402    #[test]
403    fn falls_back_to_cgroup_observation_then_signal() {
404        assert_eq!(
405            resolve_exit_reason(Some(137), None, Some(true), None),
406            Some("memory-exhaustion (cgroup-oom-killer)".to_string())
407        );
408        assert_eq!(
409            resolve_exit_reason(Some(139), Some("no marker here"), Some(false), None),
410            Some("signal (SIGSEGV)".to_string())
411        );
412        assert_eq!(resolve_exit_reason(Some(0), Some("fine"), None, None), None);
413    }
414}
start_command/exit_reason.rs

start_command/
exit_reason.rs
