start_command/
exit_reason.rs

1//! Exit reason hints for finished executions.
2//!
3//! A numeric exit code alone is frequently misleading: a Bun/Node process that
4//! exhausts its heap aborts itself, so the container reports `exitCode 139`
5//! with `OOMKilled false` even though the run died of memory exhaustion (issue
6//! #162, related to #144, #148 and #151). `start` already reads the tail of the
7//! log to find the terminal footer, so the same tail is scanned for well-known
8//! fatal markers and the finding is surfaced as an extra `exitReason` field.
9//!
10//! The same scan answers a narrower question for consumers that key off
11//! `oomKilled`: was this run killed by memory exhaustion at all? That answer is
12//! surfaced as `memoryExhausted` plus `memoryExhaustedReason`, the log line that
13//! carried the evidence (issue #165).
14//!
15//! Both fields are hints, never verdicts: they never change `status`,
16//! `exitCode` or `oomKilled`.
17
18use regex::Regex;
19
20/// Fatal markers, most specific first. The first matching entry wins.
21const EXIT_REASON_MARKERS: [(&str, &str); 5] = [
22    (
23        "memory-exhaustion (v8-heap-limit)",
24        r"(?i)FATAL ERROR:[^\r\n]*(?:Reached heap limit|JavaScript heap out of memory)",
25    ),
26    (
27        "memory-exhaustion (v8-heap-limit)",
28        r"(?i)JavaScript heap out of memory",
29    ),
30    (
31        "memory-exhaustion (kernel-oom-killer)",
32        r"(?i)Out of memory: Kill(?:ed)? process|oom-kill(?:er)?[: ]|Killed process \d+",
33    ),
34    (
35        "memory-exhaustion (go-runtime)",
36        r"(?i)fatal error: runtime: out of memory",
37    ),
38    (
39        "memory-exhaustion (allocation-failure)",
40        r"(?i)std::bad_alloc|Cannot allocate memory|memory allocation of \d+ bytes failed|Allocation failed - process out of memory|Array buffer allocation failed",
41    ),
42];
43
44/// Exit codes above 128 encode the signal that killed the process.
45/// Only the signals that actually show up in command logs are named.
46const SIGNAL_NAMES: [(i32, &str); 10] = [
47    (1, "SIGHUP"),
48    (2, "SIGINT"),
49    (3, "SIGQUIT"),
50    (4, "SIGILL"),
51    (6, "SIGABRT"),
52    (8, "SIGFPE"),
53    (9, "SIGKILL"),
54    (11, "SIGSEGV"),
55    (13, "SIGPIPE"),
56    (15, "SIGTERM"),
57];
58
59/// Reasons carrying this prefix report memory exhaustion, whatever the
60/// mechanism (runtime self-abort, kernel OOM killer, failed allocation).
61const MEMORY_EXHAUSTION_PREFIX: &str = "memory-exhaustion";
62
63/// A matched marker line longer than this is truncated: the evidence travels in
64/// `--status` output, and some runtimes print very long single-line traces.
65const MAX_MARKER_LINE_LENGTH: usize = 300;
66
67/// A fatal marker found in the log: its category and the line that carried it.
68#[derive(Debug, Clone, PartialEq, Eq)]
69pub struct ExitReasonMarker {
70    pub reason: String,
71    pub line: String,
72}
73
74/// The memory-exhaustion observation derived from a log tail (issue #165).
75#[derive(Debug, Clone, PartialEq, Eq)]
76pub struct MemoryExhaustion {
77    pub memory_exhausted: bool,
78    pub memory_exhausted_reason: String,
79}
80
81/// Extract the whole (trimmed, length-bounded) line containing byte `index`.
82fn extract_line_at(text: &str, index: usize) -> String {
83    let start = text[..index].rfind('\n').map(|i| i + 1).unwrap_or(0);
84    let end = text[index..]
85        .find('\n')
86        .map(|i| index + i)
87        .unwrap_or(text.len());
88    let line = text[start..end].trim();
89    if line.chars().count() <= MAX_MARKER_LINE_LENGTH {
90        return line.to_string();
91    }
92    let truncated: String = line.chars().take(MAX_MARKER_LINE_LENGTH).collect();
93    format!("{}...", truncated)
94}
95
96/// Scan log text for the first accepted fatal marker, most specific first.
97pub fn find_exit_reason_marker(
98    text: Option<&str>,
99    accept: impl Fn(&str) -> bool,
100) -> Option<ExitReasonMarker> {
101    let text = text?;
102    if text.is_empty() {
103        return None;
104    }
105    for (reason, pattern) in EXIT_REASON_MARKERS {
106        if !accept(reason) {
107            continue;
108        }
109        if let Ok(re) = Regex::new(pattern) {
110            if let Some(found) = re.find(text) {
111                return Some(ExitReasonMarker {
112                    reason: reason.to_string(),
113                    line: extract_line_at(text, found.start()),
114                });
115            }
116        }
117    }
118    None
119}
120
121/// Scan log text for a known fatal marker.
122pub fn detect_exit_reason(text: Option<&str>) -> Option<String> {
123    find_exit_reason_marker(text, |_| true).map(|marker| marker.reason)
124}
125
126/// Scan log text for a marker that reports memory exhaustion specifically.
127pub fn detect_memory_marker(text: Option<&str>) -> Option<ExitReasonMarker> {
128    find_exit_reason_marker(text, |reason| reason.starts_with(MEMORY_EXHAUSTION_PREFIX))
129}
130
131/// Decide whether an execution died of memory exhaustion, and say why.
132///
133/// Only abnormal exits are explained: a command that merely printed a fatal
134/// marker (an `rg` dump, a quoted incident log) and then succeeded is not a
135/// memory failure. An observation, never a verdict (issue #151).
136pub fn resolve_memory_exhaustion(
137    exit_code: Option<i32>,
138    log_tail: Option<&str>,
139    oom_killed: Option<bool>,
140) -> Option<MemoryExhaustion> {
141    match exit_code {
142        Some(code) if code != 0 => {}
143        _ => return None,
144    }
145    if let Some(marker) = detect_memory_marker(log_tail) {
146        return Some(MemoryExhaustion {
147            memory_exhausted: true,
148            memory_exhausted_reason: marker.line,
149        });
150    }
151    if oom_killed == Some(true) {
152        return Some(MemoryExhaustion {
153            memory_exhausted: true,
154            memory_exhausted_reason: "Docker reported State.OOMKilled=true".to_string(),
155        });
156    }
157    None
158}
159
160/// Map a shell exit code to the signal name it encodes.
161pub fn signal_name_for_exit_code(exit_code: Option<i32>) -> Option<String> {
162    let code = exit_code?;
163    if code <= 128 || code > 128 + 64 {
164        return None;
165    }
166    SIGNAL_NAMES
167        .iter()
168        .find(|(signal, _)| *signal == code - 128)
169        .map(|(_, name)| name.to_string())
170}
171
172/// Resolve the best available hint for why an execution ended.
173///
174/// Precedence: the log marker (evidence written by the command itself), then
175/// the cgroup OOM observation, then the signal encoded in the exit code.
176pub fn resolve_exit_reason(
177    exit_code: Option<i32>,
178    log_tail: Option<&str>,
179    oom_killed: Option<bool>,
180) -> Option<String> {
181    if let Some(reason) = detect_exit_reason(log_tail) {
182        return Some(reason);
183    }
184
185    if oom_killed == Some(true) {
186        return Some("memory-exhaustion (cgroup-oom-killer)".to_string());
187    }
188
189    signal_name_for_exit_code(exit_code).map(|name| format!("signal ({})", name))
190}
191
192#[cfg(test)]
193mod tests {
194    use super::*;
195
196    #[test]
197    fn detects_v8_heap_limit_from_the_incident_log() {
198        let tail =
199            "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory";
200        assert_eq!(
201            detect_exit_reason(Some(tail)),
202            Some("memory-exhaustion (v8-heap-limit)".to_string())
203        );
204    }
205
206    #[test]
207    fn detects_kernel_oom_killer() {
208        assert_eq!(
209            detect_exit_reason(Some("Out of memory: Killed process 1234 (bun)")),
210            Some("memory-exhaustion (kernel-oom-killer)".to_string())
211        );
212    }
213
214    #[test]
215    fn detects_other_runtimes_reporting_their_own_exhaustion() {
216        assert_eq!(
217            detect_exit_reason(Some("fatal error: runtime: out of memory")),
218            Some("memory-exhaustion (go-runtime)".to_string())
219        );
220        assert_eq!(
221            detect_exit_reason(Some("Array buffer allocation failed")),
222            Some("memory-exhaustion (allocation-failure)".to_string())
223        );
224    }
225
226    #[test]
227    fn returns_none_for_ordinary_output() {
228        assert_eq!(detect_exit_reason(Some("all tests passed")), None);
229        assert_eq!(detect_exit_reason(None), None);
230    }
231
232    #[test]
233    fn maps_signal_exit_codes() {
234        assert_eq!(signal_name_for_exit_code(Some(137)), Some("SIGKILL".into()));
235        assert_eq!(signal_name_for_exit_code(Some(139)), Some("SIGSEGV".into()));
236        assert_eq!(signal_name_for_exit_code(Some(0)), None);
237        assert_eq!(signal_name_for_exit_code(Some(1)), None);
238        assert_eq!(signal_name_for_exit_code(None), None);
239    }
240
241    #[test]
242    fn log_marker_wins_over_signal_and_cgroup_flag() {
243        let tail =
244            "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory";
245        assert_eq!(
246            resolve_exit_reason(Some(139), Some(tail), Some(false)),
247            Some("memory-exhaustion (v8-heap-limit)".to_string())
248        );
249    }
250
251    #[test]
252    fn memory_marker_returns_the_whole_matched_line() {
253        let tail = "building...\nFATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory\n----- Native stack trace -----\n";
254        let marker = detect_memory_marker(Some(tail)).expect("marker");
255        assert_eq!(marker.reason, "memory-exhaustion (v8-heap-limit)");
256        assert_eq!(
257            marker.line,
258            "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory"
259        );
260    }
261
262    #[test]
263    fn memory_marker_line_is_length_bounded() {
264        let tail = format!(
265            "{}JavaScript heap out of memory{}",
266            "x".repeat(400),
267            "y".repeat(400)
268        );
269        let marker = detect_memory_marker(Some(&tail)).expect("marker");
270        assert!(marker.line.chars().count() <= MAX_MARKER_LINE_LENGTH + 3);
271        assert!(marker.line.ends_with("..."));
272    }
273
274    #[test]
275    fn memory_exhaustion_explains_only_abnormal_exits() {
276        let tail =
277            "FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory";
278        let observed =
279            resolve_memory_exhaustion(Some(139), Some(tail), Some(false)).expect("observed");
280        assert!(observed.memory_exhausted);
281        assert_eq!(observed.memory_exhausted_reason, tail);
282        // The same marker with a successful exit is just output, not a failure.
283        assert_eq!(
284            resolve_memory_exhaustion(Some(0), Some(tail), Some(false)),
285            None
286        );
287        assert_eq!(resolve_memory_exhaustion(None, Some(tail), None), None);
288    }
289
290    #[test]
291    fn memory_exhaustion_falls_back_to_the_container_flag() {
292        let observed =
293            resolve_memory_exhaustion(Some(137), Some("no marker"), Some(true)).expect("observed");
294        assert_eq!(
295            observed.memory_exhausted_reason,
296            "Docker reported State.OOMKilled=true"
297        );
298        assert_eq!(
299            resolve_memory_exhaustion(Some(1), Some("no marker"), Some(false)),
300            None
301        );
302    }
303
304    #[test]
305    fn falls_back_to_cgroup_observation_then_signal() {
306        assert_eq!(
307            resolve_exit_reason(Some(137), None, Some(true)),
308            Some("memory-exhaustion (cgroup-oom-killer)".to_string())
309        );
310        assert_eq!(
311            resolve_exit_reason(Some(139), Some("no marker here"), Some(false)),
312            Some("signal (SIGSEGV)".to_string())
313        );
314        assert_eq!(resolve_exit_reason(Some(0), Some("fine"), None), None);
315    }
316}
start_command/exit_reason.rs

start_command/
exit_reason.rs
