start_command/
exit_evidence.rs

1//! Exit-time evidence; cumulative cgroup counters and OOMKilled are historical.
2use crate::execution_control::{CommandRunner, SystemCommandRunner};
3use chrono::{DateTime, Duration};
4pub const MAIN_OOM: &str = "Exit evidence: main-oom (recent cgroup oom_kill delta)";
5pub const DOCKER_OOM: &str = "Exit evidence: main-oom (recent Docker oom event)";
6pub const DAEMON_RESTART: &str = "Exit evidence: docker-daemon-restart (Docker service journal)";
7
8pub fn recent_oom_delta(sample: &str, finished_at: &str) -> bool {
9    let fields = sample.split_whitespace().collect::<Vec<_>>();
10    let changed = fields.get(4).and_then(|s| s.parse::<i64>().ok());
11    let observed = fields.get(5).and_then(|s| s.parse::<i64>().ok());
12    match (
13        changed,
14        observed,
15        DateTime::parse_from_rfc3339(finished_at).ok(),
16    ) {
17        (Some(changed), Some(observed), Some(finished)) => {
18            changed > 0
19                && changed <= observed
20                && finished.timestamp().abs_diff(changed) <= 3
21                && finished.timestamp().abs_diff(observed) <= 3
22        }
23        _ => false,
24    }
25}
26
27pub fn from_log(log: Option<&str>) -> (bool, bool) {
28    let last = log
29        .unwrap_or("")
30        .lines()
31        .rfind(|s| s.starts_with("Exit evidence:"));
32    (
33        last == Some(MAIN_OOM) || last == Some(DOCKER_OOM),
34        last == Some(DAEMON_RESTART),
35    )
36}
37
38pub fn recent_docker_oom(events: &str, finished_at: &str) -> bool {
39    let Ok(finished) = DateTime::parse_from_rfc3339(finished_at) else {
40        return false;
41    };
42    let finished = finished.timestamp_millis() as f64 / 1000.0;
43    events.lines().any(|line| {
44        let line = line.trim();
45        !line.is_empty()
46            && line.bytes().all(|c| c.is_ascii_digit())
47            && line.parse::<u64>().is_ok_and(|at| {
48                let gap = finished - at as f64;
49                at > 0 && (-1.0..=3.0).contains(&gap)
50            })
51    })
52}
53
54/// Terminal evidence remains useful if the user moves or truncates the log.
55pub fn record_tail(
56    record: &crate::execution_store::ExecutionRecord,
57    tail: Option<String>,
58) -> Option<String> {
59    let stored = record
60        .options
61        .get("exitEvidence")
62        .filter(|value| value.is_object());
63    let marker = if stored
64        .and_then(|s| s.get("daemonRestart"))
65        .and_then(serde_json::Value::as_bool)
66        == Some(true)
67    {
68        Some(DAEMON_RESTART)
69    } else if stored
70        .and_then(|s| s.get("mainOom"))
71        .and_then(serde_json::Value::as_bool)
72        == Some(true)
73    {
74        Some(MAIN_OOM)
75    } else if stored.is_some() {
76        Some("Exit evidence: unavailable (no attributed exit-time OOM or daemon restart evidence)")
77    } else {
78        None
79    };
80    marker
81        .map(|marker| format!("{}\n{}\n", tail.as_deref().unwrap_or(""), marker))
82        .or(tail)
83}
84
85pub fn daemon_restart_evidence(journal: &str, container_id: &str) -> bool {
86    !container_id.is_empty()
87        && journal.lines().any(|s| {
88            [
89                "Main process exited",
90                "Stopping Docker",
91                "Starting Docker",
92                "Daemon shutdown complete",
93            ]
94            .iter()
95            .any(|m| s.contains(m))
96        })
97        && journal.lines().any(|s| {
98            s.contains(container_id)
99                && ["failed to exit", "using the force", "force kill"]
100                    .iter()
101                    .any(|m| s.contains(m))
102        })
103}
104
105pub fn collect<R: CommandRunner>(
106    name: &str,
107    sample: &str,
108    finished: &str,
109    exit: &str,
110    oom: &str,
111    runner: &R,
112) -> String {
113    let local = std::env::var("DOCKER_HOST")
114        .map(|s| s.starts_with("unix://"))
115        .unwrap_or(true);
116    if exit == "137" && !name.is_empty() {
117        if let Ok(end) = DateTime::parse_from_rfc3339(finished) {
118            let id = runner.run(
119                &crate::docker_cleanup::docker_command().to_string_lossy(),
120                &["inspect".into(), "-f".into(), "{{.Id}}".into(), name.into()],
121            );
122            let id_text = id.stdout.trim();
123            if id.success && id_text.len() == 64 && id_text.chars().all(|c| c.is_ascii_hexdigit()) {
124                let journal = if local {
125                    runner.run(
126                        "journalctl",
127                        &[
128                            "-u".into(),
129                            "docker.service".into(),
130                            "--since".into(),
131                            (end - Duration::seconds(60)).to_rfc3339(),
132                            "--until".into(),
133                            (end + Duration::seconds(1)).to_rfc3339(),
134                            "--no-pager".into(),
135                            "-o".into(),
136                            "cat".into(),
137                        ],
138                    )
139                } else {
140                    Default::default()
141                };
142                if journal.success && daemon_restart_evidence(&journal.stdout, id_text) {
143                    return DAEMON_RESTART.into();
144                }
145                let events = runner.run(
146                    &crate::docker_cleanup::docker_command().to_string_lossy(),
147                    &[
148                        "events".into(),
149                        "--since".into(),
150                        (end - Duration::seconds(60)).to_rfc3339(),
151                        "--until".into(),
152                        (end + Duration::seconds(1)).to_rfc3339(),
153                        "--filter".into(),
154                        format!("container={}", id_text),
155                        "--filter".into(),
156                        "event=oom".into(),
157                        "--format".into(),
158                        "{{.Time}}".into(),
159                    ],
160                );
161                if events.success && recent_docker_oom(&events.stdout, finished) {
162                    return DOCKER_OOM.into();
163                }
164                if std::env::var("START_DEBUG").as_deref() == Ok("1") {
165                    eprintln!("[DEBUG] Exit evidence: container={} finishedAt={} eventsAvailable={} oomEvents={:?} sample={:?}", id_text, finished, events.success, events.stdout, sample);
166                }
167            }
168        }
169    }
170    if oom == "true"
171        && (exit == "137" || exit.parse::<i32>().is_ok_and(|n| n < 0))
172        && recent_oom_delta(sample, finished)
173    {
174        return MAIN_OOM.into();
175    }
176    "Exit evidence: unavailable (no attributed exit-time OOM or daemon restart evidence)".into()
177}
178
179pub fn main(args: &[String]) {
180    if args.len() >= 5 {
181        println!(
182            "{}",
183            collect(
184                &args[0],
185                &args[1],
186                &args[2],
187                &args[3],
188                &args[4],
189                &SystemCommandRunner
190            )
191        );
192    }
193}
194
195pub fn snippet(name: &str, log: &str) -> String {
196    use crate::isolation::isolation_log::shell_quote;
197    let exe = std::env::current_exe()
198        .ok()
199        .map(|p| p.to_string_lossy().into_owned())
200        .unwrap_or_else(|| "$".into());
201    let stderr = if std::env::var("START_DEBUG").as_deref() == Ok("1") {
202        "2>&1"
203    } else {
204        "2>/dev/null"
205    };
206    format!("{} __start-exit-evidence {} \"$__start_command_cgroup\" \"$__start_command_finished\" \"$__start_command_exit\" \"$__start_command_oom\" >> {} {}", shell_quote(&exe), shell_quote(name), log, stderr)
207}
start_command/exit_evidence.rs

start_command/
exit_evidence.rs
