start_command/
exit_evidence.rs

1//! Exit-time evidence; cumulative cgroup counters and OOMKilled are historical.
2use crate::execution_control::{CommandRunner, SystemCommandRunner};
3use chrono::{DateTime, Duration};
4pub const MAIN_OOM: &str = "Exit evidence: main-oom (recent cgroup oom_kill delta)";
5pub const DAEMON_RESTART: &str = "Exit evidence: docker-daemon-restart (Docker service journal)";
6
7pub fn recent_oom_delta(sample: &str, finished_at: &str) -> bool {
8    let fields = sample.split_whitespace().collect::<Vec<_>>();
9    let changed = fields.get(4).and_then(|s| s.parse::<i64>().ok());
10    let observed = fields.get(5).and_then(|s| s.parse::<i64>().ok());
11    match (
12        changed,
13        observed,
14        DateTime::parse_from_rfc3339(finished_at).ok(),
15    ) {
16        (Some(changed), Some(observed), Some(finished)) => {
17            changed > 0
18                && changed <= observed
19                && finished.timestamp().abs_diff(changed) <= 3
20                && finished.timestamp().abs_diff(observed) <= 3
21        }
22        _ => false,
23    }
24}
25
26pub fn from_log(log: Option<&str>) -> (bool, bool) {
27    let last = log
28        .unwrap_or("")
29        .lines()
30        .rfind(|s| s.starts_with("Exit evidence:"));
31    (last == Some(MAIN_OOM), last == Some(DAEMON_RESTART))
32}
33
34/// Terminal evidence remains useful if the user moves or truncates the log.
35pub fn record_tail(
36    record: &crate::execution_store::ExecutionRecord,
37    tail: Option<String>,
38) -> Option<String> {
39    let stored = record.options.get("exitEvidence");
40    let marker = if stored
41        .and_then(|s| s.get("daemonRestart"))
42        .and_then(serde_json::Value::as_bool)
43        == Some(true)
44    {
45        Some(DAEMON_RESTART)
46    } else if stored
47        .and_then(|s| s.get("mainOom"))
48        .and_then(serde_json::Value::as_bool)
49        == Some(true)
50    {
51        Some(MAIN_OOM)
52    } else {
53        None
54    };
55    marker
56        .map(|marker| format!("{}\n{}\n", tail.as_deref().unwrap_or(""), marker))
57        .or(tail)
58}
59
60pub fn daemon_restart_evidence(journal: &str, container_id: &str) -> bool {
61    !container_id.is_empty()
62        && journal.lines().any(|s| {
63            [
64                "Main process exited",
65                "Stopping Docker",
66                "Starting Docker",
67                "Daemon shutdown complete",
68            ]
69            .iter()
70            .any(|m| s.contains(m))
71        })
72        && journal.lines().any(|s| {
73            s.contains(container_id)
74                && ["failed to exit", "using the force", "force kill"]
75                    .iter()
76                    .any(|m| s.contains(m))
77        })
78}
79
80pub fn collect<R: CommandRunner>(
81    name: &str,
82    sample: &str,
83    finished: &str,
84    exit: &str,
85    oom: &str,
86    runner: &R,
87) -> String {
88    let local = std::env::var("DOCKER_HOST")
89        .map(|s| s.starts_with("unix://"))
90        .unwrap_or(true);
91    if local && exit == "137" {
92        if let Ok(end) = DateTime::parse_from_rfc3339(finished) {
93            let id = runner.run(
94                &crate::docker_cleanup::docker_command().to_string_lossy(),
95                &["inspect".into(), "-f".into(), "{{.Id}}".into(), name.into()],
96            );
97            let id_text = id.stdout.trim();
98            if id.success && id_text.len() == 64 && id_text.chars().all(|c| c.is_ascii_hexdigit()) {
99                let journal = runner.run(
100                    "journalctl",
101                    &[
102                        "-u".into(),
103                        "docker.service".into(),
104                        "--since".into(),
105                        (end - Duration::seconds(60)).to_rfc3339(),
106                        "--until".into(),
107                        (end + Duration::seconds(1)).to_rfc3339(),
108                        "--no-pager".into(),
109                        "-o".into(),
110                        "cat".into(),
111                    ],
112                );
113                if journal.success && daemon_restart_evidence(&journal.stdout, id_text) {
114                    return DAEMON_RESTART.into();
115                }
116            }
117        }
118    }
119    if oom == "true"
120        && (exit == "137" || exit.parse::<i32>().is_ok_and(|n| n < 0))
121        && recent_oom_delta(sample, finished)
122    {
123        return MAIN_OOM.into();
124    }
125    "Exit evidence: unavailable (no attributed exit-time OOM or daemon restart evidence)".into()
126}
127
128pub fn main(args: &[String]) {
129    if args.len() >= 5 {
130        println!(
131            "{}",
132            collect(
133                &args[0],
134                &args[1],
135                &args[2],
136                &args[3],
137                &args[4],
138                &SystemCommandRunner
139            )
140        );
141    }
142}
143
144pub fn snippet(name: &str, log: &str) -> String {
145    use crate::isolation::isolation_log::shell_quote;
146    let exe = std::env::current_exe()
147        .ok()
148        .map(|p| p.to_string_lossy().into_owned())
149        .unwrap_or_else(|| "$".into());
150    format!("{} __start-exit-evidence {} \"$__start_command_cgroup\" \"$__start_command_finished\" \"$__start_command_exit\" \"$__start_command_oom\" >> {} 2>/dev/null", shell_quote(&exe), shell_quote(name), log)
151}
start_command/exit_evidence.rs

start_command/
exit_evidence.rs
