start_command/
status_probe.rs

1//! Live probing of a detached execution's backend.
2//!
3//! Split out of `status_formatter` so that "what is this session doing right
4//! now" (a `docker inspect`, a `screen -ls`, a `/proc` lookup) stays separate
5//! from "how do we render a record". Both halves are needed by `--status`, and
6//! together they no longer fit in one file under the repository's 1000-line
7//! budget.
8
9use crate::detached_finalize::{
10    END_TIME_SOURCE_DOCKER_FINISHED_AT, END_TIME_SOURCE_LOG_FOOTER, END_TIME_SOURCE_OBSERVED_AT,
11};
12use crate::docker_cleanup::docker_command;
13use crate::docker_post_mortem::normalize_docker_timestamp;
14use crate::execution_store::ExecutionRecord;
15use crate::status_footer::LogFooter;
16use chrono::Utc;
17use std::process::Command;
18
19/// Live state of a detached docker container by name.
20#[derive(Clone, Default)]
21pub struct DockerState {
22    pub running: bool,
23    exit_code: Option<i32>,
24    oom_killed: Option<bool>,
25    /// `.State.StartedAt`, normalized (issue #171.1).
26    started_at: Option<String>,
27    /// `.State.FinishedAt`, normalized — the authoritative finish time of a
28    /// detached container, and the reason `--status` no longer has to invent
29    /// one (issue #170.2).
30    finished_at: Option<String>,
31}
32
33/// Inspect the live state of a detached docker container by name.
34///
35/// Distinguishes "running", "stopped (with a real exit code)", and "cannot be
36/// inspected at all". The last case matters on slow Docker-in-Docker hosts
37/// (issue #136): right after `docker run -d` returns, `docker inspect <name>`
38/// can transiently fail because the container is not visible yet. A failed
39/// inspect must NOT be read as "stopped"; it means "unknown", so callers can
40/// keep the session running instead of fabricating a terminal `-1` result.
41///
42/// Returns None when the container cannot be inspected (not found yet, removed,
43/// or docker error).
44pub(crate) fn inspect_docker_state(session_name: &str) -> Option<DockerState> {
45    let output = Command::new(docker_command())
46        .args([
47            "inspect",
48            "-f",
49            concat!(
50                "{{.State.Running}} {{.State.ExitCode}} {{.State.OOMKilled}} ",
51                "{{.State.StartedAt}} {{.State.FinishedAt}}"
52            ),
53            session_name,
54        ])
55        .output()
56        .ok()?;
57    if !output.status.success() {
58        return None;
59    }
60    let stdout = String::from_utf8_lossy(&output.stdout);
61    let trimmed = stdout.trim();
62    if trimmed.is_empty() {
63        return None;
64    }
65    let mut parts = trimmed.split_whitespace();
66    let running = parts.next() == Some("true");
67    let exit_code = parts.next().and_then(|value| value.parse::<i32>().ok());
68    let oom_killed = parts.next().and_then(|value| match value {
69        "true" => Some(true),
70        "false" => Some(false),
71        _ => None,
72    });
73    let started_at = normalize_docker_timestamp(parts.next());
74    let finished_at = normalize_docker_timestamp(parts.next());
75    Some(DockerState {
76        running,
77        exit_code,
78        oom_killed,
79        started_at,
80        finished_at,
81    })
82}
83
84pub(crate) fn is_detached_docker_record(record: &ExecutionRecord) -> bool {
85    record.options.get("isolated").and_then(|v| v.as_str()) == Some("docker")
86        && record.options.get("isolationMode").and_then(|v| v.as_str()) == Some("detached")
87        && record
88            .options
89            .get("sessionName")
90            .and_then(|v| v.as_str())
91            .is_some()
92}
93
94pub(crate) fn read_docker_state(record: &ExecutionRecord) -> Option<DockerState> {
95    if record.options.get("isolated")?.as_str()? != "docker" {
96        return None;
97    }
98    let session_name = record.options.get("sessionName")?.as_str()?;
99    let state = inspect_docker_state(session_name)?;
100    if let Some(attempt) = &record.attempt {
101        let older = state
102            .started_at
103            .as_deref()
104            .and_then(|s| chrono::DateTime::parse_from_rfc3339(s).ok())
105            .zip(chrono::DateTime::parse_from_rfc3339(&attempt.started_at).ok())
106            .is_some_and(|(started, since)| started < since);
107        if older {
108            return None;
109        }
110    }
111    Some(state)
112}
113
114/// Best-effort terminal exit code reported by the isolation backend itself
115/// (currently docker via `docker inspect .State.ExitCode`). Returns None when
116/// the backend cannot provide a real code, so callers never surface the `-1`
117/// sentinel for a session whose real exit code is simply not available yet.
118/// A running container has no terminal exit code (docker reports `0` for it),
119/// so only a stopped container contributes one.
120pub(crate) fn backend_exit_code(docker_state: Option<&DockerState>) -> Option<i32> {
121    let state = docker_state?;
122    if state.running {
123        None
124    } else {
125        state.exit_code
126    }
127}
128
129/// Reconcile the OOM observation from the stored record and from `docker inspect`.
130///
131/// `State.OOMKilled` is a container-cgroup flag: the kernel sets it when ANY
132/// process in the cgroup is OOM-killed and it is never cleared for the life of
133/// the container (moby/moby#47618). It is therefore an *observation*, never a
134/// verdict about the session (issue #151) — a container that lost one child
135/// process keeps running and can still exit `0`. Once observed, the flag stays
136/// `true` for the record.
137pub(crate) fn resolve_oom_observation(
138    record: &ExecutionRecord,
139    docker_state: Option<&DockerState>,
140) -> Option<bool> {
141    let inspected = docker_state.and_then(|state| state.oom_killed);
142    if record.oom_killed == Some(true) || inspected == Some(true) {
143        return Some(true);
144    }
145    if record.oom_killed == Some(false) || inspected == Some(false) {
146        return Some(false);
147    }
148    None
149}
150
151/// Check if a detached isolation session is still running
152/// Returns Some(true) if running, Some(false) if not, None if unable to determine
153pub fn is_detached_session_alive(record: &ExecutionRecord) -> Option<bool> {
154    let session_name = record.options.get("sessionName")?.as_str()?;
155    let isolation_mode = record.options.get("isolationMode")?.as_str()?;
156    let isolated = record.options.get("isolated")?.as_str()?;
157
158    if isolation_mode != "detached" {
159        return None;
160    }
161
162    match isolated {
163        "screen" => {
164            let output = Command::new("screen").args(["-ls"]).output().ok()?;
165            let stdout = String::from_utf8_lossy(&output.stdout);
166            Some(stdout.contains(session_name))
167        }
168        "tmux" => {
169            let status = Command::new("tmux")
170                .args(["has-session", "-t", session_name])
171                .output()
172                .ok()?;
173            Some(status.status.success())
174        }
175        "docker" => {
176            // A failed inspect means the container is not visible yet (still
177            // being created on a slow DinD host) or already removed — not
178            // "stopped". Return None (unknown) so the session is not falsely
179            // marked finished (issue #136).
180            inspect_docker_state(session_name).map(|state| state.running)
181        }
182        "ssh" => {
183            // For SSH, check if the local wrapper PID is still running
184            #[cfg(unix)]
185            {
186                if let Some(pid) = record.pid {
187                    let result = unsafe { libc::kill(pid as i32, 0) };
188                    Some(result == 0)
189                } else {
190                    None
191                }
192            }
193            #[cfg(not(unix))]
194            {
195                let _ = record.pid;
196                None
197            }
198        }
199        _ => None,
200    }
201}
202
203/// Set `end_time` on a record that just became terminal, and record where the
204/// timestamp came from (issue #170.2).
205///
206/// Precedence: the container's own `State.FinishedAt`, then the anchored
207/// `Finished:` footer `start` wrote, and only then the time the end was
208/// *observed* — which is marked as such instead of masquerading as a finish
209/// time, the way an unconditional `Utc::now()` used to.
210pub(crate) fn apply_end_time(
211    record: &mut ExecutionRecord,
212    docker_state: Option<&DockerState>,
213    footer: &LogFooter,
214) {
215    if let Some(started_at) = docker_state.and_then(|state| state.started_at.clone()) {
216        record.container_started_at = Some(started_at);
217    }
218    if record.end_time.is_some() {
219        return;
220    }
221    if let Some(finished_at) = docker_state.and_then(|state| state.finished_at.clone()) {
222        record.end_time = Some(finished_at);
223        record.end_time_source = Some(END_TIME_SOURCE_DOCKER_FINISHED_AT.to_string());
224        return;
225    }
226    if let Some(finished_at) = footer.finished_at.clone() {
227        record.end_time = Some(finished_at);
228        record.end_time_source = Some(END_TIME_SOURCE_LOG_FOOTER.to_string());
229        return;
230    }
231    let now = Utc::now().to_rfc3339();
232    record.end_time = Some(now.clone());
233    record.end_time_source = Some(END_TIME_SOURCE_OBSERVED_AT.to_string());
234    record.observed_at = Some(now);
235}
start_command/status_probe.rs

start_command/
status_probe.rs
