start_command/
status_probe.rs

1//! Live probing of a detached execution's backend.
2//!
3//! Split out of `status_formatter` so that "what is this session doing right
4//! now" (a `docker inspect`, a `screen -ls`, a `/proc` lookup) stays separate
5//! from "how do we render a record". Both halves are needed by `--status`, and
6//! together they no longer fit in one file under the repository's 1000-line
7//! budget.
8
9use crate::detached_finalize::{
10    END_TIME_SOURCE_DOCKER_FINISHED_AT, END_TIME_SOURCE_LOG_FOOTER, END_TIME_SOURCE_OBSERVED_AT,
11};
12use crate::docker_cleanup::docker_command;
13use crate::docker_post_mortem::normalize_docker_timestamp;
14use crate::execution_store::ExecutionRecord;
15use crate::status_footer::LogFooter;
16use chrono::Utc;
17use std::process::Command;
18
19/// Live state of a detached docker container by name.
20#[derive(Clone, Default)]
21pub struct DockerState {
22    pub running: bool,
23    exit_code: Option<i32>,
24    oom_killed: Option<bool>,
25    /// `.State.StartedAt`, normalized (issue #171.1).
26    started_at: Option<String>,
27    /// `.State.FinishedAt`, normalized — the authoritative finish time of a
28    /// detached container, and the reason `--status` no longer has to invent
29    /// one (issue #170.2).
30    finished_at: Option<String>,
31}
32
33/// Inspect the live state of a detached docker container by name.
34///
35/// Distinguishes "running", "stopped (with a real exit code)", and "cannot be
36/// inspected at all". The last case matters on slow Docker-in-Docker hosts
37/// (issue #136): right after `docker run -d` returns, `docker inspect <name>`
38/// can transiently fail because the container is not visible yet. A failed
39/// inspect must NOT be read as "stopped"; it means "unknown", so callers can
40/// keep the session running instead of fabricating a terminal `-1` result.
41///
42/// Returns None when the container cannot be inspected (not found yet, removed,
43/// or docker error).
44pub(crate) fn inspect_docker_state(session_name: &str) -> Option<DockerState> {
45    let output = Command::new(docker_command())
46        .args([
47            "inspect",
48            "-f",
49            concat!(
50                "{{.State.Running}} {{.State.ExitCode}} {{.State.OOMKilled}} ",
51                "{{.State.StartedAt}} {{.State.FinishedAt}}"
52            ),
53            session_name,
54        ])
55        .output()
56        .ok()?;
57    if !output.status.success() {
58        return None;
59    }
60    let stdout = String::from_utf8_lossy(&output.stdout);
61    let trimmed = stdout.trim();
62    if trimmed.is_empty() {
63        return None;
64    }
65    let mut parts = trimmed.split_whitespace();
66    let running = parts.next() == Some("true");
67    let exit_code = parts.next().and_then(|value| value.parse::<i32>().ok());
68    let oom_killed = parts.next().and_then(|value| match value {
69        "true" => Some(true),
70        "false" => Some(false),
71        _ => None,
72    });
73    let started_at = normalize_docker_timestamp(parts.next());
74    let finished_at = normalize_docker_timestamp(parts.next());
75    Some(DockerState {
76        running,
77        exit_code,
78        oom_killed,
79        started_at,
80        finished_at,
81    })
82}
83
84pub(crate) fn is_detached_docker_record(record: &ExecutionRecord) -> bool {
85    record.options.get("isolated").and_then(|v| v.as_str()) == Some("docker")
86        && record.options.get("isolationMode").and_then(|v| v.as_str()) == Some("detached")
87        && record
88            .options
89            .get("sessionName")
90            .and_then(|v| v.as_str())
91            .is_some()
92}
93
94pub(crate) fn read_docker_state(record: &ExecutionRecord) -> Option<DockerState> {
95    if record.options.get("isolated")?.as_str()? != "docker" {
96        return None;
97    }
98    let session_name = record.options.get("sessionName")?.as_str()?;
99    inspect_docker_state(session_name)
100}
101
102/// Best-effort terminal exit code reported by the isolation backend itself
103/// (currently docker via `docker inspect .State.ExitCode`). Returns None when
104/// the backend cannot provide a real code, so callers never surface the `-1`
105/// sentinel for a session whose real exit code is simply not available yet.
106/// A running container has no terminal exit code (docker reports `0` for it),
107/// so only a stopped container contributes one.
108pub(crate) fn backend_exit_code(docker_state: Option<&DockerState>) -> Option<i32> {
109    let state = docker_state?;
110    if state.running {
111        None
112    } else {
113        state.exit_code
114    }
115}
116
117/// Reconcile the OOM observation from the stored record and from `docker inspect`.
118///
119/// `State.OOMKilled` is a container-cgroup flag: the kernel sets it when ANY
120/// process in the cgroup is OOM-killed and it is never cleared for the life of
121/// the container (moby/moby#47618). It is therefore an *observation*, never a
122/// verdict about the session (issue #151) — a container that lost one child
123/// process keeps running and can still exit `0`. Once observed, the flag stays
124/// `true` for the record.
125pub(crate) fn resolve_oom_observation(
126    record: &ExecutionRecord,
127    docker_state: Option<&DockerState>,
128) -> Option<bool> {
129    let inspected = docker_state.and_then(|state| state.oom_killed);
130    if record.oom_killed == Some(true) || inspected == Some(true) {
131        return Some(true);
132    }
133    if record.oom_killed == Some(false) || inspected == Some(false) {
134        return Some(false);
135    }
136    None
137}
138
139/// Check if a detached isolation session is still running
140/// Returns Some(true) if running, Some(false) if not, None if unable to determine
141pub fn is_detached_session_alive(record: &ExecutionRecord) -> Option<bool> {
142    let session_name = record.options.get("sessionName")?.as_str()?;
143    let isolation_mode = record.options.get("isolationMode")?.as_str()?;
144    let isolated = record.options.get("isolated")?.as_str()?;
145
146    if isolation_mode != "detached" {
147        return None;
148    }
149
150    match isolated {
151        "screen" => {
152            let output = Command::new("screen").args(["-ls"]).output().ok()?;
153            let stdout = String::from_utf8_lossy(&output.stdout);
154            Some(stdout.contains(session_name))
155        }
156        "tmux" => {
157            let status = Command::new("tmux")
158                .args(["has-session", "-t", session_name])
159                .output()
160                .ok()?;
161            Some(status.status.success())
162        }
163        "docker" => {
164            // A failed inspect means the container is not visible yet (still
165            // being created on a slow DinD host) or already removed — not
166            // "stopped". Return None (unknown) so the session is not falsely
167            // marked finished (issue #136).
168            inspect_docker_state(session_name).map(|state| state.running)
169        }
170        "ssh" => {
171            // For SSH, check if the local wrapper PID is still running
172            #[cfg(unix)]
173            {
174                if let Some(pid) = record.pid {
175                    let result = unsafe { libc::kill(pid as i32, 0) };
176                    Some(result == 0)
177                } else {
178                    None
179                }
180            }
181            #[cfg(not(unix))]
182            {
183                let _ = record.pid;
184                None
185            }
186        }
187        _ => None,
188    }
189}
190
191/// Set `end_time` on a record that just became terminal, and record where the
192/// timestamp came from (issue #170.2).
193///
194/// Precedence: the container's own `State.FinishedAt`, then the anchored
195/// `Finished:` footer `start` wrote, and only then the time the end was
196/// *observed* — which is marked as such instead of masquerading as a finish
197/// time, the way an unconditional `Utc::now()` used to.
198pub(crate) fn apply_end_time(
199    record: &mut ExecutionRecord,
200    docker_state: Option<&DockerState>,
201    footer: &LogFooter,
202) {
203    if let Some(started_at) = docker_state.and_then(|state| state.started_at.clone()) {
204        record.container_started_at = Some(started_at);
205    }
206    if record.end_time.is_some() {
207        return;
208    }
209    if let Some(finished_at) = docker_state.and_then(|state| state.finished_at.clone()) {
210        record.end_time = Some(finished_at);
211        record.end_time_source = Some(END_TIME_SOURCE_DOCKER_FINISHED_AT.to_string());
212        return;
213    }
214    if let Some(finished_at) = footer.finished_at.clone() {
215        record.end_time = Some(finished_at);
216        record.end_time_source = Some(END_TIME_SOURCE_LOG_FOOTER.to_string());
217        return;
218    }
219    let now = Utc::now().to_rfc3339();
220    record.end_time = Some(now.clone());
221    record.end_time_source = Some(END_TIME_SOURCE_OBSERVED_AT.to_string());
222    record.observed_at = Some(now);
223}
start_command/status_probe.rs

start_command/
status_probe.rs
