start_command/
execution_resume.rs

1//! Resume tracked detached executions (issue #162).
2//!
3//! `--resume <id>` restarts a stored execution, and `--resume <id> -- <command>`
4//! runs a *different* command against the same container filesystem. Both keep
5//! the original execution UUID so `--status`, `--list` and `--upload-log` keep
6//! addressing one logical session across restarts.
7//!
8//! Three strategies, chosen from the probed session state:
9//! - `DockerStart`: the container still exists and the stored command is re-run
10//!   by `docker start` (its original entrypoint).
11//! - `DockerSnapshot`: the container still exists but a new command was given,
12//!   so its filesystem is committed to an image and a derived container runs
13//!   the new command. This avoids `docker start -ai`, which would re-run the
14//!   original entrypoint from scratch.
15//! - `Relaunch`: nothing is left of the session, so the command is launched
16//!   again through the stored isolation options.
17
18use std::path::PathBuf;
19
20use chrono::Utc;
21use serde_json::{json, Value};
22
23use crate::docker_cleanup::{
24    build_docker_runtime_args, docker_command, docker_networks,
25    get_docker_container_cleanup_policy, start_detached_docker_completion_watcher_with,
26    DockerWatcherOptions,
27};
28use crate::docker_resource_limits::{build_resource_limits_status_line, normalize_resource_limits};
29use crate::execution_control::{CommandRunner, SystemCommandRunner};
30use crate::execution_store::{ExecutionRecord, ExecutionStatus, ExecutionStore};
31use crate::isolation::isolation_log::append_log_file;
32use crate::isolation::{run_isolated, IsolationOptions, IsolationResult};
33use crate::output_blocks::escape_for_links_notation;
34use crate::session_probe::{probe_session, SessionProbe, SessionState};
35
36/// Strategies `build_resume_plan` can pick.
37#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38pub enum ResumeMode {
39    DockerStart,
40    DockerSnapshot,
41    Relaunch,
42}
43
44impl ResumeMode {
45    pub fn as_str(self) -> &'static str {
46        match self {
47            ResumeMode::DockerStart => "docker-start",
48            ResumeMode::DockerSnapshot => "docker-snapshot",
49            ResumeMode::Relaunch => "relaunch",
50        }
51    }
52}
53
54/// One command run while carrying out a resume plan.
55#[derive(Debug, Clone, PartialEq, Eq)]
56pub struct ResumeStep {
57    pub command: String,
58    pub args: Vec<String>,
59    pub description: String,
60}
61
62/// How a stored execution will be resumed.
63#[derive(Debug, Clone)]
64pub struct ResumePlan {
65    pub mode: ResumeMode,
66    pub backend: String,
67    pub session_name: String,
68    /// Set only when a snapshot-derived container replaces the original one.
69    pub new_session_name: Option<String>,
70    pub snapshot_image: Option<String>,
71    pub command: String,
72    pub attempt: u64,
73    /// Resource limits the new container is started with (snapshot resumes
74    /// only, issue #176): `docker commit` does not carry the HostConfig over.
75    pub resource_limits: Vec<String>,
76    pub steps: Vec<ResumeStep>,
77    pub launch_options: Option<IsolationOptions>,
78    pub message: String,
79}
80
81/// Result of `resume_execution`.
82pub struct ExecutionResumeResult {
83    pub success: bool,
84    pub output: Option<String>,
85    pub error: Option<String>,
86}
87
88/// Side effects a resume performs beyond running plan steps.
89///
90/// Injectable so tests can exercise the full resume flow without spawning a
91/// completion watcher, launching an isolation backend, or reading log files.
92pub trait ResumeHooks {
93    /// Attach a fresh completion watcher to a resumed docker session.
94    fn start_watcher(&self, session_name: &str, record: &ExecutionRecord);
95    /// Report attachment failures while keeping existing injectable hooks compatible.
96    fn attach_watcher(&self, session_name: &str, record: &ExecutionRecord) -> Result<(), String> {
97        self.start_watcher(session_name, record);
98        Ok(())
99    }
100
101    /// Launch the command again through its isolation backend.
102    fn relaunch(&self, backend: &str, command: &str, options: &IsolationOptions)
103        -> IsolationResult;
104
105    /// Resolve the terminal result of a session that ended unsupervised.
106    fn reconcile(&self, record: &ExecutionRecord) -> ExecutionRecord;
107}
108
109/// The real side effects: docker watcher, isolation backends, log evidence.
110#[derive(Debug, Default)]
111pub struct SystemResumeHooks;
112
113impl ResumeHooks for SystemResumeHooks {
114    fn start_watcher(&self, session_name: &str, record: &ExecutionRecord) {
115        let _ = self.attach_watcher(session_name, record);
116    }
117
118    fn attach_watcher(&self, session_name: &str, record: &ExecutionRecord) -> Result<(), String> {
119        let log_path = (!record.log_path.is_empty()).then(|| PathBuf::from(&record.log_path));
120        start_detached_docker_completion_watcher_with(
121            session_name,
122            get_docker_container_cleanup_policy(&build_launch_options(record)),
123            log_path.as_ref(),
124            // The resumed session gets the same record, so the watcher that
125            // outlives this process finalizes it too (issue #170.1).
126            Some(record.uuid.as_str()),
127            &DockerWatcherOptions {
128                since: record
129                    .attempt
130                    .as_ref()
131                    .map(|attempt| attempt.started_at.clone()),
132                attempt_number: record.attempt.as_ref().map(|attempt| attempt.number),
133                recover_on_kill: keeps_kill_recovery(session_name, record),
134            },
135        )
136    }
137
138    fn relaunch(
139        &self,
140        backend: &str,
141        command: &str,
142        options: &IsolationOptions,
143    ) -> IsolationResult {
144        run_isolated(backend, command, options)
145    }
146
147    fn reconcile(&self, record: &ExecutionRecord) -> ExecutionRecord {
148        crate::status_formatter::enrich_detached_status(record)
149    }
150}
151
152/// `docker start` re-runs the launch-time selector, so a container resumed in
153/// place or relaunched through isolation keeps its kill recovery (issue #176);
154/// a snapshot-derived container has no selector.
155pub fn keeps_kill_recovery(session_name: &str, record: &ExecutionRecord) -> bool {
156    record.attempt.as_ref().is_none_or(|attempt| {
157        matches!(
158            attempt.mode.as_str(),
159            "docker-start" | "automatic-recovery" | "relaunch"
160        )
161    }) && record_option(record, "sessionName") == Some(session_name)
162        && record
163            .options
164            .get("onKillResume")
165            .is_some_and(|value| value.as_u64().unwrap_or(0) > 0)
166}
167
168fn record_option<'a>(record: &'a ExecutionRecord, key: &str) -> Option<&'a str> {
169    record.options.get(key).and_then(|value| value.as_str())
170}
171
172fn record_flag(record: &ExecutionRecord, key: &str) -> bool {
173    record
174        .options
175        .get(key)
176        .and_then(|value| value.as_bool())
177        .unwrap_or(false)
178}
179
180fn record_strings(record: &ExecutionRecord, key: &str) -> Vec<String> {
181    record
182        .options
183        .get(key)
184        .and_then(|value| value.as_array())
185        .map(|items| {
186            items
187                .iter()
188                .filter_map(|item| item.as_str().map(str::to_string))
189                .collect()
190        })
191        .unwrap_or_default()
192}
193
194/// Build the docker image name used to snapshot a container before running a
195/// new command in it. Docker repository names must be lowercase and limited to
196/// `[a-z0-9._-]`, so session names are sanitized.
197pub fn build_snapshot_image_name(session_name: &str, attempt: u64) -> String {
198    let mut sanitized = String::new();
199    let mut pending_dash = false;
200    for ch in session_name.to_lowercase().chars() {
201        if ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '.' || ch == '_' || ch == '-' {
202            if pending_dash {
203                sanitized.push('-');
204                pending_dash = false;
205            }
206            sanitized.push(ch);
207        } else if !sanitized.is_empty() || !pending_dash {
208            pending_dash = true;
209        }
210    }
211    if pending_dash {
212        sanitized.push('-');
213    }
214    let sanitized = sanitized.trim_start_matches(['-', '.']).to_string();
215    let sanitized = if sanitized.is_empty() {
216        "session".to_string()
217    } else {
218        sanitized
219    };
220    format!("start-command-resume/{}:{}", sanitized, attempt)
221}
222
223/// Build the container name for a snapshot-based resume.
224pub fn build_resumed_session_name(session_name: &str, attempt: u64) -> String {
225    format!("{}-resume-{}", session_name, attempt)
226}
227
228/// Rebuild the isolation options stored on a record so the command can be
229/// launched again with the same configuration.
230pub fn build_launch_options(record: &ExecutionRecord) -> IsolationOptions {
231    let networks = record_strings(record, "networks");
232    IsolationOptions {
233        session: record_option(record, "sessionName").map(str::to_string),
234        image: record_option(record, "image").map(str::to_string),
235        volumes: record_strings(record, "volumes"),
236        mounts: record_strings(record, "mounts"),
237        env: record_strings(record, "env"),
238        privileged: record_flag(record, "privileged"),
239        network: networks.first().cloned(),
240        networks,
241        network_aliases: record_strings(record, "networkAliases"),
242        // Limits captured from the container on an earlier resume (issue #176).
243        resource_limits: normalize_resource_limits(record.options.get("resourceLimits")),
244        cpu_penalty_config: record
245            .options
246            .get("cpuPenaltyConfig")
247            .and_then(|v| serde_json::from_value(v.clone()).ok()),
248        on_kill_resume: record
249            .options
250            .get("onKillResume")
251            .and_then(Value::as_u64)
252            .map(|n| n as u32),
253        recovery_command: record_option(record, "recoveryCommand").map(str::to_string),
254        endpoint: record_option(record, "endpoint").map(str::to_string),
255        detached: true,
256        user: record_option(record, "user").map(str::to_string),
257        keep_alive: record_flag(record, "keepAlive"),
258        auto_remove_docker_container: record_flag(record, "autoRemoveDockerContainer"),
259        always_cleanup_container: record_flag(record, "alwaysCleanupContainer"),
260        keep_container: record_flag(record, "keepContainer"),
261        keep_container_on_fail: record_flag(record, "keepContainerOnFail"),
262        shell: record_option(record, "shell").unwrap_or("auto").to_string(),
263        // Append to the same log so one logical session keeps one gap-free record.
264        log_path: (!record.log_path.is_empty()).then(|| PathBuf::from(&record.log_path)),
265        // The relaunched session keeps writing to this record, so its watcher
266        // is the one that must mark it terminal (issue #170.1).
267        execution_id: Some(record.uuid.clone()),
268        defer_completion_watcher: true,
269    }
270}
271
272/// Build the steps that start a snapshot-derived container.
273///
274/// `docker run` can only join one network, so a session that was launched on
275/// several networks is rebuilt the same way `run_isolated` builds it: create
276/// the container, connect the additional networks, then start it. Dropping to a
277/// single `docker run` when there is nothing extra to connect keeps the common
278/// case to one command.
279fn build_snapshot_start_steps(
280    docker: &str,
281    container_args: Vec<String>,
282    extra_networks: &[String],
283    new_session_name: &str,
284    snapshot_image: &str,
285) -> Vec<ResumeStep> {
286    if extra_networks.is_empty() {
287        let mut args = vec!["run".to_string(), "-d".to_string()];
288        args.extend(container_args);
289        return vec![ResumeStep {
290            command: docker.to_string(),
291            args,
292            description: format!("Run the new command in {}", new_session_name),
293        }];
294    }
295
296    let mut create_args = vec!["create".to_string()];
297    create_args.extend(container_args);
298    let mut steps = vec![ResumeStep {
299        command: docker.to_string(),
300        args: create_args,
301        description: format!("Create {} from {}", new_session_name, snapshot_image),
302    }];
303    for network in extra_networks {
304        steps.push(ResumeStep {
305            command: docker.to_string(),
306            args: vec![
307                "network".to_string(),
308                "connect".to_string(),
309                network.clone(),
310                new_session_name.to_string(),
311            ],
312            description: format!("Connect {} to network {}", new_session_name, network),
313        });
314    }
315    steps.push(ResumeStep {
316        command: docker.to_string(),
317        args: vec!["start".to_string(), new_session_name.to_string()],
318        description: format!("Run the new command in {}", new_session_name),
319    });
320    steps
321}
322
323fn docker_snapshot_plan(
324    record: &ExecutionRecord,
325    backend: &str,
326    session_name: &str,
327    command: &str,
328    attempt: u64,
329    live_resource_limits: Option<Vec<String>>,
330) -> ResumePlan {
331    let snapshot_image = build_snapshot_image_name(session_name, attempt);
332    let new_session_name = build_resumed_session_name(session_name, attempt);
333    let docker = docker_command().to_string_lossy().to_string();
334
335    let mut launch_options = build_launch_options(record);
336    // `docker commit` keeps the filesystem but not the HostConfig, so the
337    // limits must be passed to the new container explicitly (issue #176).
338    if let Some(limits) = live_resource_limits {
339        launch_options.resource_limits = limits;
340    }
341    let mut container_args = vec!["--name".to_string(), new_session_name.clone()];
342    if let Some(user) = record_option(record, "user") {
343        container_args.push("--user".to_string());
344        container_args.push(user.to_string());
345    }
346    container_args.extend(
347        build_docker_runtime_args(&launch_options)
348            .into_iter()
349            .map(str::to_string),
350    );
351    container_args.push(snapshot_image.clone());
352    container_args.push("sh".to_string());
353    container_args.push("-c".to_string());
354    container_args.push(command.to_string());
355
356    let mut steps = vec![ResumeStep {
357        command: docker.clone(),
358        args: vec![
359            "commit".to_string(),
360            session_name.to_string(),
361            snapshot_image.clone(),
362        ],
363        description: format!("Snapshot container {} as {}", session_name, snapshot_image),
364    }];
365    steps.extend(build_snapshot_start_steps(
366        &docker,
367        container_args,
368        &docker_networks(&launch_options)
369            .into_iter()
370            .skip(1)
371            .map(str::to_string)
372            .collect::<Vec<_>>(),
373        &new_session_name,
374        &snapshot_image,
375    ));
376
377    ResumePlan {
378        mode: ResumeMode::DockerSnapshot,
379        backend: backend.to_string(),
380        session_name: session_name.to_string(),
381        new_session_name: Some(new_session_name.clone()),
382        snapshot_image: Some(snapshot_image.clone()),
383        command: command.to_string(),
384        attempt,
385        resource_limits: launch_options.resource_limits.clone(),
386        steps,
387        launch_options: None,
388        message: format!(
389            "Resumed session in new container {} from snapshot of {}",
390            new_session_name, session_name
391        ),
392    }
393}
394
395/// Decide how a stored execution should be resumed.
396pub fn build_resume_plan(
397    record: &ExecutionRecord,
398    new_command: Option<&str>,
399    probe: &SessionProbe,
400) -> Result<ResumePlan, String> {
401    build_resume_plan_with_limits(record, new_command, probe, None)
402}
403
404/// [`build_resume_plan`] with the resource limits read from the stopped
405/// container with `docker inspect`; falls back to the stored ones when `None`.
406pub fn build_resume_plan_with_limits(
407    record: &ExecutionRecord,
408    new_command: Option<&str>,
409    probe: &SessionProbe,
410    live_resource_limits: Option<Vec<String>>,
411) -> Result<ResumePlan, String> {
412    let session_name = record_option(record, "sessionName")
413        .ok_or_else(|| "Execution record does not contain an isolation session name.".to_string())?
414        .to_string();
415
416    if record_option(record, "isolationMode") != Some("detached") {
417        return Err("Only detached isolated executions can be resumed.".to_string());
418    }
419
420    if probe.alive {
421        return Err(format!(
422            "Session \"{}\" is still running. Use `$ --attach {}` to re-enter it, or `$ --stop {}` first.",
423            session_name, record.uuid, record.uuid
424        ));
425    }
426
427    let backend = record_option(record, "isolated")
428        .unwrap_or("unknown")
429        .to_string();
430    let command = new_command.unwrap_or(record.command.as_str()).to_string();
431    if command.is_empty() {
432        return Err(format!(
433            "Execution \"{}\" has no stored command to resume.",
434            record.uuid
435        ));
436    }
437
438    let attempt = record
439        .options
440        .get("resumeCount")
441        .and_then(|value| value.as_u64())
442        .unwrap_or(0)
443        + 1;
444
445    if backend == "docker" && probe.state == SessionState::Stopped {
446        return Ok(match new_command {
447            None => ResumePlan {
448                mode: ResumeMode::DockerStart,
449                backend,
450                new_session_name: None,
451                snapshot_image: None,
452                command,
453                attempt,
454                resource_limits: Vec::new(),
455                steps: vec![ResumeStep {
456                    command: docker_command().to_string_lossy().to_string(),
457                    args: vec!["start".to_string(), session_name.clone()],
458                    description: format!("Start stopped container {}", session_name),
459                }],
460                launch_options: None,
461                message: format!("Resumed detached docker container: {}", session_name),
462                session_name,
463            },
464            Some(new_command) => docker_snapshot_plan(
465                record,
466                &backend,
467                &session_name,
468                new_command,
469                attempt,
470                live_resource_limits,
471            ),
472        });
473    }
474
475    Ok(ResumePlan {
476        mode: ResumeMode::Relaunch,
477        message: format!("Relaunched {} session: {}", backend, session_name),
478        backend,
479        new_session_name: None,
480        snapshot_image: None,
481        command,
482        attempt,
483        resource_limits: Vec::new(),
484        steps: Vec::new(),
485        launch_options: Some(build_launch_options(record)),
486        session_name,
487    })
488}
489
490/// Fields reported after a resume attempt.
491pub struct ResumeResultFields<'a> {
492    pub identifier: &'a str,
493    pub uuid: &'a str,
494    pub mode: ResumeMode,
495    pub backend: &'a str,
496    pub session_name: &'a str,
497    pub previous_session_name: Option<&'a str>,
498    pub snapshot_image: Option<&'a str>,
499    /// Limits the snapshot-derived container was started with (issue #176).
500    pub resource_limits: &'a [String],
501    pub command: &'a str,
502    pub message: &'a str,
503}
504
505/// Format a resume result as links notation.
506pub fn format_resume_result_as_links_notation(result: &ResumeResultFields) -> String {
507    let mut lines = vec![
508        "executionResume".to_string(),
509        format!(
510            "  identifier {}",
511            escape_for_links_notation(result.identifier)
512        ),
513        format!("  uuid {}", escape_for_links_notation(result.uuid)),
514        format!("  mode {}", escape_for_links_notation(result.mode.as_str())),
515        format!("  backend {}", escape_for_links_notation(result.backend)),
516        format!(
517            "  sessionName {}",
518            escape_for_links_notation(result.session_name)
519        ),
520    ];
521    if let Some(previous) = result.previous_session_name {
522        lines.push(format!(
523            "  previousSessionName {}",
524            escape_for_links_notation(previous)
525        ));
526    }
527    if let Some(image) = result.snapshot_image {
528        lines.push(format!(
529            "  snapshotImage {}",
530            escape_for_links_notation(image)
531        ));
532    }
533    if !result.resource_limits.is_empty() {
534        lines.push(format!(
535            "  resourceLimits {}",
536            escape_for_links_notation(&result.resource_limits.join(" "))
537        ));
538    }
539    lines.push(format!(
540        "  command {}",
541        escape_for_links_notation(result.command)
542    ));
543    lines.push(format!(
544        "  message {}",
545        escape_for_links_notation(result.message)
546    ));
547    lines.join("\n")
548}
549
550/// Format a resume result in the requested output format.
551pub fn format_resume_result(result: &ResumeResultFields, output_format: Option<&str>) -> String {
552    match output_format {
553        Some("json") => {
554            let mut value = json!({
555                "identifier": result.identifier,
556                "uuid": result.uuid,
557                "mode": result.mode.as_str(),
558                "backend": result.backend,
559                "sessionName": result.session_name,
560                "previousSessionName": result.previous_session_name,
561                "snapshotImage": result.snapshot_image,
562                "resourceLimits": (!result.resource_limits.is_empty())
563                    .then_some(result.resource_limits),
564                "command": result.command,
565                "message": result.message,
566            });
567            if let Value::Object(ref mut map) = value {
568                map.retain(|_, entry| !entry.is_null());
569            }
570            serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{}".to_string())
571        }
572        Some("text") => {
573            let mut lines = vec![
574                format!("Resume Mode:   {}", result.mode.as_str()),
575                format!("UUID:          {}", result.uuid),
576                format!("Backend:       {}", result.backend),
577                format!("Session Name:  {}", result.session_name),
578                format!("Command:       {}", result.command),
579            ];
580            if !result.resource_limits.is_empty() {
581                lines.push(format!(
582                    "Resource Limits: {}",
583                    result.resource_limits.join(" ")
584                ));
585            }
586            lines.push(result.message.to_string());
587            lines.join("\n")
588        }
589        _ => format_resume_result_as_links_notation(result),
590    }
591}
592
593/// Apply the resume outcome to the stored record, keeping the original UUID so
594/// one logical session stays addressable across restarts.
595pub fn apply_resume_to_record(
596    record: &mut ExecutionRecord,
597    plan: &ResumePlan,
598    container_id: Option<&str>,
599) {
600    let attempt = crate::execution_attempt::create_attempt(
601        record,
602        plan.mode.as_str(),
603        active_session_name(plan),
604    );
605    crate::execution_attempt::archive_attempt(record);
606    record.attempt = Some(attempt);
607    record
608        .options
609        .insert("resumeCount".to_string(), json!(plan.attempt));
610    record.options.insert(
611        "resumedAt".to_string(),
612        json!(record.attempt.as_ref().unwrap().started_at),
613    );
614    // A resume is a new deliberate start: launch-time recovery applies again.
615    record.options.remove("stopRequestedAt");
616    record.options.remove("exitEvidence");
617    if !plan.resource_limits.is_empty() {
618        record
619            .options
620            .insert("resourceLimits".to_string(), json!(plan.resource_limits));
621    }
622
623    if let Some(new_session_name) = &plan.new_session_name {
624        let mut history = record
625            .options
626            .get("sessionNameHistory")
627            .and_then(|value| value.as_array())
628            .cloned()
629            .unwrap_or_default();
630        if let Some(previous) = record.options.get("sessionName").cloned() {
631            history.push(previous);
632        }
633        record
634            .options
635            .insert("sessionNameHistory".to_string(), Value::Array(history));
636        record
637            .options
638            .insert("sessionName".to_string(), json!(new_session_name));
639    }
640    if let Some(snapshot_image) = &plan.snapshot_image {
641        record
642            .options
643            .insert("image".to_string(), json!(snapshot_image));
644    }
645    if let Some(container_id) = container_id {
646        record
647            .options
648            .insert("containerId".to_string(), json!(container_id));
649    }
650
651    record.command = plan.command.clone();
652    record.status = ExecutionStatus::Executing;
653    record.exit_code = None;
654    record.end_time = None;
655    record.exit_reason = None;
656    record.oom_killed = None;
657}
658
659fn active_session_name(plan: &ResumePlan) -> &str {
660    plan.new_session_name
661        .as_deref()
662        .unwrap_or(&plan.session_name)
663}
664
665#[path = "execution_resume_flow.rs"]
666mod resume_flow;
667pub use resume_flow::{
668    resume_execution, resume_execution_with, resume_execution_with_options,
669    resume_execution_with_resources,
670};
start_command/execution_resume.rs

start_command/
execution_resume.rs
