1use std::path::PathBuf;
19
20use chrono::Utc;
21use serde_json::{json, Value};
22
23use crate::docker_cleanup::{
24 build_docker_runtime_args, docker_command, docker_networks,
25 get_docker_container_cleanup_policy, start_detached_docker_completion_watcher_with,
26 DockerWatcherOptions,
27};
28use crate::docker_resource_limits::{build_resource_limits_status_line, normalize_resource_limits};
29use crate::execution_control::{CommandRunner, SystemCommandRunner};
30use crate::execution_store::{ExecutionRecord, ExecutionStatus, ExecutionStore};
31use crate::isolation::isolation_log::append_log_file;
32use crate::isolation::{run_isolated, IsolationOptions, IsolationResult};
33use crate::output_blocks::escape_for_links_notation;
34use crate::session_probe::{probe_session, SessionProbe, SessionState};
35
36#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38pub enum ResumeMode {
39 DockerStart,
40 DockerSnapshot,
41 Relaunch,
42}
43
44impl ResumeMode {
45 pub fn as_str(self) -> &'static str {
46 match self {
47 ResumeMode::DockerStart => "docker-start",
48 ResumeMode::DockerSnapshot => "docker-snapshot",
49 ResumeMode::Relaunch => "relaunch",
50 }
51 }
52}
53
54#[derive(Debug, Clone, PartialEq, Eq)]
56pub struct ResumeStep {
57 pub command: String,
58 pub args: Vec<String>,
59 pub description: String,
60}
61
62#[derive(Debug, Clone)]
64pub struct ResumePlan {
65 pub mode: ResumeMode,
66 pub backend: String,
67 pub session_name: String,
68 pub new_session_name: Option<String>,
70 pub snapshot_image: Option<String>,
71 pub command: String,
72 pub attempt: u64,
73 pub resource_limits: Vec<String>,
76 pub steps: Vec<ResumeStep>,
77 pub launch_options: Option<IsolationOptions>,
78 pub message: String,
79}
80
81pub struct ExecutionResumeResult {
83 pub success: bool,
84 pub output: Option<String>,
85 pub error: Option<String>,
86 pub exit_code: Option<i32>,
89}
90
91pub trait ResumeHooks {
96 fn start_watcher(&self, session_name: &str, record: &ExecutionRecord);
98 fn attach_watcher(&self, session_name: &str, record: &ExecutionRecord) -> Result<(), String> {
100 self.start_watcher(session_name, record);
101 Ok(())
102 }
103
104 fn relaunch(&self, backend: &str, command: &str, options: &IsolationOptions)
106 -> IsolationResult;
107
108 fn reconcile(&self, record: &ExecutionRecord) -> ExecutionRecord;
110}
111
112#[derive(Debug, Default)]
114pub struct SystemResumeHooks;
115
116impl ResumeHooks for SystemResumeHooks {
117 fn start_watcher(&self, session_name: &str, record: &ExecutionRecord) {
118 let _ = self.attach_watcher(session_name, record);
119 }
120
121 fn attach_watcher(&self, session_name: &str, record: &ExecutionRecord) -> Result<(), String> {
122 let log_path = (!record.log_path.is_empty()).then(|| PathBuf::from(&record.log_path));
123 start_detached_docker_completion_watcher_with(
124 session_name,
125 get_docker_container_cleanup_policy(&build_launch_options(record)),
126 log_path.as_ref(),
127 Some(record.uuid.as_str()),
130 &DockerWatcherOptions {
131 since: record
132 .attempt
133 .as_ref()
134 .map(|attempt| attempt.started_at.clone()),
135 attempt_number: record.attempt.as_ref().map(|attempt| attempt.number),
136 recover_on_kill: keeps_kill_recovery(session_name, record),
137 },
138 )
139 }
140
141 fn relaunch(
142 &self,
143 backend: &str,
144 command: &str,
145 options: &IsolationOptions,
146 ) -> IsolationResult {
147 run_isolated(backend, command, options)
148 }
149
150 fn reconcile(&self, record: &ExecutionRecord) -> ExecutionRecord {
151 crate::status_formatter::enrich_detached_status(record)
152 }
153}
154
155pub fn keeps_kill_recovery(session_name: &str, record: &ExecutionRecord) -> bool {
159 record.attempt.as_ref().is_none_or(|attempt| {
160 matches!(
161 attempt.mode.as_str(),
162 "docker-start" | "automatic-recovery" | "relaunch"
163 )
164 }) && record_option(record, "sessionName") == Some(session_name)
165 && record
166 .options
167 .get("onKillResume")
168 .is_some_and(|value| value.as_u64().unwrap_or(0) > 0)
169}
170
171fn record_option<'a>(record: &'a ExecutionRecord, key: &str) -> Option<&'a str> {
172 record.options.get(key).and_then(|value| value.as_str())
173}
174
175fn record_flag(record: &ExecutionRecord, key: &str) -> bool {
176 record
177 .options
178 .get(key)
179 .and_then(|value| value.as_bool())
180 .unwrap_or(false)
181}
182
183fn record_strings(record: &ExecutionRecord, key: &str) -> Vec<String> {
184 record
185 .options
186 .get(key)
187 .and_then(|value| value.as_array())
188 .map(|items| {
189 items
190 .iter()
191 .filter_map(|item| item.as_str().map(str::to_string))
192 .collect()
193 })
194 .unwrap_or_default()
195}
196
197pub fn build_snapshot_image_name(session_name: &str, attempt: u64) -> String {
201 let mut sanitized = String::new();
202 let mut pending_dash = false;
203 for ch in session_name.to_lowercase().chars() {
204 if ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '.' || ch == '_' || ch == '-' {
205 if pending_dash {
206 sanitized.push('-');
207 pending_dash = false;
208 }
209 sanitized.push(ch);
210 } else if !sanitized.is_empty() || !pending_dash {
211 pending_dash = true;
212 }
213 }
214 if pending_dash {
215 sanitized.push('-');
216 }
217 let sanitized = sanitized.trim_start_matches(['-', '.']).to_string();
218 let sanitized = if sanitized.is_empty() {
219 "session".to_string()
220 } else {
221 sanitized
222 };
223 format!("start-command-resume/{}:{}", sanitized, attempt)
224}
225
226pub fn build_resumed_session_name(session_name: &str, attempt: u64) -> String {
228 format!("{}-resume-{}", session_name, attempt)
229}
230
231pub fn build_launch_options(record: &ExecutionRecord) -> IsolationOptions {
234 let networks = record_strings(record, "networks");
235 IsolationOptions {
236 session: record_option(record, "sessionName").map(str::to_string),
237 labels: record_strings(record, "labels"),
238 uuid: Some(record.uuid.clone()),
239 root_session: Some(
240 record_option(record, "rootSession")
241 .or_else(|| {
242 record
243 .options
244 .get("sessionNameHistory")
245 .and_then(Value::as_array)
246 .and_then(|history| history.first())
247 .and_then(Value::as_str)
248 })
249 .unwrap_or_else(|| record_option(record, "sessionName").unwrap_or("session"))
250 .to_string(),
251 ),
252 resume_count: record
253 .options
254 .get("resumeCount")
255 .and_then(Value::as_u64)
256 .unwrap_or(0) as u32
257 + 1,
258 image: record_option(record, "image").map(str::to_string),
259 volumes: record_strings(record, "volumes"),
260 mounts: record_strings(record, "mounts"),
261 env: record_strings(record, "env"),
262 privileged: record_flag(record, "privileged"),
263 init: record_flag(record, "init"),
264 network: networks.first().cloned(),
265 networks,
266 network_aliases: record_strings(record, "networkAliases"),
267 resource_limits: normalize_resource_limits(record.options.get("resourceLimits")),
269 cpu_penalty_config: record
270 .options
271 .get("cpuPenaltyConfig")
272 .and_then(|v| serde_json::from_value(v.clone()).ok()),
273 on_kill_resume: record
274 .options
275 .get("onKillResume")
276 .and_then(Value::as_u64)
277 .map(|n| n as u32),
278 recovery_command: record_option(record, "recoveryCommand").map(str::to_string),
279 endpoint: record_option(record, "endpoint").map(str::to_string),
280 detached: true,
281 user: record_option(record, "user").map(str::to_string),
282 keep_alive: record_flag(record, "keepAlive"),
283 auto_remove_docker_container: record_flag(record, "autoRemoveDockerContainer"),
284 always_cleanup_container: record_flag(record, "alwaysCleanupContainer"),
285 keep_container: record_flag(record, "keepContainer"),
286 keep_container_on_fail: record_flag(record, "keepContainerOnFail"),
287 shell: record_option(record, "shell").unwrap_or("auto").to_string(),
288 log_path: (!record.log_path.is_empty()).then(|| PathBuf::from(&record.log_path)),
290 execution_id: Some(record.uuid.clone()),
293 defer_completion_watcher: true,
294 attempt_number: crate::execution_attempt::next_attempt_number(record),
296 }
297}
298
299fn build_snapshot_start_steps(
307 docker: &str,
308 container_args: Vec<String>,
309 extra_networks: &[String],
310 new_session_name: &str,
311 snapshot_image: &str,
312) -> Vec<ResumeStep> {
313 if extra_networks.is_empty() {
314 let mut args = vec!["run".to_string(), "-d".to_string()];
315 args.extend(container_args);
316 return vec![ResumeStep {
317 command: docker.to_string(),
318 args,
319 description: format!("Run the new command in {}", new_session_name),
320 }];
321 }
322
323 let mut create_args = vec!["create".to_string()];
324 create_args.extend(container_args);
325 let mut steps = vec![ResumeStep {
326 command: docker.to_string(),
327 args: create_args,
328 description: format!("Create {} from {}", new_session_name, snapshot_image),
329 }];
330 for network in extra_networks {
331 steps.push(ResumeStep {
332 command: docker.to_string(),
333 args: vec![
334 "network".to_string(),
335 "connect".to_string(),
336 network.clone(),
337 new_session_name.to_string(),
338 ],
339 description: format!("Connect {} to network {}", new_session_name, network),
340 });
341 }
342 steps.push(ResumeStep {
343 command: docker.to_string(),
344 args: vec!["start".to_string(), new_session_name.to_string()],
345 description: format!("Run the new command in {}", new_session_name),
346 });
347 steps
348}
349
350fn docker_snapshot_plan(
351 record: &ExecutionRecord,
352 backend: &str,
353 session_name: &str,
354 command: &str,
355 attempt: u64,
356 live_resource_limits: Option<Vec<String>>,
357) -> ResumePlan {
358 let snapshot_image = build_snapshot_image_name(session_name, attempt);
359 let new_session_name = build_resumed_session_name(session_name, attempt);
360 let docker = docker_command().to_string_lossy().to_string();
361
362 let mut launch_options = build_launch_options(record);
363 launch_options.session = Some(new_session_name.clone());
364 launch_options.resume_count = attempt as u32;
365 launch_options.labels =
366 crate::isolation_metadata::docker_attribution_labels(&launch_options, &new_session_name);
367 if let Some(limits) = live_resource_limits {
370 launch_options.resource_limits = limits;
371 }
372 let mut container_args = vec!["--name".to_string(), new_session_name.clone()];
373 container_args.extend([
374 "--label".into(),
375 format!(
376 "{}={}",
377 crate::docker_snapshot_safety::SNAPSHOT_IMAGE_LABEL,
378 snapshot_image
379 ),
380 ]);
381 if let Some(user) = record_option(record, "user") {
382 container_args.push("--user".to_string());
383 container_args.push(user.to_string());
384 }
385 container_args.extend(build_docker_runtime_args(&launch_options));
386 container_args.push(snapshot_image.clone());
387 container_args.extend(crate::docker_command_handoff::build_command_handoff_args(
388 &["sh".into(), "-c".into(), command.into()],
389 &crate::docker_command_handoff::command_handoff_path(&new_session_name),
390 ));
391
392 let mut steps = vec![ResumeStep {
393 command: docker.clone(),
394 args: vec![
395 "commit".to_string(),
396 session_name.to_string(),
397 snapshot_image.clone(),
398 ],
399 description: format!("Snapshot container {} as {}", session_name, snapshot_image),
400 }];
401 steps.extend(build_snapshot_start_steps(
402 &docker,
403 container_args,
404 &docker_networks(&launch_options)
405 .into_iter()
406 .skip(1)
407 .map(str::to_string)
408 .collect::<Vec<_>>(),
409 &new_session_name,
410 &snapshot_image,
411 ));
412
413 ResumePlan {
414 mode: ResumeMode::DockerSnapshot,
415 backend: backend.to_string(),
416 session_name: session_name.to_string(),
417 new_session_name: Some(new_session_name.clone()),
418 snapshot_image: Some(snapshot_image.clone()),
419 command: command.to_string(),
420 attempt,
421 resource_limits: launch_options.resource_limits.clone(),
422 steps,
423 launch_options: None,
424 message: format!(
425 "Resumed session in new container {} from snapshot of {}",
426 new_session_name, session_name
427 ),
428 }
429}
430
431pub fn build_resume_plan(
433 record: &ExecutionRecord,
434 new_command: Option<&str>,
435 probe: &SessionProbe,
436) -> Result<ResumePlan, String> {
437 build_resume_plan_with_limits(record, new_command, probe, None)
438}
439
440pub fn build_resume_plan_with_limits(
443 record: &ExecutionRecord,
444 new_command: Option<&str>,
445 probe: &SessionProbe,
446 live_resource_limits: Option<Vec<String>>,
447) -> Result<ResumePlan, String> {
448 let session_name = record_option(record, "sessionName")
449 .ok_or_else(|| "Execution record does not contain an isolation session name.".to_string())?
450 .to_string();
451
452 if record_option(record, "isolationMode") != Some("detached") {
453 return Err("Only detached isolated executions can be resumed.".to_string());
454 }
455
456 if probe.alive {
457 return Err(format!(
458 "Session \"{}\" is still running. Use `$ --attach {}` to re-enter it, or `$ --stop {}` first.",
459 session_name, record.uuid, record.uuid
460 ));
461 }
462
463 let backend = record_option(record, "isolated")
464 .unwrap_or("unknown")
465 .to_string();
466 let command = new_command.unwrap_or(record.command.as_str()).to_string();
467 if command.is_empty() {
468 return Err(format!(
469 "Execution \"{}\" has no stored command to resume.",
470 record.uuid
471 ));
472 }
473
474 let attempt = record
475 .options
476 .get("resumeCount")
477 .and_then(|value| value.as_u64())
478 .unwrap_or(0)
479 + 1;
480
481 if backend == "docker" && probe.state == SessionState::Stopped {
482 return Ok(match new_command {
483 None | Some(_)
484 if !record_flag(record, "forceSnapshot")
485 && (new_command.is_none() || record_flag(record, "commandHandoff")) =>
486 {
487 ResumePlan {
488 mode: ResumeMode::DockerStart,
489 backend,
490 new_session_name: None,
491 snapshot_image: None,
492 command,
493 attempt,
494 resource_limits: Vec::new(),
495 steps: vec![ResumeStep {
496 command: docker_command().to_string_lossy().to_string(),
497 args: vec!["start".to_string(), session_name.clone()],
498 description: format!("Start stopped container {}", session_name),
499 }],
500 launch_options: None,
501 message: if new_command.is_some() {
502 format!("Resumed new command in the same detached docker container: {} (no filesystem snapshot)", session_name)
503 } else {
504 format!("Resumed detached docker container: {}", session_name)
505 },
506 session_name,
507 }
508 }
509 Some(new_command) => docker_snapshot_plan(
510 record,
511 &backend,
512 &session_name,
513 new_command,
514 attempt,
515 live_resource_limits,
516 ),
517 None => docker_snapshot_plan(
518 record,
519 &backend,
520 &session_name,
521 &command,
522 attempt,
523 live_resource_limits,
524 ),
525 });
526 }
527
528 Ok(ResumePlan {
529 mode: ResumeMode::Relaunch,
530 message: format!("Relaunched {} session: {}", backend, session_name),
531 backend,
532 new_session_name: None,
533 snapshot_image: None,
534 command,
535 attempt,
536 resource_limits: Vec::new(),
537 steps: Vec::new(),
538 launch_options: Some(build_launch_options(record)),
539 session_name,
540 })
541}
542
543pub struct ResumeResultFields<'a> {
545 pub identifier: &'a str,
546 pub uuid: &'a str,
547 pub mode: ResumeMode,
548 pub backend: &'a str,
549 pub session_name: &'a str,
550 pub previous_session_name: Option<&'a str>,
551 pub snapshot_image: Option<&'a str>,
552 pub resource_limits: &'a [String],
554 pub command: &'a str,
555 pub message: &'a str,
556}
557
558pub fn format_resume_result_as_links_notation(result: &ResumeResultFields) -> String {
560 let mut lines = vec![
561 "executionResume".to_string(),
562 format!(
563 " identifier {}",
564 escape_for_links_notation(result.identifier)
565 ),
566 format!(" uuid {}", escape_for_links_notation(result.uuid)),
567 format!(" mode {}", escape_for_links_notation(result.mode.as_str())),
568 format!(" backend {}", escape_for_links_notation(result.backend)),
569 format!(
570 " sessionName {}",
571 escape_for_links_notation(result.session_name)
572 ),
573 ];
574 if let Some(previous) = result.previous_session_name {
575 lines.push(format!(
576 " previousSessionName {}",
577 escape_for_links_notation(previous)
578 ));
579 }
580 if let Some(image) = result.snapshot_image {
581 lines.push(format!(
582 " snapshotImage {}",
583 escape_for_links_notation(image)
584 ));
585 }
586 if !result.resource_limits.is_empty() {
587 lines.push(format!(
588 " resourceLimits {}",
589 escape_for_links_notation(&result.resource_limits.join(" "))
590 ));
591 }
592 lines.push(format!(
593 " command {}",
594 escape_for_links_notation(result.command)
595 ));
596 lines.push(format!(
597 " message {}",
598 escape_for_links_notation(result.message)
599 ));
600 lines.join("\n")
601}
602
603pub fn format_resume_result(result: &ResumeResultFields, output_format: Option<&str>) -> String {
605 match output_format {
606 Some("json") => {
607 let mut value = json!({
608 "identifier": result.identifier,
609 "uuid": result.uuid,
610 "mode": result.mode.as_str(),
611 "backend": result.backend,
612 "sessionName": result.session_name,
613 "previousSessionName": result.previous_session_name,
614 "snapshotImage": result.snapshot_image,
615 "resourceLimits": (!result.resource_limits.is_empty())
616 .then_some(result.resource_limits),
617 "command": result.command,
618 "message": result.message,
619 });
620 if let Value::Object(ref mut map) = value {
621 map.retain(|_, entry| !entry.is_null());
622 }
623 serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{}".to_string())
624 }
625 Some("text") => {
626 let mut lines = vec![
627 format!("Resume Mode: {}", result.mode.as_str()),
628 format!("UUID: {}", result.uuid),
629 format!("Backend: {}", result.backend),
630 format!("Session Name: {}", result.session_name),
631 format!("Command: {}", result.command),
632 ];
633 if !result.resource_limits.is_empty() {
634 lines.push(format!(
635 "Resource Limits: {}",
636 result.resource_limits.join(" ")
637 ));
638 }
639 lines.push(result.message.to_string());
640 lines.join("\n")
641 }
642 _ => format_resume_result_as_links_notation(result),
643 }
644}
645
646pub fn apply_resume_to_record(
649 record: &mut ExecutionRecord,
650 plan: &ResumePlan,
651 container_id: Option<&str>,
652) {
653 let attempt = crate::execution_attempt::create_attempt(
654 record,
655 plan.mode.as_str(),
656 active_session_name(plan),
657 );
658 crate::execution_attempt::archive_attempt(record);
659 record.attempt = Some(attempt);
660 record
661 .options
662 .insert("resumeCount".to_string(), json!(plan.attempt));
663 record.options.insert(
664 "resumedAt".to_string(),
665 json!(record.attempt.as_ref().unwrap().started_at),
666 );
667 record.options.remove("stopRequestedAt");
669 record.options.remove("exitEvidence");
670 if !plan.resource_limits.is_empty() {
671 record
672 .options
673 .insert("resourceLimits".to_string(), json!(plan.resource_limits));
674 }
675
676 if !record.options.contains_key("rootSession") {
677 let root = record
678 .options
679 .get("sessionNameHistory")
680 .and_then(Value::as_array)
681 .and_then(|history| history.first())
682 .and_then(Value::as_str)
683 .or_else(|| record_option(record, "sessionName"))
684 .unwrap_or("session")
685 .to_string();
686 record.options.insert("rootSession".into(), json!(root));
687 }
688 if let Some(new_session_name) = &plan.new_session_name {
689 let mut history = record
690 .options
691 .get("sessionNameHistory")
692 .and_then(|value| value.as_array())
693 .cloned()
694 .unwrap_or_default();
695 if let Some(previous) = record.options.get("sessionName").cloned() {
696 history.push(previous);
697 }
698 record
699 .options
700 .insert("sessionNameHistory".to_string(), Value::Array(history));
701 record
702 .options
703 .insert("sessionName".to_string(), json!(new_session_name));
704 }
705 if let Some(snapshot_image) = &plan.snapshot_image {
706 record
707 .options
708 .insert("image".to_string(), json!(snapshot_image));
709 record
710 .options
711 .insert("snapshotImage".into(), json!(snapshot_image));
712 record.options.insert("commandHandoff".into(), json!(true));
713 }
714 if plan.mode == ResumeMode::Relaunch && plan.backend == "docker" {
715 record.options.insert("commandHandoff".into(), json!(true));
716 }
717 if let Some(container_id) = container_id {
718 record
719 .options
720 .insert("containerId".to_string(), json!(container_id));
721 }
722
723 record.command = plan.command.clone();
724 record.status = ExecutionStatus::Executing;
725 record.exit_code = None;
726 record.end_time = None;
727 record.exit_reason = None;
728 record.oom_killed = None;
729}
730
731fn active_session_name(plan: &ResumePlan) -> &str {
732 plan.new_session_name
733 .as_deref()
734 .unwrap_or(&plan.session_name)
735}
736
737#[path = "execution_resume_flow.rs"]
738mod resume_flow;
739pub use resume_flow::{
740 resume_execution, resume_execution_with, resume_execution_with_options,
741 resume_execution_with_resources,
742};