1use std::path::PathBuf;
19
20use chrono::Utc;
21use serde_json::{json, Value};
22
23use crate::docker_cleanup::{
24 build_docker_runtime_args, docker_command, docker_networks,
25 get_docker_container_cleanup_policy, start_detached_docker_completion_watcher_with,
26 DockerWatcherOptions,
27};
28use crate::docker_resource_limits::{build_resource_limits_status_line, normalize_resource_limits};
29use crate::execution_control::{CommandRunner, SystemCommandRunner};
30use crate::execution_store::{ExecutionRecord, ExecutionStatus, ExecutionStore};
31use crate::isolation::isolation_log::append_log_file;
32use crate::isolation::{run_isolated, IsolationOptions, IsolationResult};
33use crate::output_blocks::escape_for_links_notation;
34use crate::session_probe::{probe_session, SessionProbe, SessionState};
35
36#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38pub enum ResumeMode {
39 DockerStart,
40 DockerSnapshot,
41 Relaunch,
42}
43
44impl ResumeMode {
45 pub fn as_str(self) -> &'static str {
46 match self {
47 ResumeMode::DockerStart => "docker-start",
48 ResumeMode::DockerSnapshot => "docker-snapshot",
49 ResumeMode::Relaunch => "relaunch",
50 }
51 }
52}
53
54#[derive(Debug, Clone, PartialEq, Eq)]
56pub struct ResumeStep {
57 pub command: String,
58 pub args: Vec<String>,
59 pub description: String,
60}
61
62#[derive(Debug, Clone)]
64pub struct ResumePlan {
65 pub mode: ResumeMode,
66 pub backend: String,
67 pub session_name: String,
68 pub new_session_name: Option<String>,
70 pub snapshot_image: Option<String>,
71 pub command: String,
72 pub attempt: u64,
73 pub resource_limits: Vec<String>,
76 pub steps: Vec<ResumeStep>,
77 pub launch_options: Option<IsolationOptions>,
78 pub message: String,
79}
80
81pub struct ExecutionResumeResult {
83 pub success: bool,
84 pub output: Option<String>,
85 pub error: Option<String>,
86}
87
88pub trait ResumeHooks {
93 fn start_watcher(&self, session_name: &str, record: &ExecutionRecord);
95 fn attach_watcher(&self, session_name: &str, record: &ExecutionRecord) -> Result<(), String> {
97 self.start_watcher(session_name, record);
98 Ok(())
99 }
100
101 fn relaunch(&self, backend: &str, command: &str, options: &IsolationOptions)
103 -> IsolationResult;
104
105 fn reconcile(&self, record: &ExecutionRecord) -> ExecutionRecord;
107}
108
109#[derive(Debug, Default)]
111pub struct SystemResumeHooks;
112
113impl ResumeHooks for SystemResumeHooks {
114 fn start_watcher(&self, session_name: &str, record: &ExecutionRecord) {
115 let _ = self.attach_watcher(session_name, record);
116 }
117
118 fn attach_watcher(&self, session_name: &str, record: &ExecutionRecord) -> Result<(), String> {
119 let log_path = (!record.log_path.is_empty()).then(|| PathBuf::from(&record.log_path));
120 start_detached_docker_completion_watcher_with(
121 session_name,
122 get_docker_container_cleanup_policy(&build_launch_options(record)),
123 log_path.as_ref(),
124 Some(record.uuid.as_str()),
127 &DockerWatcherOptions {
128 since: record
129 .attempt
130 .as_ref()
131 .map(|attempt| attempt.started_at.clone()),
132 attempt_number: record.attempt.as_ref().map(|attempt| attempt.number),
133 recover_on_kill: keeps_kill_recovery(session_name, record),
134 },
135 )
136 }
137
138 fn relaunch(
139 &self,
140 backend: &str,
141 command: &str,
142 options: &IsolationOptions,
143 ) -> IsolationResult {
144 run_isolated(backend, command, options)
145 }
146
147 fn reconcile(&self, record: &ExecutionRecord) -> ExecutionRecord {
148 crate::status_formatter::enrich_detached_status(record)
149 }
150}
151
152pub fn keeps_kill_recovery(session_name: &str, record: &ExecutionRecord) -> bool {
156 record.attempt.as_ref().is_none_or(|attempt| {
157 matches!(
158 attempt.mode.as_str(),
159 "docker-start" | "automatic-recovery" | "relaunch"
160 )
161 }) && record_option(record, "sessionName") == Some(session_name)
162 && record
163 .options
164 .get("onKillResume")
165 .is_some_and(|value| value.as_u64().unwrap_or(0) > 0)
166}
167
168fn record_option<'a>(record: &'a ExecutionRecord, key: &str) -> Option<&'a str> {
169 record.options.get(key).and_then(|value| value.as_str())
170}
171
172fn record_flag(record: &ExecutionRecord, key: &str) -> bool {
173 record
174 .options
175 .get(key)
176 .and_then(|value| value.as_bool())
177 .unwrap_or(false)
178}
179
180fn record_strings(record: &ExecutionRecord, key: &str) -> Vec<String> {
181 record
182 .options
183 .get(key)
184 .and_then(|value| value.as_array())
185 .map(|items| {
186 items
187 .iter()
188 .filter_map(|item| item.as_str().map(str::to_string))
189 .collect()
190 })
191 .unwrap_or_default()
192}
193
194pub fn build_snapshot_image_name(session_name: &str, attempt: u64) -> String {
198 let mut sanitized = String::new();
199 let mut pending_dash = false;
200 for ch in session_name.to_lowercase().chars() {
201 if ch.is_ascii_lowercase() || ch.is_ascii_digit() || ch == '.' || ch == '_' || ch == '-' {
202 if pending_dash {
203 sanitized.push('-');
204 pending_dash = false;
205 }
206 sanitized.push(ch);
207 } else if !sanitized.is_empty() || !pending_dash {
208 pending_dash = true;
209 }
210 }
211 if pending_dash {
212 sanitized.push('-');
213 }
214 let sanitized = sanitized.trim_start_matches(['-', '.']).to_string();
215 let sanitized = if sanitized.is_empty() {
216 "session".to_string()
217 } else {
218 sanitized
219 };
220 format!("start-command-resume/{}:{}", sanitized, attempt)
221}
222
223pub fn build_resumed_session_name(session_name: &str, attempt: u64) -> String {
225 format!("{}-resume-{}", session_name, attempt)
226}
227
228pub fn build_launch_options(record: &ExecutionRecord) -> IsolationOptions {
231 let networks = record_strings(record, "networks");
232 IsolationOptions {
233 session: record_option(record, "sessionName").map(str::to_string),
234 labels: record_strings(record, "labels"),
235 uuid: Some(record.uuid.clone()),
236 root_session: Some(
237 record_option(record, "rootSession")
238 .or_else(|| {
239 record
240 .options
241 .get("sessionNameHistory")
242 .and_then(Value::as_array)
243 .and_then(|history| history.first())
244 .and_then(Value::as_str)
245 })
246 .unwrap_or_else(|| record_option(record, "sessionName").unwrap_or("session"))
247 .to_string(),
248 ),
249 resume_count: record
250 .options
251 .get("resumeCount")
252 .and_then(Value::as_u64)
253 .unwrap_or(0) as u32
254 + 1,
255 image: record_option(record, "image").map(str::to_string),
256 volumes: record_strings(record, "volumes"),
257 mounts: record_strings(record, "mounts"),
258 env: record_strings(record, "env"),
259 privileged: record_flag(record, "privileged"),
260 network: networks.first().cloned(),
261 networks,
262 network_aliases: record_strings(record, "networkAliases"),
263 resource_limits: normalize_resource_limits(record.options.get("resourceLimits")),
265 cpu_penalty_config: record
266 .options
267 .get("cpuPenaltyConfig")
268 .and_then(|v| serde_json::from_value(v.clone()).ok()),
269 on_kill_resume: record
270 .options
271 .get("onKillResume")
272 .and_then(Value::as_u64)
273 .map(|n| n as u32),
274 recovery_command: record_option(record, "recoveryCommand").map(str::to_string),
275 endpoint: record_option(record, "endpoint").map(str::to_string),
276 detached: true,
277 user: record_option(record, "user").map(str::to_string),
278 keep_alive: record_flag(record, "keepAlive"),
279 auto_remove_docker_container: record_flag(record, "autoRemoveDockerContainer"),
280 always_cleanup_container: record_flag(record, "alwaysCleanupContainer"),
281 keep_container: record_flag(record, "keepContainer"),
282 keep_container_on_fail: record_flag(record, "keepContainerOnFail"),
283 shell: record_option(record, "shell").unwrap_or("auto").to_string(),
284 log_path: (!record.log_path.is_empty()).then(|| PathBuf::from(&record.log_path)),
286 execution_id: Some(record.uuid.clone()),
289 defer_completion_watcher: true,
290 }
291}
292
293fn build_snapshot_start_steps(
301 docker: &str,
302 container_args: Vec<String>,
303 extra_networks: &[String],
304 new_session_name: &str,
305 snapshot_image: &str,
306) -> Vec<ResumeStep> {
307 if extra_networks.is_empty() {
308 let mut args = vec!["run".to_string(), "-d".to_string()];
309 args.extend(container_args);
310 return vec![ResumeStep {
311 command: docker.to_string(),
312 args,
313 description: format!("Run the new command in {}", new_session_name),
314 }];
315 }
316
317 let mut create_args = vec!["create".to_string()];
318 create_args.extend(container_args);
319 let mut steps = vec![ResumeStep {
320 command: docker.to_string(),
321 args: create_args,
322 description: format!("Create {} from {}", new_session_name, snapshot_image),
323 }];
324 for network in extra_networks {
325 steps.push(ResumeStep {
326 command: docker.to_string(),
327 args: vec![
328 "network".to_string(),
329 "connect".to_string(),
330 network.clone(),
331 new_session_name.to_string(),
332 ],
333 description: format!("Connect {} to network {}", new_session_name, network),
334 });
335 }
336 steps.push(ResumeStep {
337 command: docker.to_string(),
338 args: vec!["start".to_string(), new_session_name.to_string()],
339 description: format!("Run the new command in {}", new_session_name),
340 });
341 steps
342}
343
344fn docker_snapshot_plan(
345 record: &ExecutionRecord,
346 backend: &str,
347 session_name: &str,
348 command: &str,
349 attempt: u64,
350 live_resource_limits: Option<Vec<String>>,
351) -> ResumePlan {
352 let snapshot_image = build_snapshot_image_name(session_name, attempt);
353 let new_session_name = build_resumed_session_name(session_name, attempt);
354 let docker = docker_command().to_string_lossy().to_string();
355
356 let mut launch_options = build_launch_options(record);
357 launch_options.session = Some(new_session_name.clone());
358 launch_options.resume_count = attempt as u32;
359 launch_options.labels =
360 crate::isolation_metadata::docker_attribution_labels(&launch_options, &new_session_name);
361 if let Some(limits) = live_resource_limits {
364 launch_options.resource_limits = limits;
365 }
366 let mut container_args = vec!["--name".to_string(), new_session_name.clone()];
367 container_args.extend([
368 "--label".into(),
369 format!(
370 "{}={}",
371 crate::docker_snapshot_safety::SNAPSHOT_IMAGE_LABEL,
372 snapshot_image
373 ),
374 ]);
375 if let Some(user) = record_option(record, "user") {
376 container_args.push("--user".to_string());
377 container_args.push(user.to_string());
378 }
379 container_args.extend(
380 build_docker_runtime_args(&launch_options)
381 .into_iter()
382 .map(str::to_string),
383 );
384 container_args.push(snapshot_image.clone());
385 container_args.extend(crate::docker_command_handoff::build_command_handoff_args(
386 &["sh".into(), "-c".into(), command.into()],
387 &crate::docker_command_handoff::command_handoff_path(&new_session_name),
388 ));
389
390 let mut steps = vec![ResumeStep {
391 command: docker.clone(),
392 args: vec![
393 "commit".to_string(),
394 session_name.to_string(),
395 snapshot_image.clone(),
396 ],
397 description: format!("Snapshot container {} as {}", session_name, snapshot_image),
398 }];
399 steps.extend(build_snapshot_start_steps(
400 &docker,
401 container_args,
402 &docker_networks(&launch_options)
403 .into_iter()
404 .skip(1)
405 .map(str::to_string)
406 .collect::<Vec<_>>(),
407 &new_session_name,
408 &snapshot_image,
409 ));
410
411 ResumePlan {
412 mode: ResumeMode::DockerSnapshot,
413 backend: backend.to_string(),
414 session_name: session_name.to_string(),
415 new_session_name: Some(new_session_name.clone()),
416 snapshot_image: Some(snapshot_image.clone()),
417 command: command.to_string(),
418 attempt,
419 resource_limits: launch_options.resource_limits.clone(),
420 steps,
421 launch_options: None,
422 message: format!(
423 "Resumed session in new container {} from snapshot of {}",
424 new_session_name, session_name
425 ),
426 }
427}
428
429pub fn build_resume_plan(
431 record: &ExecutionRecord,
432 new_command: Option<&str>,
433 probe: &SessionProbe,
434) -> Result<ResumePlan, String> {
435 build_resume_plan_with_limits(record, new_command, probe, None)
436}
437
438pub fn build_resume_plan_with_limits(
441 record: &ExecutionRecord,
442 new_command: Option<&str>,
443 probe: &SessionProbe,
444 live_resource_limits: Option<Vec<String>>,
445) -> Result<ResumePlan, String> {
446 let session_name = record_option(record, "sessionName")
447 .ok_or_else(|| "Execution record does not contain an isolation session name.".to_string())?
448 .to_string();
449
450 if record_option(record, "isolationMode") != Some("detached") {
451 return Err("Only detached isolated executions can be resumed.".to_string());
452 }
453
454 if probe.alive {
455 return Err(format!(
456 "Session \"{}\" is still running. Use `$ --attach {}` to re-enter it, or `$ --stop {}` first.",
457 session_name, record.uuid, record.uuid
458 ));
459 }
460
461 let backend = record_option(record, "isolated")
462 .unwrap_or("unknown")
463 .to_string();
464 let command = new_command.unwrap_or(record.command.as_str()).to_string();
465 if command.is_empty() {
466 return Err(format!(
467 "Execution \"{}\" has no stored command to resume.",
468 record.uuid
469 ));
470 }
471
472 let attempt = record
473 .options
474 .get("resumeCount")
475 .and_then(|value| value.as_u64())
476 .unwrap_or(0)
477 + 1;
478
479 if backend == "docker" && probe.state == SessionState::Stopped {
480 return Ok(match new_command {
481 None | Some(_)
482 if !record_flag(record, "forceSnapshot")
483 && (new_command.is_none() || record_flag(record, "commandHandoff")) =>
484 {
485 ResumePlan {
486 mode: ResumeMode::DockerStart,
487 backend,
488 new_session_name: None,
489 snapshot_image: None,
490 command,
491 attempt,
492 resource_limits: Vec::new(),
493 steps: vec![ResumeStep {
494 command: docker_command().to_string_lossy().to_string(),
495 args: vec!["start".to_string(), session_name.clone()],
496 description: format!("Start stopped container {}", session_name),
497 }],
498 launch_options: None,
499 message: if new_command.is_some() {
500 format!("Resumed new command in the same detached docker container: {} (no filesystem snapshot)", session_name)
501 } else {
502 format!("Resumed detached docker container: {}", session_name)
503 },
504 session_name,
505 }
506 }
507 Some(new_command) => docker_snapshot_plan(
508 record,
509 &backend,
510 &session_name,
511 new_command,
512 attempt,
513 live_resource_limits,
514 ),
515 None => docker_snapshot_plan(
516 record,
517 &backend,
518 &session_name,
519 &command,
520 attempt,
521 live_resource_limits,
522 ),
523 });
524 }
525
526 Ok(ResumePlan {
527 mode: ResumeMode::Relaunch,
528 message: format!("Relaunched {} session: {}", backend, session_name),
529 backend,
530 new_session_name: None,
531 snapshot_image: None,
532 command,
533 attempt,
534 resource_limits: Vec::new(),
535 steps: Vec::new(),
536 launch_options: Some(build_launch_options(record)),
537 session_name,
538 })
539}
540
541pub struct ResumeResultFields<'a> {
543 pub identifier: &'a str,
544 pub uuid: &'a str,
545 pub mode: ResumeMode,
546 pub backend: &'a str,
547 pub session_name: &'a str,
548 pub previous_session_name: Option<&'a str>,
549 pub snapshot_image: Option<&'a str>,
550 pub resource_limits: &'a [String],
552 pub command: &'a str,
553 pub message: &'a str,
554}
555
556pub fn format_resume_result_as_links_notation(result: &ResumeResultFields) -> String {
558 let mut lines = vec![
559 "executionResume".to_string(),
560 format!(
561 " identifier {}",
562 escape_for_links_notation(result.identifier)
563 ),
564 format!(" uuid {}", escape_for_links_notation(result.uuid)),
565 format!(" mode {}", escape_for_links_notation(result.mode.as_str())),
566 format!(" backend {}", escape_for_links_notation(result.backend)),
567 format!(
568 " sessionName {}",
569 escape_for_links_notation(result.session_name)
570 ),
571 ];
572 if let Some(previous) = result.previous_session_name {
573 lines.push(format!(
574 " previousSessionName {}",
575 escape_for_links_notation(previous)
576 ));
577 }
578 if let Some(image) = result.snapshot_image {
579 lines.push(format!(
580 " snapshotImage {}",
581 escape_for_links_notation(image)
582 ));
583 }
584 if !result.resource_limits.is_empty() {
585 lines.push(format!(
586 " resourceLimits {}",
587 escape_for_links_notation(&result.resource_limits.join(" "))
588 ));
589 }
590 lines.push(format!(
591 " command {}",
592 escape_for_links_notation(result.command)
593 ));
594 lines.push(format!(
595 " message {}",
596 escape_for_links_notation(result.message)
597 ));
598 lines.join("\n")
599}
600
601pub fn format_resume_result(result: &ResumeResultFields, output_format: Option<&str>) -> String {
603 match output_format {
604 Some("json") => {
605 let mut value = json!({
606 "identifier": result.identifier,
607 "uuid": result.uuid,
608 "mode": result.mode.as_str(),
609 "backend": result.backend,
610 "sessionName": result.session_name,
611 "previousSessionName": result.previous_session_name,
612 "snapshotImage": result.snapshot_image,
613 "resourceLimits": (!result.resource_limits.is_empty())
614 .then_some(result.resource_limits),
615 "command": result.command,
616 "message": result.message,
617 });
618 if let Value::Object(ref mut map) = value {
619 map.retain(|_, entry| !entry.is_null());
620 }
621 serde_json::to_string_pretty(&value).unwrap_or_else(|_| "{}".to_string())
622 }
623 Some("text") => {
624 let mut lines = vec![
625 format!("Resume Mode: {}", result.mode.as_str()),
626 format!("UUID: {}", result.uuid),
627 format!("Backend: {}", result.backend),
628 format!("Session Name: {}", result.session_name),
629 format!("Command: {}", result.command),
630 ];
631 if !result.resource_limits.is_empty() {
632 lines.push(format!(
633 "Resource Limits: {}",
634 result.resource_limits.join(" ")
635 ));
636 }
637 lines.push(result.message.to_string());
638 lines.join("\n")
639 }
640 _ => format_resume_result_as_links_notation(result),
641 }
642}
643
644pub fn apply_resume_to_record(
647 record: &mut ExecutionRecord,
648 plan: &ResumePlan,
649 container_id: Option<&str>,
650) {
651 let attempt = crate::execution_attempt::create_attempt(
652 record,
653 plan.mode.as_str(),
654 active_session_name(plan),
655 );
656 crate::execution_attempt::archive_attempt(record);
657 record.attempt = Some(attempt);
658 record
659 .options
660 .insert("resumeCount".to_string(), json!(plan.attempt));
661 record.options.insert(
662 "resumedAt".to_string(),
663 json!(record.attempt.as_ref().unwrap().started_at),
664 );
665 record.options.remove("stopRequestedAt");
667 record.options.remove("exitEvidence");
668 if !plan.resource_limits.is_empty() {
669 record
670 .options
671 .insert("resourceLimits".to_string(), json!(plan.resource_limits));
672 }
673
674 if !record.options.contains_key("rootSession") {
675 let root = record
676 .options
677 .get("sessionNameHistory")
678 .and_then(Value::as_array)
679 .and_then(|history| history.first())
680 .and_then(Value::as_str)
681 .or_else(|| record_option(record, "sessionName"))
682 .unwrap_or("session")
683 .to_string();
684 record.options.insert("rootSession".into(), json!(root));
685 }
686 if let Some(new_session_name) = &plan.new_session_name {
687 let mut history = record
688 .options
689 .get("sessionNameHistory")
690 .and_then(|value| value.as_array())
691 .cloned()
692 .unwrap_or_default();
693 if let Some(previous) = record.options.get("sessionName").cloned() {
694 history.push(previous);
695 }
696 record
697 .options
698 .insert("sessionNameHistory".to_string(), Value::Array(history));
699 record
700 .options
701 .insert("sessionName".to_string(), json!(new_session_name));
702 }
703 if let Some(snapshot_image) = &plan.snapshot_image {
704 record
705 .options
706 .insert("image".to_string(), json!(snapshot_image));
707 record
708 .options
709 .insert("snapshotImage".into(), json!(snapshot_image));
710 record.options.insert("commandHandoff".into(), json!(true));
711 }
712 if plan.mode == ResumeMode::Relaunch && plan.backend == "docker" {
713 record.options.insert("commandHandoff".into(), json!(true));
714 }
715 if let Some(container_id) = container_id {
716 record
717 .options
718 .insert("containerId".to_string(), json!(container_id));
719 }
720
721 record.command = plan.command.clone();
722 record.status = ExecutionStatus::Executing;
723 record.exit_code = None;
724 record.end_time = None;
725 record.exit_reason = None;
726 record.oom_killed = None;
727}
728
729fn active_session_name(plan: &ResumePlan) -> &str {
730 plan.new_session_name
731 .as_deref()
732 .unwrap_or(&plan.session_name)
733}
734
735#[path = "execution_resume_flow.rs"]
736mod resume_flow;
737pub use resume_flow::{
738 resume_execution, resume_execution_with, resume_execution_with_options,
739 resume_execution_with_resources,
740};