1use crate::snapshot::{RestoreReport, SnapshotReport};
10use serde::Serialize;
11use std::time::Duration;
12
13#[derive(Debug, Clone, Copy, PartialEq, Eq)]
15pub struct ContinuityPolicy {
16 pub min_interval: Duration,
19}
20
21impl Default for ContinuityPolicy {
22 fn default() -> Self {
23 Self {
24 min_interval: Duration::ZERO,
25 }
26 }
27}
28
29#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
31pub struct LostName {
32 pub name: String,
33 pub reason: String,
34}
35
36#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
38#[serde(rename_all = "camelCase")]
39pub struct ContinuityReport {
40 pub reason: String,
42 pub snapshot_path: String,
43 pub restored: Vec<String>,
45 pub lost: Vec<LostName>,
47 pub interrupted_cell: bool,
49 pub unsaved_cells: u64,
51 #[serde(skip_serializing_if = "Option::is_none")]
54 pub restore_error: Option<String>,
55}
56
57#[derive(Debug, Clone)]
59pub(crate) struct Crash {
60 pub reason: String,
61 pub interrupted_cell: bool,
62 pub unsaved_cells: u64,
63 pub restore_error: Option<String>,
64}
65
66fn lost_from(names: &[String], reason: impl Fn(&str) -> Option<String>) -> Vec<LostName> {
67 names
68 .iter()
69 .map(|name| LostName {
70 name: name.clone(),
71 reason: reason(name).unwrap_or_else(|| "not restorable".into()),
72 })
73 .collect()
74}
75
76pub(crate) fn recovery_report(
79 crash: Crash,
80 snapshot_path: String,
81 restore: Result<Option<RestoreReport>, String>,
82 checkpoint: Option<&SnapshotReport>,
83) -> ContinuityReport {
84 let all_checkpoint_names = |why: &str| -> Vec<LostName> {
85 let Some(checkpoint) = checkpoint else {
86 return Vec::new();
87 };
88 let mut lost = lost_from(&checkpoint.saved, |_| Some(why.to_string()));
89 lost.extend(lost_from(&checkpoint.skipped, |name| {
90 checkpoint.reason(name).map(String::from)
91 }));
92 lost
93 };
94 let (restored, mut lost, restore_error) = match restore {
95 Ok(Some(report)) if !report.missing => {
96 let mut lost = lost_from(&report.skipped, |name| report.reason(name).map(String::from));
97 if let Some(checkpoint) = checkpoint {
99 for name in &checkpoint.saved {
100 if !report.restored.contains(name) && !report.skipped.contains(name) {
101 lost.push(LostName {
102 name: name.clone(),
103 reason: "absent from the restored snapshot".into(),
104 });
105 }
106 }
107 }
108 (report.restored, lost, crash.restore_error)
109 }
110 Ok(_) => (
111 Vec::new(),
112 all_checkpoint_names("snapshot file is missing"),
113 crash.restore_error,
114 ),
115 Err(error) => (
116 Vec::new(),
117 all_checkpoint_names(&format!("snapshot not restored: {error}")),
118 Some(error),
119 ),
120 };
121 lost.sort_by(|left, right| left.name.cmp(&right.name));
122 lost.dedup_by(|left, right| left.name == right.name);
123 ContinuityReport {
124 reason: crash.reason,
125 snapshot_path,
126 restored,
127 lost,
128 interrupted_cell: crash.interrupted_cell,
129 unsaved_cells: crash.unsaved_cells,
130 restore_error,
131 }
132}
133
134impl ContinuityReport {
135 pub fn render(&self) -> String {
137 let mut text = format!("[RLM kernel restarted after: {}]\n", self.reason);
138 match &self.restore_error {
139 Some(error) => text.push_str(&format!(
140 "[State not restored ({error}); the namespace started empty]\n"
141 )),
142 None if self.restored.is_empty() => text.push_str("[No state restored]\n"),
143 None => text.push_str(&format!("[Restored: {}]\n", self.restored.join(", "))),
144 }
145 if !self.lost.is_empty() {
146 let lost: Vec<String> = self
147 .lost
148 .iter()
149 .map(|lost| format!("{} ({})", lost.name, lost.reason))
150 .collect();
151 text.push_str(&format!("[Lost: {}]\n", lost.join("; ")));
152 }
153 if self.interrupted_cell {
154 text.push_str("[The cell running at the failure did not finish; its changes to Python state are lost]\n");
155 }
156 if self.unsaved_cells > 0 {
157 text.push_str(&format!(
158 "[{} completed cell(s) after the last checkpoint are lost; restored values predate them]\n",
159 self.unsaved_cells
160 ));
161 }
162 text
163 }
164}