Skip to main content

kernel/
continuity.rs

1//! Kernel state continuity (Rust-only; kernel.ts has none): the REPL's state survives a kernel crash or restart.
2//!
3//! With a [`ContinuityPolicy`] the supervisor snapshots the namespace after every cell (debounced by
4//! `min_interval`, with a flush once idle), so a crash loses at most the cell that was running. When the kernel
5//! dies it is restarted at once and restored from the latest checkpoint, and the next cell result reports exactly
6//! which names came back and which were lost, with the reason for each (A08: state that cannot be restored is
7//! reported, never silently replaced).
8
9use crate::snapshot::{RestoreReport, SnapshotReport};
10use serde::Serialize;
11use std::time::Duration;
12
13/// How the supervisor keeps the REPL state recoverable. Requires `KernelOptions::snapshot_path`.
14#[derive(Debug, Clone, Copy, PartialEq, Eq)]
15pub struct ContinuityPolicy {
16    /// Least time between two checkpoints. Cells finishing sooner are coalesced into one snapshot taken once the
17    /// interval has passed (the idle flush). Zero snapshots after every cell.
18    pub min_interval: Duration,
19}
20
21impl Default for ContinuityPolicy {
22    fn default() -> Self {
23        Self {
24            min_interval: Duration::ZERO,
25        }
26    }
27}
28
29/// A name that did not come back, and why.
30#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
31pub struct LostName {
32    pub name: String,
33    pub reason: String,
34}
35
36/// What a recovery restored and lost. Attached to the first cell result after the kernel restarted.
37#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
38#[serde(rename_all = "camelCase")]
39pub struct ContinuityReport {
40    /// Why the kernel restarted: the failure the previous kernel ended with.
41    pub reason: String,
42    pub snapshot_path: String,
43    /// Names restored from the checkpoint, sorted.
44    pub restored: Vec<String>,
45    /// Names the checkpoint knew that did not come back, sorted by name.
46    pub lost: Vec<LostName>,
47    /// A cell was running when the kernel died; whatever it changed is gone.
48    pub interrupted_cell: bool,
49    /// Cells that completed after the checkpoint; their changes are gone too (non-zero only with a debounce).
50    pub unsaved_cells: u64,
51    /// Set when the checkpoint could not be restored (refused, unreadable, or the kernel died restoring it); the
52    /// kernel then starts with an empty namespace.
53    #[serde(skip_serializing_if = "Option::is_none")]
54    pub restore_error: Option<String>,
55}
56
57/// The failure a recovery reports, recorded when the kernel dies.
58#[derive(Debug, Clone)]
59pub(crate) struct Crash {
60    pub reason: String,
61    pub interrupted_cell: bool,
62    pub unsaved_cells: u64,
63    pub restore_error: Option<String>,
64}
65
66fn lost_from(names: &[String], reason: impl Fn(&str) -> Option<String>) -> Vec<LostName> {
67    names
68        .iter()
69        .map(|name| LostName {
70            name: name.clone(),
71            reason: reason(name).unwrap_or_else(|| "not restorable".into()),
72        })
73        .collect()
74}
75
76/// What a recovery restored and lost. `restore` is the kernel's report, or why restoring failed; `checkpoint` is the
77/// latest snapshot this supervisor wrote, which names what existed when restoring is impossible.
78pub(crate) fn recovery_report(
79    crash: Crash,
80    snapshot_path: String,
81    restore: Result<Option<RestoreReport>, String>,
82    checkpoint: Option<&SnapshotReport>,
83) -> ContinuityReport {
84    let all_checkpoint_names = |why: &str| -> Vec<LostName> {
85        let Some(checkpoint) = checkpoint else {
86            return Vec::new();
87        };
88        let mut lost = lost_from(&checkpoint.saved, |_| Some(why.to_string()));
89        lost.extend(lost_from(&checkpoint.skipped, |name| {
90            checkpoint.reason(name).map(String::from)
91        }));
92        lost
93    };
94    let (restored, mut lost, restore_error) = match restore {
95        Ok(Some(report)) if !report.missing => {
96            let mut lost = lost_from(&report.skipped, |name| report.reason(name).map(String::from));
97            // A name the checkpoint saved but the restore did not bring back (another writer replaced the file).
98            if let Some(checkpoint) = checkpoint {
99                for name in &checkpoint.saved {
100                    if !report.restored.contains(name) && !report.skipped.contains(name) {
101                        lost.push(LostName {
102                            name: name.clone(),
103                            reason: "absent from the restored snapshot".into(),
104                        });
105                    }
106                }
107            }
108            (report.restored, lost, crash.restore_error)
109        }
110        Ok(_) => (
111            Vec::new(),
112            all_checkpoint_names("snapshot file is missing"),
113            crash.restore_error,
114        ),
115        Err(error) => (
116            Vec::new(),
117            all_checkpoint_names(&format!("snapshot not restored: {error}")),
118            Some(error),
119        ),
120    };
121    lost.sort_by(|left, right| left.name.cmp(&right.name));
122    lost.dedup_by(|left, right| left.name == right.name);
123    ContinuityReport {
124        reason: crash.reason,
125        snapshot_path,
126        restored,
127        lost,
128        interrupted_cell: crash.interrupted_cell,
129        unsaved_cells: crash.unsaved_cells,
130        restore_error,
131    }
132}
133
134impl ContinuityReport {
135    /// The notice the model reads at the start of the cell's stderr.
136    pub fn render(&self) -> String {
137        let mut text = format!("[RLM kernel restarted after: {}]\n", self.reason);
138        match &self.restore_error {
139            Some(error) => text.push_str(&format!(
140                "[State not restored ({error}); the namespace started empty]\n"
141            )),
142            None if self.restored.is_empty() => text.push_str("[No state restored]\n"),
143            None => text.push_str(&format!("[Restored: {}]\n", self.restored.join(", "))),
144        }
145        if !self.lost.is_empty() {
146            let lost: Vec<String> = self
147                .lost
148                .iter()
149                .map(|lost| format!("{} ({})", lost.name, lost.reason))
150                .collect();
151            text.push_str(&format!("[Lost: {}]\n", lost.join("; ")));
152        }
153        if self.interrupted_cell {
154            text.push_str("[The cell running at the failure did not finish; its changes to Python state are lost]\n");
155        }
156        if self.unsaved_cells > 0 {
157            text.push_str(&format!(
158                "[{} completed cell(s) after the last checkpoint are lost; restored values predate them]\n",
159                self.unsaved_cells
160            ));
161        }
162        text
163    }
164}