Skip to main content

kernel/
lib.rs

1//! Supervisor for Ultron's persistent Python kernel: a port of the host side of
2//! `packages/coding-agent/src/ultron/rlm/kernel.ts`. The Python side is the unchanged `runtime.py`.
3//!
4//! The kernel runs `python runtime.py --protocol-fds` in its own process group. Protocol frames (one JSON object per
5//! line, at most 1 MiB) arrive on the kernel's fd 3 and host messages go to its fd 4; stdin is /dev/null, and fd 1 and
6//! fd 2 are plain output attributed to the running cell. After a cell, runtime.py writes a flush marker naming the
7//! cell's request id to fd 1 and fd 2 and then sends `done`, listing the markers it wrote; the cell resolves once the
8//! host has read them, so the cell's raw output is complete.
9//!
10//! Calls block; cells run one at a time in call order. A [`Signal`] cancels a call: a queued call is rejected without
11//! touching the kernel, a running one kills the kernel's process tree. Any kernel failure (exit, protocol violation,
12//! startup timeout, CPU backstop) rejects every pending call with an explicit error and loses the Python state; the
13//! next call starts a fresh kernel. Host requests from Python go to a [`HostHandler`], each on its own thread.
14//!
15//! Snapshots (kernel.ts `snapshot`/`restore`): runtime.py writes a data-only snapshot of the namespace; the host
16//! signs it with an HMAC key the kernel never sees and verifies the signature before the kernel may read it back.
17//! With `snapshot_path` a new kernel restores it before its first call; once a started kernel failed or a running
18//! call was cancelled it does not (kernel.ts never silently resurrects an older checkpoint). A [`ContinuityPolicy`]
19//! replaces that rule: checkpoints follow every cell, a dead kernel is restarted and restored at once, and the next
20//! cell result reports what was restored and lost (see [`continuity`]).
21
22mod config;
23pub mod continuity;
24pub mod env;
25pub mod ops;
26mod output;
27mod process;
28mod sandboxed;
29pub mod snapshot;
30pub mod snapshot_store;
31pub mod store;
32pub mod tree_memory;
33
34pub use continuity::{ContinuityPolicy, ContinuityReport, LostName};
35pub use process::kill_command_group;
36pub use snapshot::{
37    RestoreReport, SnapshotReport, load_or_create_snapshot_key, process_snapshot_key, sign_snapshot, verify_snapshot,
38};
39pub use tree_memory::TreeMemoryBackend;
40
41pub use config::{KernelResourceLimits, LimitOverrides, kernel_environment, kernel_resource_limits, rlm_output_budget};
42
43/// The root agent's lane; every other lane is a child's (the session journal's `session_journal::run::MAIN_LANE`).
44pub const ROOT_LANE: &str = "main";
45
46/// Whether `viewer`'s kernel sees work owned by `owner`: the root lane sees every lane's, a child lane its own.
47pub fn sees(viewer: &str, owner: &str) -> bool {
48    viewer == ROOT_LANE || viewer == owner
49}
50
51use command_fds::{CommandFdExt, FdMapping};
52use continuity::{Crash, recovery_report};
53use output::{
54    MAX_FRAME_BYTES, MAX_ORPHAN_CHARS, RawCapture, Utf8Stream, append_bounded, find, flush_marker, keep_tail,
55    marker_prefix_len, orphan_text,
56};
57use rustix::process::{WaitId, WaitIdOptions};
58use serde::Serialize;
59use serde_json::{Map, Value};
60use std::collections::{HashMap, VecDeque};
61use std::io::{Read, Write};
62use std::os::unix::fs::OpenOptionsExt;
63use std::os::unix::process::{CommandExt, ExitStatusExt};
64use std::path::{Path, PathBuf};
65use std::process::{Command, ExitStatus, Stdio};
66use std::sync::mpsc;
67use std::sync::{Arc, Condvar, Mutex, MutexGuard, PoisonError, Weak};
68use std::time::{Duration, Instant};
69
70/// A failed call: the kernel failed, was shut down, the call was cancelled, or its request could not be sent.
71#[derive(Debug, Clone, PartialEq, Eq)]
72pub struct KernelError {
73    message: String,
74}
75
76impl KernelError {
77    pub fn new(message: impl Into<String>) -> Self {
78        Self {
79            message: message.into(),
80        }
81    }
82
83    pub fn message(&self) -> &str {
84        &self.message
85    }
86}
87
88impl std::fmt::Display for KernelError {
89    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
90        formatter.write_str(&self.message)
91    }
92}
93
94impl std::error::Error for KernelError {}
95
96#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
97#[serde(rename_all = "lowercase")]
98pub enum Status {
99    Ok,
100    Error,
101}
102
103#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
104pub struct CellError {
105    pub ename: String,
106    pub evalue: String,
107    pub traceback: Vec<String>,
108}
109
110/// What a cell (or another request) produced. Serializes as kernel.ts `KernelExecutionResult`.
111#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
112pub struct ExecutionResult {
113    pub status: Status,
114    pub stdout: String,
115    pub stderr: String,
116    #[serde(skip_serializing_if = "Option::is_none")]
117    pub result: Option<String>,
118    #[serde(skip_serializing_if = "Option::is_none")]
119    pub error: Option<CellError>,
120}
121
122/// A call's full reply: the result plus the snapshot or restore report a done frame carried (kernel.ts
123/// `KernelExecutionResult` with `snapshot?` / `restore?`), and, on the first cell after a recovery, what it
124/// restored and lost.
125#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
126pub struct KernelReply {
127    #[serde(flatten)]
128    pub result: ExecutionResult,
129    #[serde(skip_serializing_if = "Option::is_none")]
130    pub snapshot: Option<SnapshotReport>,
131    #[serde(skip_serializing_if = "Option::is_none")]
132    pub restore: Option<RestoreReport>,
133    #[serde(skip_serializing_if = "Option::is_none")]
134    pub recovery: Option<ContinuityReport>,
135}
136
137impl KernelReply {
138    fn from_result(result: ExecutionResult) -> Self {
139        Self {
140            result,
141            snapshot: None,
142            restore: None,
143            recovery: None,
144        }
145    }
146}
147
148type Listener = Box<dyn FnOnce(String) + Send>;
149
150#[derive(Default)]
151struct SignalState {
152    reason: Option<String>,
153    listeners: Vec<(u64, Listener)>,
154    next: u64,
155}
156
157/// Cancellation, like an `AbortSignal`: aborted once, with a reason. A call's signal cancels it; the signal a
158/// [`HostHandler`] receives is aborted when its cell completes or fails.
159#[derive(Clone, Default)]
160pub struct Signal {
161    inner: Arc<(Mutex<SignalState>, Condvar)>,
162}
163
164impl Signal {
165    pub fn new() -> Self {
166        Self::default()
167    }
168
169    /// Aborts with `reason`; later calls do nothing.
170    pub fn abort(&self, reason: impl Into<String>) {
171        let reason = reason.into();
172        let listeners = {
173            let mut state = jsrt::sync::lock(&self.inner.0);
174            if state.reason.is_some() {
175                return;
176            }
177            state.reason = Some(reason.clone());
178            std::mem::take(&mut state.listeners)
179        };
180        self.inner.1.notify_all();
181        for (_, listener) in listeners {
182            listener(reason.clone());
183        }
184    }
185
186    pub fn reason(&self) -> Option<String> {
187        jsrt::sync::lock(&self.inner.0).reason.clone()
188    }
189
190    pub fn is_aborted(&self) -> bool {
191        jsrt::sync::lock(&self.inner.0).reason.is_some()
192    }
193
194    /// Blocks until aborted or `timeout` passes; whether it is aborted.
195    pub fn wait(&self, timeout: Duration) -> bool {
196        let state = jsrt::sync::lock(&self.inner.0);
197        let (state, _) = self
198            .inner
199            .1
200            .wait_timeout_while(state, timeout, |state| state.reason.is_none())
201            .unwrap_or_else(PoisonError::into_inner);
202        state.reason.is_some()
203    }
204
205    /// Runs `listener` once, on abort (on the aborting thread); None (and `listener` is dropped) when already
206    /// aborted. [`Signal::forget`] removes it.
207    pub fn on_abort(&self, listener: impl FnOnce(String) + Send + 'static) -> Option<u64> {
208        self.listen(Box::new(listener)).ok()
209    }
210
211    pub fn forget(&self, id: u64) {
212        jsrt::sync::lock(&self.inner.0)
213            .listeners
214            .retain(|(listener, _)| *listener != id);
215    }
216
217    /// Runs `listener` once on abort, or at once (on this thread) with the reason when already aborted, until the
218    /// returned guard drops.
219    pub fn link(&self, listener: impl FnOnce(String) + Send + 'static) -> Link {
220        let id = match self.listen(Box::new(listener)) {
221            Ok(id) => Some(id),
222            Err((listener, reason)) => {
223                listener(reason);
224                None
225            }
226        };
227        Link {
228            signal: self.clone(),
229            id,
230        }
231    }
232
233    /// Adds `listener`; it comes back with the reason when already aborted.
234    fn listen(&self, listener: Listener) -> Result<u64, (Listener, String)> {
235        let mut state = jsrt::sync::lock(&self.inner.0);
236        if let Some(reason) = state.reason.clone() {
237            return Err((listener, reason));
238        }
239        state.next += 1;
240        let id = state.next;
241        state.listeners.push((id, listener));
242        Ok(id)
243    }
244
245    /// [`signals::child::run`], cancelled when this signal aborts (at once when it already has).
246    pub fn run_child(
247        &self,
248        command: &mut std::process::Command,
249        input: Option<&[u8]>,
250        bound: signals::child::Bound,
251    ) -> std::io::Result<signals::child::Output> {
252        let running = signals::child::start(command, input, bound)?;
253        let watched = running.watched().clone();
254        let _link = self.link(move |_| watched.cancel());
255        Ok(running.finish())
256    }
257}
258
259/// A [`Signal::link`] listener, removed when this drops.
260pub struct Link {
261    signal: Signal,
262    id: Option<u64>,
263}
264
265impl Drop for Link {
266    fn drop(&mut self) {
267        if let Some(id) = self.id {
268            self.signal.forget(id);
269        }
270    }
271}
272
273/// Serves host requests from Python (`await rlm.host_request(type, payload)` and every API built on it). Runs on its
274/// own thread per request; `signal` is aborted when the requesting cell completes, Python cancels the request, or the
275/// kernel fails. The outcome is still sent while the kernel lives (Python drops one nobody awaits). `Err` becomes a
276/// `RuntimeError` with that message in Python.
277pub trait HostHandler: Send + Sync {
278    fn handle(&self, kind: &str, payload: Map<String, Value>, signal: &Signal) -> Result<Value, String>;
279}
280
281impl<F> HostHandler for F
282where
283    F: Fn(&str, Map<String, Value>, &Signal) -> Result<Value, String> + Send + Sync,
284{
285    fn handle(&self, kind: &str, payload: Map<String, Value>, signal: &Signal) -> Result<Value, String> {
286        self(kind, payload, signal)
287    }
288}
289
290/// Test handler: replies `{"type": kind, "payload": payload}`, and refuses `test.fail` with "refused test.fail".
291pub struct EchoHandler;
292
293impl HostHandler for EchoHandler {
294    fn handle(&self, kind: &str, payload: Map<String, Value>, _signal: &Signal) -> Result<Value, String> {
295        if kind == "test.fail" {
296            return Err(format!("refused {kind}"));
297        }
298        let mut reply = Map::new();
299        reply.insert("type".into(), Value::String(kind.into()));
300        reply.insert("payload".into(), Value::Object(payload));
301        Ok(Value::Object(reply))
302    }
303}
304
305pub struct KernelOptions {
306    pub cwd: PathBuf,
307    pub runtime_path: PathBuf,
308    /// Interpreter; default `ULTRON_PYTHON`, else /usr/bin/python3 when it exists, else `python3`.
309    pub python: Option<PathBuf>,
310    /// Bounds initialization, not execution.
311    pub startup_timeout: Duration,
312    /// Overrides `ULTRON_RLM_MAX_MEMORY_MB` / `ULTRON_RLM_MAX_CPU_SECONDS`; 0 disables a limit.
313    pub limits: LimitOverrides,
314    /// Extra environment for the kernel, applied after the credential filter.
315    pub env: Vec<(String, String)>,
316    /// The default path of [`Kernel::snapshot`] / [`Kernel::restore`]; a new kernel restores it before its first
317    /// call (kernel.ts `snapshotPath`).
318    pub snapshot_path: Option<PathBuf>,
319    /// HMAC key that signs and verifies snapshots in the host; the kernel never receives it. Default: a key of this
320    /// process, so snapshots then restore only within it.
321    pub snapshot_key: Option<Vec<u8>>,
322    /// Checkpoint after cells and recover from kernel death; requires `snapshot_path`.
323    pub continuity: Option<ContinuityPolicy>,
324    /// Start runtime.py inside this launcher's sandbox (the session's bash policy, plus the snapshot and spill
325    /// directories writable and the interpreter readable), so a cell's direct `open()` is contained too. Docker
326    /// cannot carry the protocol descriptors: with it (or nothing working) `auto` runs unsandboxed and `required`
327    /// refuses to start. None: unsandboxed.
328    pub sandbox: Option<Arc<sandbox::Launcher>>,
329    /// The whole-tree memory cap (kernel.ts `maxTreeMemoryMb` / `treeMemoryBackend`); None: no tree cap.
330    pub tree_memory: Option<TreeMemoryOptions>,
331    /// The environment the kernel starts from (before the credential filter) and where its `ULTRON_*` settings
332    /// (`ULTRON_PYTHON`, limits, output budget, tree memory) are read: a session's own environment, so the kernel
333    /// never sees the server process's PATH or variables. None: this process's environment.
334    pub base_env: Option<Vec<(String, String)>>,
335}
336
337/// How a kernel caps the memory of its whole process tree ([`tree_memory`]).
338#[derive(Debug, Clone, PartialEq, Eq)]
339pub struct TreeMemoryOptions {
340    /// The cap in MiB; overrides `ULTRON_RLM_MAX_TREE_MEMORY_MB`, whose default is twice the per-process limit.
341    /// 0 disables it.
342    pub max_mb: Option<u64>,
343    /// `auto` | `cgroup` | `watchdog` | `off`; None: `ULTRON_RLM_TREE_MEMORY_BACKEND`, else `auto` (a capped cgroup
344    /// scope when this system allows one, else the watchdog).
345    pub backend: Option<String>,
346    /// The scope launcher (`systemd-run`), probed once before its first use.
347    pub launcher: PathBuf,
348}
349
350impl Default for TreeMemoryOptions {
351    fn default() -> Self {
352        Self {
353            max_mb: None,
354            backend: None,
355            launcher: "systemd-run".into(),
356        }
357    }
358}
359
360impl KernelOptions {
361    pub fn new(cwd: impl Into<PathBuf>, runtime_path: impl Into<PathBuf>) -> Self {
362        Self {
363            cwd: cwd.into(),
364            runtime_path: runtime_path.into(),
365            python: None,
366            startup_timeout: Duration::from_secs(10),
367            limits: LimitOverrides::default(),
368            env: Vec::new(),
369            snapshot_path: None,
370            snapshot_key: None,
371            continuity: None,
372            sandbox: None,
373            tree_memory: None,
374            base_env: None,
375        }
376    }
377
378    /// `name` in the kernel's base environment ([`KernelOptions::base_env`], else this process's).
379    pub fn var(&self, name: &str) -> Option<String> {
380        match &self.base_env {
381            Some(env) => env.iter().find(|(key, _)| key == name).map(|(_, value)| value.clone()),
382            None => std::env::var(name).ok(),
383        }
384    }
385
386    /// The kernel's base environment ([`KernelOptions::base_env`], else this process's).
387    fn base_environment(&self) -> Vec<(std::ffi::OsString, std::ffi::OsString)> {
388        match &self.base_env {
389            Some(env) => env.iter().map(|(key, value)| (key.into(), value.into())).collect(),
390            None => std::env::vars_os().collect(),
391        }
392    }
393}
394
395/// runtime.py exit codes for limits it could not report in-band.
396const EXIT_MEMORY: i32 = 86;
397const EXIT_CPU: i32 = 87;
398const SIGKILL: i32 = 9;
399const SIGTERM: i32 = 15;
400const SIGXCPU: i32 = 24;
401const MAX_HOST_REQUESTS: usize = 16;
402/// How long a finished cell waits for its flush markers on fd 1 and fd 2.
403const RAW_FLUSH_TIMEOUT: Duration = Duration::from_secs(5);
404const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(1);
405/// How long a closed protocol channel waits for the exit status, so a limit's exit code is what gets reported.
406const CLOSED_CHANNEL_GRACE: Duration = Duration::from_millis(200);
407const WATCHDOG_INTERVAL: Duration = Duration::from_millis(500);
408const PLACEHOLDER_ID: &str = "00000000-0000-0000-0000-000000000000";
409const SHUT_DOWN: &str = "RLM kernel is shut down";
410const NOT_RUNNING: &str = "RLM kernel is not running";
411
412fn memory_limit_message(limits: &KernelResourceLimits) -> String {
413    format!(
414        "RLM kernel exceeded its memory limit ({} MiB per process, ULTRON_RLM_MAX_MEMORY_MB) and was stopped; the next cell starts a fresh kernel without earlier Python state",
415        limits.max_memory_mb
416    )
417}
418
419fn cpu_limit_message(limits: &KernelResourceLimits) -> String {
420    format!(
421        "RLM kernel exceeded its CPU limit ({} CPU-seconds per cell, ULTRON_RLM_MAX_CPU_SECONDS) and was stopped; the next cell starts a fresh kernel without earlier Python state",
422        limits.max_cpu_seconds
423    )
424}
425
426/// A frame value as text: strings as they are, null as empty, anything else as JSON.
427fn frame_string(value: Option<&Value>) -> String {
428    match value {
429        None | Some(Value::Null) => String::new(),
430        Some(Value::String(text)) => text.clone(),
431        Some(other) => other.to_string(),
432    }
433}
434
435fn encode_frame(frame: &Map<String, Value>) -> Result<String, KernelError> {
436    let line = serde_json::to_string(frame)
437        .map_err(|error| KernelError::new(format!("RLM kernel cannot encode outbound frame: {error}")))?;
438    if line.len() > MAX_FRAME_BYTES {
439        return Err(KernelError::new(format!(
440            "RLM kernel outbound frame exceeds 1 MiB ({MAX_FRAME_BYTES} bytes)"
441        )));
442    }
443    Ok(line + "\n")
444}
445
446fn object(entries: impl IntoIterator<Item = (&'static str, Value)>) -> Map<String, Value> {
447    entries
448        .into_iter()
449        .map(|(key, value)| (key.to_string(), value))
450        .collect()
451}
452
453/// A random (version 4) UUID: the request id, which also names the cell's flush marker.
454fn random_uuid() -> Result<String, KernelError> {
455    let mut bytes = [0u8; 16];
456    getrandom::fill(&mut bytes).map_err(|error| KernelError::new(format!("RLM kernel request id: {error}")))?;
457    Ok(jsrt::random::uuid_v4_from(bytes))
458}
459
460/// Lone UTF-16 surrogate escapes (which Python's json.dumps emits for surrogate-escaped text) become U+FFFD: a Rust
461/// string cannot hold them, and refusing the frame would end the kernel.
462fn replace_lone_surrogates(line: &str) -> std::borrow::Cow<'_, str> {
463    if !line.contains("\\u") {
464        return line.into();
465    }
466    let hex = |index: usize| -> Option<u16> {
467        let digits = line.get(index..index + 6)?.strip_prefix("\\u")?;
468        u16::from_str_radix(digits, 16).ok()
469    };
470    let bytes = line.as_bytes();
471    let mut text = String::with_capacity(line.len());
472    let (mut index, mut copied) = (0, 0);
473    while index < bytes.len() {
474        if bytes[index] != b'\\' {
475            index += 1;
476            continue;
477        }
478        match hex(index) {
479            Some(0xd800..=0xdbff) if matches!(hex(index + 6), Some(0xdc00..=0xdfff)) => index += 12,
480            Some(0xd800..=0xdfff) => {
481                text.push_str(&line[copied..index]);
482                text.push_str("\\ufffd");
483                index += 6;
484                copied = index;
485            }
486            _ => index += 2,
487        }
488    }
489    if copied == 0 {
490        return line.into();
491    }
492    text.push_str(&line[copied..]);
493    text.into()
494}
495
496/// Exit of the kernel's root process. `exited` is set (under the state lock) before the process is reaped, so a
497/// kill under that lock that sees it unset cannot hit a reused pid.
498#[derive(Default)]
499struct Exit {
500    state: Mutex<(bool, Option<ExitStatus>)>,
501    changed: Condvar,
502}
503
504impl Exit {
505    fn exited(&self) -> bool {
506        jsrt::sync::lock(&self.state).0
507    }
508
509    /// Waits up to `timeout` for the exit status; whether it is known.
510    fn wait_reaped(&self, timeout: Duration) -> bool {
511        let state = jsrt::sync::lock(&self.state);
512        let (state, _) = self
513            .changed
514            .wait_timeout_while(state, timeout, |state| state.1.is_none())
515            .unwrap_or_else(PoisonError::into_inner);
516        state.1.is_some()
517    }
518}
519
520#[derive(Default)]
521struct RawStream {
522    decoder: Utf8Stream,
523    /// Bytes that may start the running cell's flush marker, held until the next chunk.
524    held: Vec<u8>,
525    /// Output from while no cell owned the stream; reported at the start of the next cell.
526    orphan: String,
527    orphan_dropped: bool,
528    /// Whether the runtime's startup marker has arrived; output before it is startup diagnostics.
529    started: bool,
530}
531
532struct Waiter {
533    op: u64,
534    state: KernelReply,
535    controller: Signal,
536    handler: Option<Arc<dyn HostHandler>>,
537}
538
539/// Raw output of the execute cell that owns fd 1 and fd 2 until its flush markers arrive (or time out).
540struct CellOutput {
541    request_id: String,
542    marker: Vec<u8>,
543    captures: [RawCapture; 2],
544    orphan: [String; 2],
545    /// Set by the done frame: the markers runtime.py wrote and the host still waits for.
546    awaiting: Option<[bool; 2]>,
547    deadline: Option<Instant>,
548    /// The finished cell, once its done frame has arrived.
549    finished: Option<Waiter>,
550}
551
552struct Generation {
553    id: u64,
554    pid: i32,
555    /// How the tree cap is enforced for this process.
556    tree_backend: TreeMemoryBackend,
557    /// The cgroup v2 directory of the kernel's scope, resolved once the runtime is up, and its OOM kills then.
558    cgroup_dir: Option<PathBuf>,
559    oom_kills_at_start: u64,
560    /// Descendants seen by watchdog scans (pid to start time), killed with the tree even if they left it.
561    known_descendants: Arc<Mutex<HashMap<i32, u64>>>,
562    writer: mpsc::Sender<String>,
563    exit: Arc<Exit>,
564    raw: [RawStream; 2],
565    output: Option<CellOutput>,
566    pending: HashMap<String, Waiter>,
567    active_cell: Option<String>,
568    /// In-flight host requests and the signal each one's handler gets (aborted by its cell's end or a cancel).
569    host_requests: HashMap<String, Signal>,
570    ready_received: bool,
571    initialized: bool,
572    startup_deadline: Instant,
573    cell_cpu_start: Option<f64>,
574    /// The process the CPU backstop measures: the root, or under a sandbox the Python process found at the first cell.
575    cpu_pid: Option<i32>,
576    sandbox: Option<sandbox::Mechanism>,
577    buffer: Vec<u8>,
578    /// The snapshot this generation restores before it counts as started.
579    restore_path: Option<PathBuf>,
580    /// After `ready`, startup is completed by a waiting caller (restore, recovery report) instead of the reader.
581    host_completes_startup: bool,
582    /// A caller is completing startup.
583    completing: bool,
584}
585
586/// A `host_request` frame: `cell` is the execute request whose code made it (absent outside a cell).
587struct HostRequest {
588    id: String,
589    kind: String,
590    payload: Map<String, Value>,
591    cell: Option<String>,
592}
593
594struct Op {
595    running: bool,
596    outcome: Option<Result<KernelReply, KernelError>>,
597    /// The controller of the cell this call runs.
598    cell: Option<Signal>,
599    handler: Option<Arc<dyn HostHandler>>,
600    /// A recovery report taken by this execute call when its cell was sent.
601    recovery: Option<ContinuityReport>,
602}
603
604/// Continuity bookkeeping (see [`continuity`]).
605#[derive(Default)]
606struct Continuity {
607    /// The latest checkpoint written to the configured snapshot path.
608    checkpoint: Option<SnapshotReport>,
609    /// Cells completed, and the count at the latest checkpoint.
610    cells: u64,
611    checkpoint_cells: u64,
612    last_checkpoint: Option<Instant>,
613    /// A cell completed since the latest checkpoint.
614    dirty: bool,
615    snapshot_queued: bool,
616    idle_flush_scheduled: bool,
617    /// The failure the next recovery reports.
618    crash: Option<Crash>,
619    /// A recovery report for the next cell.
620    notice: Option<ContinuityReport>,
621    last_recovery: Option<ContinuityReport>,
622    /// The failed generation was running: start a new one without waiting for a call.
623    restart: bool,
624}
625
626#[derive(Default)]
627struct State {
628    generation: Option<Generation>,
629    next_generation: u64,
630    /// The error the latest failed generation reported.
631    failure: Option<(u64, KernelError)>,
632    closed: bool,
633    stderr: String,
634    ops: HashMap<u64, Op>,
635    queue: VecDeque<u64>,
636    next_op: u64,
637    /// kernel.ts `autoRestore = false`: never restore the configured snapshot into a later generation.
638    no_auto_restore: bool,
639    continuity: Continuity,
640}
641
642impl State {
643    fn current(&mut self, generation: u64) -> Option<&mut Generation> {
644        self.generation.as_mut().filter(|current| current.id == generation)
645    }
646
647    fn diagnostics(&self) -> String {
648        if self.stderr.is_empty() {
649            String::new()
650        } else {
651            format!(": {}", self.stderr)
652        }
653    }
654
655    fn resolve(&mut self, waiter: Waiter) {
656        if let Some(op) = self.ops.get_mut(&waiter.op)
657            && op.outcome.is_none()
658        {
659            op.outcome = Some(Ok(waiter.state));
660        }
661    }
662
663    fn reject_ops(&mut self, error: &KernelError) {
664        for op in self.ops.values_mut() {
665            op.outcome.get_or_insert_with(|| Err(error.clone()));
666        }
667        self.queue.clear();
668    }
669}
670
671struct Shared {
672    options: KernelOptions,
673    handler: Arc<dyn HostHandler>,
674    limits: KernelResourceLimits,
675    /// The tree cap in MiB (0: none).
676    tree_memory_mb: u64,
677    snapshot_key: Vec<u8>,
678    state: Mutex<State>,
679    changed: Condvar,
680}
681
682/// One kernel: a sequence of Python processes (generations), each replaced after a failure.
683pub struct Kernel {
684    shared: Arc<Shared>,
685}
686
687impl Kernel {
688    pub fn new(options: KernelOptions, handler: Arc<dyn HostHandler>) -> Result<Self, KernelError> {
689        let limits = kernel_resource_limits(|name| options.var(name), options.limits)?;
690        if options.startup_timeout.is_zero() || options.startup_timeout > Duration::from_millis(2_147_483_647) {
691            return Err(KernelError::new(
692                "RLM startupTimeoutMs must be a positive, finite timer duration",
693            ));
694        }
695        if options.continuity.is_some() && options.snapshot_path.is_none() {
696            return Err(KernelError::new("RLM kernel continuity requires a snapshot path"));
697        }
698        let snapshot_key = options.snapshot_key.clone().unwrap_or_else(process_snapshot_key);
699        let tree_memory_mb = match &options.tree_memory {
700            Some(tree) => tree_memory::kernel_tree_memory_limit(
701                options.var(crate::env::RLM_MAX_TREE_MEMORY_MB_ENV).as_deref(),
702                limits.max_memory_mb,
703                tree.max_mb,
704            )?,
705            None => 0,
706        };
707        Ok(Self {
708            shared: Arc::new(Shared {
709                options,
710                handler,
711                limits,
712                tree_memory_mb,
713                snapshot_key,
714                state: Mutex::new(State::default()),
715                changed: Condvar::new(),
716            }),
717        })
718    }
719
720    pub fn limits(&self) -> KernelResourceLimits {
721        self.shared.limits
722    }
723
724    /// Whether a kernel process is up and initialized.
725    pub fn is_running(&self) -> bool {
726        let state = jsrt::sync::lock(&self.shared.state);
727        !state.closed
728            && state
729                .generation
730                .as_ref()
731                .is_some_and(|generation| generation.initialized)
732    }
733
734    /// The bounded tail of the kernel's raw stderr, for diagnostics.
735    pub fn stderr_tail(&self) -> String {
736        jsrt::sync::lock(&self.shared.state).stderr.clone()
737    }
738
739    /// How many calls (cells, snapshots, checkpoints) are running or queued, in call order.
740    pub fn pending_calls(&self) -> usize {
741        jsrt::sync::lock(&self.shared.state).queue.len()
742    }
743
744    pub fn pid(&self) -> Option<u32> {
745        let state = jsrt::sync::lock(&self.shared.state);
746        state
747            .generation
748            .as_ref()
749            .and_then(|generation| u32::try_from(generation.pid).ok())
750    }
751
752    /// Proportional memory of the kernel's process tree (Linux, read from /proc) and the tree cap in bytes; None while
753    /// no kernel process runs (kernel.ts `memoryUsage`, for `rlm.pool`).
754    pub fn memory_usage(&self) -> Option<(u64, Option<u64>)> {
755        let pid = {
756            let state = jsrt::sync::lock(&self.shared.state);
757            if state.closed {
758                return None;
759            }
760            state.generation.as_ref()?.pid
761        };
762        if !cfg!(target_os = "linux") {
763            return None;
764        }
765        let cap = self.shared.tree_memory_mb;
766        Some((
767            tree_memory::tree_memory_usage(pid).0,
768            (cap > 0).then(|| cap * 1024 * 1024),
769        ))
770    }
771
772    /// How the running kernel's tree cap is enforced; None while no kernel process runs.
773    pub fn tree_memory_backend(&self) -> Option<TreeMemoryBackend> {
774        let state = jsrt::sync::lock(&self.shared.state);
775        state.generation.as_ref().map(|generation| generation.tree_backend)
776    }
777
778    /// The sandbox the running kernel is in; None when unsandboxed or not running.
779    pub fn sandbox(&self) -> Option<sandbox::Mechanism> {
780        let state = jsrt::sync::lock(&self.shared.state);
781        state.generation.as_ref().and_then(|generation| generation.sandbox)
782    }
783
784    /// Starts the kernel process if none runs and waits until it is ready.
785    pub fn start(&self) -> Result<(), KernelError> {
786        let mut state = jsrt::sync::lock(&self.shared.state);
787        let generation = self.shared.ensure_started(&mut state, true)?;
788        self.shared.wait_started(state, generation, None).map(drop)
789    }
790
791    /// Runs a cell. `handler`, when given, serves this cell's host requests instead of the kernel's handler. The first
792    /// cell after a recovery starts its stderr with the recovery notice ([`ContinuityReport::render`]).
793    pub fn execute(
794        &self,
795        code: &str,
796        signal: Option<&Signal>,
797        handler: Option<Arc<dyn HostHandler>>,
798    ) -> Result<ExecutionResult, KernelError> {
799        self.execute_cell(code, signal, handler).map(|reply| reply.result)
800    }
801
802    /// [`Kernel::execute`] with the structured recovery report of the first cell after a recovery.
803    pub fn execute_cell(
804        &self,
805        code: &str,
806        signal: Option<&Signal>,
807        handler: Option<Arc<dyn HostHandler>>,
808    ) -> Result<KernelReply, KernelError> {
809        let frame = object([("request", Value::from("execute")), ("code", Value::from(code))]);
810        self.shared.enqueue(frame, signal, handler)
811    }
812
813    /// Clears invocation scratch while keeping the API bindings and the declared `state` dict.
814    pub fn reset_scratch(&self) -> Result<ExecutionResult, KernelError> {
815        self.shared
816            .enqueue(object([("request", Value::from("reset_scratch"))]), None, None)
817            .map(|reply| reply.result)
818    }
819
820    /// Saves the namespace's restorable names to `path` (default: the configured snapshot path), signed by the host.
821    /// The reply's `snapshot` lists saved names and every skipped name with a reason; a failure leaves the previous
822    /// file intact.
823    pub fn snapshot(&self, path: Option<&Path>) -> Result<KernelReply, KernelError> {
824        let path = self.snapshot_target(path)?;
825        self.shared.enqueue(
826            object([("request", Value::from("snapshot")), ("path", Value::from(path))]),
827            None,
828            None,
829        )
830    }
831
832    /// Restores a host-verified snapshot from `path` (default: the configured snapshot path). Unsigned, tampered or
833    /// foreign-key snapshots are refused before the kernel reads them, and the namespace stays untouched.
834    pub fn restore(&self, path: Option<&Path>) -> Result<KernelReply, KernelError> {
835        let path = self.snapshot_target(path)?;
836        self.shared.enqueue(
837            object([("request", Value::from("restore")), ("path", Value::from(path))]),
838            None,
839            None,
840        )
841    }
842
843    fn snapshot_target(&self, path: Option<&Path>) -> Result<String, KernelError> {
844        path.or(self.shared.options.snapshot_path.as_deref())
845            .map(|path| path.to_string_lossy().into_owned())
846            .ok_or_else(|| KernelError::new("RLM snapshot path is not configured"))
847    }
848
849    /// The latest recovery's report, also after it was delivered with a cell.
850    pub fn last_recovery(&self) -> Option<ContinuityReport> {
851        jsrt::sync::lock(&self.shared.state).continuity.last_recovery.clone()
852    }
853
854    /// The latest checkpoint written to the configured snapshot path.
855    pub fn checkpoint(&self) -> Option<SnapshotReport> {
856        jsrt::sync::lock(&self.shared.state).continuity.checkpoint.clone()
857    }
858
859    /// Permanently closes this kernel: kills its process tree, rejects every call, and waits (bounded) for the exit.
860    pub fn shutdown(&self) {
861        let exit = {
862            let mut state = jsrt::sync::lock(&self.shared.state);
863            state.closed = true;
864            let error = KernelError::new(SHUT_DOWN);
865            let exit = match state
866                .generation
867                .as_ref()
868                .map(|generation| (generation.id, generation.exit.clone()))
869            {
870                Some((id, exit)) => {
871                    self.shared.fail(&mut state, id, error);
872                    Some(exit)
873                }
874                None => {
875                    state.reject_ops(&error);
876                    None
877                }
878            };
879            self.shared.changed.notify_all();
880            exit
881        };
882        if let Some(exit) = exit {
883            exit.wait_reaped(SHUTDOWN_TIMEOUT);
884        }
885    }
886}
887
888impl Drop for Kernel {
889    fn drop(&mut self) {
890        self.shutdown();
891    }
892}
893
894impl Shared {
895    fn with_state(&self, run: impl FnOnce(&mut State)) {
896        let mut state = jsrt::sync::lock(&self.state);
897        run(&mut state);
898        drop(state);
899        self.changed.notify_all();
900    }
901
902    fn enqueue(
903        self: &Arc<Self>,
904        frame: Map<String, Value>,
905        signal: Option<&Signal>,
906        handler: Option<Arc<dyn HostHandler>>,
907    ) -> Result<KernelReply, KernelError> {
908        if jsrt::sync::lock(&self.state).closed {
909            return Err(KernelError::new(SHUT_DOWN));
910        }
911        if let Some(reason) = signal.and_then(Signal::reason) {
912            return Err(KernelError::new(reason));
913        }
914        let mut probe = frame.clone();
915        probe.insert("id".into(), Value::from(PLACEHOLDER_ID));
916        encode_frame(&probe)?;
917        let op = {
918            let mut state = jsrt::sync::lock(&self.state);
919            if state.closed {
920                return Err(KernelError::new(SHUT_DOWN));
921            }
922            state.next_op += 1;
923            let op = state.next_op;
924            state.ops.insert(
925                op,
926                Op {
927                    running: false,
928                    outcome: None,
929                    cell: None,
930                    handler,
931                    recovery: None,
932                },
933            );
934            state.queue.push_back(op);
935            op
936        };
937        let kind = frame
938            .get("request")
939            .and_then(Value::as_str)
940            .unwrap_or_default()
941            .to_string();
942        let link = signal.map(|signal| {
943            let shared: Weak<Self> = Arc::downgrade(self);
944            signal.link(move |reason| {
945                if let Some(shared) = shared.upgrade() {
946                    shared.abort_op(op, reason);
947                }
948            })
949        });
950        let mut result = self.run(op, frame);
951        drop(link);
952        let mut state = jsrt::sync::lock(&self.state);
953        let recovery = state.ops.remove(&op).and_then(|entry| entry.recovery);
954        if state.queue.front() == Some(&op) {
955            state.queue.pop_front();
956        }
957        match &mut result {
958            Ok(reply) => {
959                if let Some(report) = recovery {
960                    reply.result.stderr = format!("{}{}", report.render(), reply.result.stderr);
961                    reply.recovery = Some(report);
962                }
963                if kind == "execute" || kind == "reset_scratch" {
964                    self.cell_completed(&mut state);
965                }
966            }
967            // Not delivered: the next cell reports it, unless a newer recovery replaced it.
968            Err(_) => {
969                if state.continuity.notice.is_none() {
970                    state.continuity.notice = recovery;
971                }
972            }
973        }
974        drop(state);
975        self.changed.notify_all();
976        result
977    }
978
979    /// Continuity: a cell may have changed the namespace; checkpoint now, or once the debounce interval has passed.
980    fn cell_completed(self: &Arc<Self>, state: &mut State) {
981        let Some(policy) = self.options.continuity else {
982            return;
983        };
984        let continuity = &mut state.continuity;
985        continuity.cells += 1;
986        continuity.dirty = true;
987        if continuity.snapshot_queued || state.closed {
988            return;
989        }
990        let due = continuity
991            .last_checkpoint
992            .map(|last| last + policy.min_interval)
993            .filter(|due| *due > Instant::now());
994        match due {
995            // Ahead of any queued cell: a crash in the next cell must find this cell's state saved.
996            None => self.queue_checkpoint(state, true),
997            Some(due) if !continuity.idle_flush_scheduled => {
998                continuity.idle_flush_scheduled = true;
999                let shared = Arc::downgrade(self);
1000                std::thread::spawn(move || {
1001                    std::thread::sleep(due.saturating_duration_since(Instant::now()));
1002                    let Some(shared) = shared.upgrade() else {
1003                        return;
1004                    };
1005                    let mut state = jsrt::sync::lock(&shared.state);
1006                    state.continuity.idle_flush_scheduled = false;
1007                    if state.continuity.dirty && !state.continuity.snapshot_queued && !state.closed {
1008                        shared.queue_checkpoint(&mut state, false);
1009                    }
1010                });
1011            }
1012            Some(_) => {}
1013        }
1014    }
1015
1016    /// Queues a checkpoint of the configured snapshot path, at the front of the queue or at its end, and runs it on
1017    /// its own thread so the cell that triggered it returns at once.
1018    fn queue_checkpoint(self: &Arc<Self>, state: &mut State, front: bool) {
1019        let Some(path) = self.options.snapshot_path.as_ref() else {
1020            return;
1021        };
1022        state.next_op += 1;
1023        let op = state.next_op;
1024        state.ops.insert(
1025            op,
1026            Op {
1027                running: false,
1028                outcome: None,
1029                cell: None,
1030                handler: None,
1031                recovery: None,
1032            },
1033        );
1034        if front {
1035            state.queue.push_front(op);
1036        } else {
1037            state.queue.push_back(op);
1038        }
1039        state.continuity.snapshot_queued = true;
1040        let frame = object([
1041            ("request", Value::from("snapshot")),
1042            ("path", Value::from(path.to_string_lossy().into_owned())),
1043        ]);
1044        let shared = self.clone();
1045        std::thread::spawn(move || {
1046            // The outcome is recorded by signed_snapshot; a failed checkpoint keeps `dirty` for the next cell.
1047            let _ = shared.run(op, frame);
1048            let mut state = jsrt::sync::lock(&shared.state);
1049            state.ops.remove(&op);
1050            if state.queue.front() == Some(&op) {
1051                state.queue.pop_front();
1052            }
1053            state.continuity.snapshot_queued = false;
1054            drop(state);
1055            shared.changed.notify_all();
1056        });
1057    }
1058
1059    fn abort_op(&self, op: u64, reason: String) {
1060        self.with_state(|state| {
1061            let error = KernelError::new(reason);
1062            let Some(entry) = state.ops.get_mut(&op) else {
1063                return;
1064            };
1065            if entry.outcome.is_some() {
1066                return;
1067            }
1068            if !entry.running {
1069                // Still queued: the process serving the preceding cell is left alone.
1070                entry.outcome = Some(Err(error));
1071                state.queue.retain(|queued| *queued != op);
1072                return;
1073            }
1074            if let Some(cell) = &entry.cell {
1075                cell.abort(error.message());
1076            }
1077            if self.options.continuity.is_none() {
1078                state.no_auto_restore = true;
1079            }
1080            match state.generation.as_ref().map(|generation| generation.id) {
1081                Some(generation) => self.fail(state, generation, error),
1082                None => entry_reject(state, op, error),
1083            }
1084        });
1085    }
1086
1087    fn take_outcome(state: &mut State, op: u64) -> Option<Result<KernelReply, KernelError>> {
1088        state.ops.get_mut(&op).and_then(|entry| entry.outcome.take())
1089    }
1090
1091    fn run(self: &Arc<Self>, op: u64, frame: Map<String, Value>) -> Result<KernelReply, KernelError> {
1092        let mut state = jsrt::sync::lock(&self.state);
1093        loop {
1094            if let Some(outcome) = Self::take_outcome(&mut state, op) {
1095                return outcome;
1096            }
1097            if state.queue.front() == Some(&op) {
1098                break;
1099            }
1100            state = self.changed.wait(state).unwrap_or_else(PoisonError::into_inner);
1101        }
1102        if let Some(entry) = state.ops.get_mut(&op) {
1103            entry.running = true;
1104        }
1105        let kind = frame
1106            .get("request")
1107            .and_then(Value::as_str)
1108            .unwrap_or_default()
1109            .to_string();
1110        // An explicit restore can also recover from a broken configured snapshot.
1111        let generation = self.ensure_started(&mut state, kind != "restore")?;
1112        let state = self.wait_started(state, generation, Some(op))?;
1113        drop(state);
1114        let path = frame.get("path").and_then(Value::as_str).unwrap_or_default();
1115        match kind.as_str() {
1116            "snapshot" => self.signed_snapshot(op, generation, Path::new(path)),
1117            "restore" => self.verified_restore(op, generation, Path::new(path)),
1118            _ => self.request(op, generation, frame),
1119        }
1120    }
1121
1122    /// Sends one request of call `op` to `generation` and waits for its done frame.
1123    fn request(
1124        self: &Arc<Self>,
1125        op: u64,
1126        generation: u64,
1127        mut frame: Map<String, Value>,
1128    ) -> Result<KernelReply, KernelError> {
1129        let mut state = jsrt::sync::lock(&self.state);
1130        let execute = frame.get("request") == Some(&Value::from("execute"));
1131        let request_id = random_uuid()?;
1132        frame.insert("id".into(), Value::from(request_id.clone()));
1133        let line = encode_frame(&frame)?;
1134        let handler = state.ops.get(&op).and_then(|entry| entry.handler.clone());
1135        let controller = Signal::new();
1136        let Some(current) = state.current(generation) else {
1137            return Err(Self::take_outcome(&mut state, op)
1138                .and_then(Result::err)
1139                .unwrap_or_else(|| KernelError::new(NOT_RUNNING)));
1140        };
1141        let state_ok = KernelReply::from_result(ExecutionResult {
1142            status: Status::Ok,
1143            stdout: String::new(),
1144            stderr: String::new(),
1145            result: None,
1146            error: None,
1147        });
1148        current.pending.insert(
1149            request_id.clone(),
1150            Waiter {
1151                op,
1152                state: state_ok,
1153                controller: controller.clone(),
1154                handler,
1155            },
1156        );
1157        if execute {
1158            if current.cpu_pid.is_none() {
1159                current.cpu_pid = sandboxed::python_pid(current.pid, &self.python(), &self.options.runtime_path);
1160            }
1161            current.cell_cpu_start = current.cpu_pid.and_then(process::process_cpu_seconds);
1162            current.active_cell = Some(request_id.clone());
1163            take_raw_output(current, &request_id);
1164            let notice = state.continuity.notice.take();
1165            if let Some(entry) = state.ops.get_mut(&op) {
1166                entry.cell = Some(controller);
1167                entry.recovery = notice;
1168            }
1169        }
1170        if let Err(error) = self.send(&mut state, generation, line) {
1171            if let Some(current) = state.current(generation) {
1172                current.pending.remove(&request_id);
1173                if current.active_cell.as_deref() == Some(request_id.as_str()) {
1174                    current.active_cell = None;
1175                }
1176            }
1177            entry_reject(&mut state, op, error.clone());
1178            self.fail(&mut state, generation, error);
1179        }
1180        let mut cpu_checked = Instant::now();
1181        loop {
1182            if let Some(outcome) = Self::take_outcome(&mut state, op) {
1183                return outcome;
1184            }
1185            let now = Instant::now();
1186            let mut timeout = WATCHDOG_INTERVAL;
1187            if let Some(current) = state.current(generation) {
1188                // A startup restore is bounded by the startup timeout.
1189                if !current.initialized {
1190                    let deadline = current.startup_deadline;
1191                    if deadline <= now {
1192                        let error = self.startup_timeout(&state);
1193                        self.fail(&mut state, generation, error);
1194                        continue;
1195                    }
1196                    timeout = timeout.min(deadline - now);
1197                }
1198                let Some(current) = state.current(generation) else {
1199                    continue;
1200                };
1201                if let Some(deadline) = current.output.as_ref().and_then(|output| output.deadline) {
1202                    if deadline <= now {
1203                        finish_cell(&mut state, generation);
1204                        continue;
1205                    }
1206                    timeout = timeout.min(deadline - now);
1207                }
1208                if now.duration_since(cpu_checked) >= WATCHDOG_INTERVAL {
1209                    cpu_checked = now;
1210                    if self.cpu_exceeded(current) {
1211                        let error = KernelError::new(cpu_limit_message(&self.limits));
1212                        self.fail(&mut state, generation, error);
1213                        continue;
1214                    }
1215                }
1216            }
1217            state = self
1218                .changed
1219                .wait_timeout(state, timeout)
1220                .unwrap_or_else(PoisonError::into_inner)
1221                .0;
1222        }
1223    }
1224
1225    /// Backstop for the per-cell CPU limit: a cell stuck in C code or swallowing the limit error is killed after a
1226    /// grace period.
1227    fn cpu_exceeded(&self, generation: &Generation) -> bool {
1228        let limit = self.limits.max_cpu_seconds as f64;
1229        if limit <= 0.0 || generation.active_cell.is_none() {
1230            return false;
1231        }
1232        let Some(start) = generation.cell_cpu_start else {
1233            return false;
1234        };
1235        let grace = (limit * 0.1).clamp(2.0, 10.0);
1236        generation
1237            .cpu_pid
1238            .and_then(process::process_cpu_seconds)
1239            .is_some_and(|used| used - start > limit + grace)
1240    }
1241
1242    /// Whether the tree cap has been hit: for a cgroup scope, an OOM kill in its memory.events since the runtime came
1243    /// up; for the watchdog, the summed memory of the kernel's tree is over the cap.
1244    fn tree_memory_exceeded(&self, generation: &mut Generation) -> bool {
1245        match generation.tree_backend {
1246            TreeMemoryBackend::Off => false,
1247            TreeMemoryBackend::Watchdog => {
1248                watchdog_tree_bytes(generation.pid, &generation.known_descendants) > self.tree_memory_mb * 1024 * 1024
1249            }
1250            TreeMemoryBackend::Cgroup => {
1251                if generation.cgroup_dir.is_none() {
1252                    // Before systemd-run execs the runtime the pid may still sit in the caller's cgroup.
1253                    let dir = tree_memory::process_cgroup_dir(generation.pid)
1254                        .filter(|dir| tree_memory::is_foreign_scope(dir));
1255                    let Some(dir) = dir.filter(|_| generation.ready_received) else {
1256                        return false;
1257                    };
1258                    generation.oom_kills_at_start = tree_memory::cgroup_oom_kills(&dir).unwrap_or(0);
1259                    generation.cgroup_dir = Some(dir);
1260                }
1261                generation
1262                    .cgroup_dir
1263                    .as_deref()
1264                    .and_then(tree_memory::cgroup_oom_kills)
1265                    .is_some_and(|kills| kills > generation.oom_kills_at_start)
1266            }
1267        }
1268    }
1269
1270    /// Polls the tree cap and kills the whole tree (through `fail`) when it is exceeded: the scope's memory.events
1271    /// every 500 ms (the kernel enforces MemoryMax itself); the watchdog faster as the tree nears the cap. The tree is
1272    /// measured outside the state lock; the poll ends with the process or once another generation runs.
1273    fn start_tree_memory_watchdog(
1274        self: &Arc<Self>,
1275        id: u64,
1276        pid: i32,
1277        backend: TreeMemoryBackend,
1278        known: Arc<Mutex<HashMap<i32, u64>>>,
1279        exit: Arc<Exit>,
1280    ) {
1281        let shared = Arc::downgrade(self);
1282        std::thread::spawn(move || {
1283            let mut delay = Duration::from_millis(500);
1284            loop {
1285                if exit.wait_reaped(delay) {
1286                    return;
1287                }
1288                let Some(shared) = shared.upgrade() else {
1289                    return;
1290                };
1291                let cap_bytes = shared.tree_memory_mb * 1024 * 1024;
1292                let bytes = (backend == TreeMemoryBackend::Watchdog).then(|| watchdog_tree_bytes(pid, &known));
1293                let mut state = jsrt::sync::lock(&shared.state);
1294                let Some(current) = state.current(id) else {
1295                    return;
1296                };
1297                let exceeded = match bytes {
1298                    Some(bytes) => {
1299                        delay = Duration::from_millis(tree_memory::watchdog_poll_ms(bytes, cap_bytes));
1300                        bytes > cap_bytes
1301                    }
1302                    None => shared.tree_memory_exceeded(current),
1303                };
1304                if exceeded {
1305                    let message = tree_memory::tree_memory_limit_message(shared.tree_memory_mb, backend);
1306                    shared.fail(&mut state, id, KernelError::new(message));
1307                    drop(state);
1308                    shared.changed.notify_all();
1309                    return;
1310                }
1311            }
1312        });
1313    }
1314
1315    /// Waits until `generation` is initialized, its startup times out, or it (or the call `op`) fails.
1316    fn wait_started<'a>(
1317        self: &'a Arc<Self>,
1318        mut state: MutexGuard<'a, State>,
1319        generation: u64,
1320        op: Option<u64>,
1321    ) -> Result<MutexGuard<'a, State>, KernelError> {
1322        loop {
1323            if let Some(op) = op
1324                && let Some(outcome) = Self::take_outcome(&mut state, op)
1325            {
1326                return Err(outcome.err().unwrap_or_else(|| KernelError::new(NOT_RUNNING)));
1327            }
1328            let Some(current) = state.current(generation) else {
1329                return Err(match &state.failure {
1330                    Some((failed, error)) if *failed == generation => error.clone(),
1331                    _ => KernelError::new(NOT_RUNNING),
1332                });
1333            };
1334            if current.initialized {
1335                return Ok(state);
1336            }
1337            if current.ready_received && !current.completing {
1338                // This caller completes startup: restore the configured snapshot, report a recovery.
1339                current.completing = true;
1340                let restore_path = current.restore_path.clone();
1341                drop(state);
1342                self.complete_startup(generation, restore_path);
1343                state = jsrt::sync::lock(&self.state);
1344                continue;
1345            }
1346            let deadline = current.startup_deadline;
1347            let now = Instant::now();
1348            if now >= deadline {
1349                let error = self.startup_timeout(&state);
1350                self.fail(&mut state, generation, error);
1351                continue;
1352            }
1353            state = self
1354                .changed
1355                .wait_timeout(state, deadline - now)
1356                .unwrap_or_else(PoisonError::into_inner)
1357                .0;
1358        }
1359    }
1360
1361    fn startup_timeout(&self, state: &State) -> KernelError {
1362        KernelError::new(format!(
1363            "RLM kernel startup timed out after {}ms{}",
1364            self.options.startup_timeout.as_millis(),
1365            state.diagnostics()
1366        ))
1367    }
1368
1369    /// The running generation, or a new one; `restore` lets a new one restore the configured snapshot.
1370    fn ensure_started(self: &Arc<Self>, state: &mut State, restore: bool) -> Result<u64, KernelError> {
1371        if state.closed {
1372            return Err(KernelError::new(SHUT_DOWN));
1373        }
1374        if let Some(generation) = &state.generation {
1375            return Ok(generation.id);
1376        }
1377        self.spawn(state, restore)
1378    }
1379
1380    /// Completes a generation's startup after `ready`: restores `restore_path` (kernel.ts `completeStartup`) and, with
1381    /// continuity, prepares the recovery report for the next cell.
1382    fn complete_startup(self: &Arc<Self>, generation: u64, restore_path: Option<PathBuf>) {
1383        let restored = restore_path.as_ref().map(|path| {
1384            let op = {
1385                let mut state = jsrt::sync::lock(&self.state);
1386                state.next_op += 1;
1387                let op = state.next_op;
1388                state.ops.insert(
1389                    op,
1390                    Op {
1391                        running: true,
1392                        outcome: None,
1393                        cell: None,
1394                        handler: None,
1395                        recovery: None,
1396                    },
1397                );
1398                op
1399            };
1400            let outcome = self.verified_restore(op, generation, path);
1401            jsrt::sync::lock(&self.state).ops.remove(&op);
1402            outcome
1403        });
1404        let mut state = jsrt::sync::lock(&self.state);
1405        if state.current(generation).is_none() {
1406            return;
1407        }
1408        let continuity = self.options.continuity.is_some();
1409        let skipped_restore = state
1410            .continuity
1411            .crash
1412            .as_ref()
1413            .and_then(|crash| crash.restore_error.clone());
1414        let restore: Result<Option<RestoreReport>, String> = match restored {
1415            // Continuity skips a checkpoint whose restore failed or killed the kernel; that failure is reported.
1416            None => skipped_restore.map_or(Ok(None), Err),
1417            Some(Ok(reply)) if reply.result.status == Status::Error => Err(reply
1418                .result
1419                .error
1420                .map_or_else(|| "unknown restore error".to_string(), |error| error.evalue)),
1421            Some(Ok(reply)) => Ok(reply.restore),
1422            Some(Err(error)) => {
1423                self.fail(&mut state, generation, error);
1424                drop(state);
1425                self.changed.notify_all();
1426                return;
1427            }
1428        };
1429        if let Err(message) = &restore
1430            && !continuity
1431        {
1432            let error = KernelError::new(format!("RLM kernel startup restore failed: {message}"));
1433            self.fail(&mut state, generation, error);
1434            drop(state);
1435            self.changed.notify_all();
1436            return;
1437        }
1438        if continuity {
1439            let crash = state.continuity.crash.take();
1440            let restored_something = matches!(&restore, Ok(Some(report)) if !report.missing);
1441            if crash.is_some() || restored_something || restore.is_err() {
1442                let crash = crash.unwrap_or_else(|| Crash {
1443                    reason: "kernel started from its checkpoint".into(),
1444                    interrupted_cell: false,
1445                    unsaved_cells: 0,
1446                    restore_error: None,
1447                });
1448                let path = self
1449                    .options
1450                    .snapshot_path
1451                    .as_ref()
1452                    .map(|path| path.display().to_string())
1453                    .unwrap_or_default();
1454                let report = recovery_report(crash, path, restore, state.continuity.checkpoint.as_ref());
1455                state.continuity.notice = Some(report.clone());
1456                state.continuity.last_recovery = Some(report);
1457            }
1458        }
1459        if let Some(current) = state.current(generation) {
1460            current.initialized = true;
1461        }
1462        drop(state);
1463        self.changed.notify_all();
1464    }
1465
1466    /// kernel.ts `signedSnapshot`: the kernel writes an unsigned temporary file; the host checks it is exactly what
1467    /// the kernel reported, signs it, and atomically replaces `path`. A failure leaves the previous snapshot intact.
1468    fn signed_snapshot(self: &Arc<Self>, op: u64, generation: u64, path: &Path) -> Result<KernelReply, KernelError> {
1469        let directory = path.parent().unwrap_or(Path::new(""));
1470        let name = path
1471            .file_name()
1472            .map(|name| name.to_string_lossy().into_owned())
1473            .unwrap_or_default();
1474        let unsigned = directory.join(format!(".{name}.{}.unsigned", random_uuid()?));
1475        let frame = object([
1476            ("request", Value::from("snapshot")),
1477            ("path", Value::from(unsigned.to_string_lossy().into_owned())),
1478        ]);
1479        let outcome = self.request(op, generation, frame).and_then(|reply| {
1480            if reply.result.status != Status::Ok {
1481                return Ok(reply);
1482            }
1483            let failed = |error: String| KernelReply::from_result(snapshot::snapshot_error(error));
1484            let content = match std::fs::read(&unsigned) {
1485                Ok(content) => content,
1486                Err(error) => return Ok(failed(format!("snapshot signing failed: {error}"))),
1487            };
1488            let reported = reply
1489                .snapshot
1490                .as_ref()
1491                .and_then(|report| report.content_sha256.as_deref());
1492            if reported != Some(jsrt::hash::sha256_hex(&content).as_str()) {
1493                return Ok(failed(
1494                    "snapshot changed after the kernel wrote it; refusing to sign it".into(),
1495                ));
1496            }
1497            let temporary = directory.join(format!(".{name}.{}.tmp", random_uuid()?));
1498            let written = std::fs::OpenOptions::new()
1499                .write(true)
1500                .create(true)
1501                .truncate(true)
1502                .mode(0o600)
1503                .open(&temporary)
1504                .and_then(|mut file| file.write_all(&sign_snapshot(&self.snapshot_key, &content)))
1505                .and_then(|()| std::fs::rename(&temporary, path));
1506            let _ = std::fs::remove_file(&temporary);
1507            if let Err(error) = written {
1508                return Ok(failed(format!("snapshot signing failed: {error}")));
1509            }
1510            if self.options.snapshot_path.as_deref() == Some(path) {
1511                let mut state = jsrt::sync::lock(&self.state);
1512                let continuity = &mut state.continuity;
1513                continuity.checkpoint = reply.snapshot.clone();
1514                continuity.checkpoint_cells = continuity.cells;
1515                continuity.last_checkpoint = Some(Instant::now());
1516                continuity.dirty = false;
1517                // A good checkpoint may be restored again, even after a refused one.
1518                if self.options.continuity.is_some() {
1519                    state.no_auto_restore = false;
1520                }
1521            }
1522            Ok(reply)
1523        });
1524        let _ = std::fs::remove_file(&unsigned);
1525        outcome
1526    }
1527
1528    /// kernel.ts `verifiedRestore`: refuses unsigned or tampered snapshots before the kernel reads them; the namespace
1529    /// stays untouched. The kernel re-hashes what it reads, so a file swapped after this check is refused too.
1530    fn verified_restore(self: &Arc<Self>, op: u64, generation: u64, path: &Path) -> Result<KernelReply, KernelError> {
1531        let shown = path.to_string_lossy().into_owned();
1532        let raw = match std::fs::read(path) {
1533            Ok(raw) => raw,
1534            // A missing snapshot is reported by the kernel as `missing`; nothing is loaded.
1535            Err(error)
1536                if matches!(
1537                    error.kind(),
1538                    std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory
1539                ) =>
1540            {
1541                let frame = object([("request", Value::from("restore")), ("path", Value::from(shown))]);
1542                return self.request(op, generation, frame);
1543            }
1544            Err(error) => {
1545                return Ok(KernelReply::from_result(snapshot::snapshot_error(format!(
1546                    "snapshot cannot be read: {error}"
1547                ))));
1548            }
1549        };
1550        match verify_snapshot(&self.snapshot_key, &raw) {
1551            Err(error) => Ok(KernelReply::from_result(snapshot::snapshot_error(error.0))),
1552            Ok(content_sha256) => {
1553                let frame = object([
1554                    ("request", Value::from("restore")),
1555                    ("path", Value::from(shown)),
1556                    ("content_sha256", Value::from(content_sha256)),
1557                ]);
1558                self.request(op, generation, frame)
1559            }
1560        }
1561    }
1562
1563    fn python(&self) -> PathBuf {
1564        self.options
1565            .python
1566            .clone()
1567            .unwrap_or_else(|| crate::env::python(|name| self.options.var(name)).into())
1568    }
1569
1570    fn spawn(self: &Arc<Self>, state: &mut State, restore: bool) -> Result<u64, KernelError> {
1571        let options = &self.options;
1572        let python = self.python();
1573        let spawn_error = |error: std::io::Error| KernelError::new(format!("RLM kernel spawn failed: {error}"));
1574        let mechanism = match &options.sandbox {
1575            Some(launcher) => sandboxed::mechanism(launcher)?,
1576            None => None,
1577        };
1578        let tree_backend = match &options.tree_memory {
1579            None => TreeMemoryBackend::Off,
1580            Some(tree) => {
1581                let requested = tree
1582                    .backend
1583                    .clone()
1584                    .or_else(|| options.var(crate::env::RLM_TREE_MEMORY_BACKEND_ENV));
1585                // Under a sandbox the scope would have to wrap the sandbox's launcher: the watchdog caps the tree.
1586                tree_memory::tree_memory_backend(self.tree_memory_mb, requested.as_deref(), || {
1587                    mechanism.is_none() && tree_memory::cgroup_scope_available(tree.launcher.as_os_str())
1588                })
1589            }
1590        };
1591        let argv: Vec<std::ffi::OsString> = vec![
1592            python.clone().into(),
1593            options.runtime_path.clone().into(),
1594            "--protocol-fds".into(),
1595        ];
1596        let budget = rlm_output_budget(options.var(crate::env::RLM_OUTPUT_BYTES_ENV).as_deref());
1597        let fixed_env: Vec<(String, String)> = vec![
1598            ("NO_COLOR".into(), "1".into()),
1599            ("PYTHONDONTWRITEBYTECODE".into(), "1".into()),
1600            (crate::env::IN_KERNEL_ENV.into(), "1".into()),
1601            (
1602                crate::env::RLM_MAX_MEMORY_MB_ENV.into(),
1603                self.limits.max_memory_mb.to_string(),
1604            ),
1605            (
1606                crate::env::RLM_MAX_CPU_SECONDS_ENV.into(),
1607                self.limits.max_cpu_seconds.to_string(),
1608            ),
1609            (crate::env::RLM_OUTPUT_BYTES_ENV.into(), budget.to_string()),
1610        ];
1611        let mut command = match (mechanism, &options.sandbox) {
1612            (Some(mechanism), Some(launcher)) => sandboxed::command(sandboxed::Launch {
1613                launcher,
1614                mechanism,
1615                python: &python,
1616                argv,
1617                cwd: &options.cwd,
1618                runtime: &options.runtime_path,
1619                snapshot_path: options.snapshot_path.as_deref(),
1620                fixed_env,
1621                extra_env: &options.env,
1622                base_env: options.base_environment(),
1623            }),
1624            _ => {
1625                let mut command = match (&options.tree_memory, tree_backend) {
1626                    // systemd-run execs the kernel itself inside the capped scope.
1627                    (Some(tree), TreeMemoryBackend::Cgroup) => {
1628                        let mut command = Command::new(&tree.launcher);
1629                        command
1630                            .args(&tree_memory::cgroup_scope_command(self.tree_memory_mb)[1..])
1631                            .args(&argv);
1632                        command
1633                    }
1634                    _ => {
1635                        let mut command = Command::new(&argv[0]);
1636                        command.args(&argv[1..]);
1637                        command
1638                    }
1639                };
1640                command
1641                    .current_dir(&options.cwd)
1642                    .env_clear()
1643                    .envs(kernel_environment(options.base_environment()))
1644                    .envs(fixed_env)
1645                    .envs(options.env.iter().map(|(name, value)| (name, value)));
1646                command
1647            }
1648        };
1649        command.process_group(0).stdout(Stdio::piped()).stderr(Stdio::piped());
1650        let (protocol_out, kernel_out) = std::io::pipe().map_err(spawn_error)?;
1651        let (kernel_in, protocol_in) = std::io::pipe().map_err(spawn_error)?;
1652        command.stdin(Stdio::null());
1653        command
1654            .fd_mappings(vec![
1655                FdMapping {
1656                    parent_fd: kernel_out.into(),
1657                    child_fd: 3,
1658                },
1659                FdMapping {
1660                    parent_fd: kernel_in.into(),
1661                    child_fd: 4,
1662                },
1663            ])
1664            .map_err(|error| KernelError::new(error.to_string()))?;
1665        let mut child = if mechanism.is_some() {
1666            // Spawned (and the command dropped) on the process-lifetime spawner thread: see `sandboxed`.
1667            sandboxed::spawn(command).map_err(spawn_error)?
1668        } else {
1669            let child = command.spawn().map_err(spawn_error)?;
1670            // The command holds the kernel's ends of the protocol pipes; close them so the host sees end of file.
1671            drop(command);
1672            child
1673        };
1674        let pid = i32::try_from(child.id()).map_err(|error| KernelError::new(error.to_string()))?;
1675        let stderr = child.stderr.take();
1676        let raw_stdout = child.stdout.take();
1677        state.next_generation += 1;
1678        let id = state.next_generation;
1679        let restore_path = if restore && !state.no_auto_restore {
1680            options.snapshot_path.clone()
1681        } else {
1682            None
1683        };
1684        let host_completes_startup =
1685            restore_path.is_some() || (options.continuity.is_some() && state.continuity.crash.is_some());
1686        let (writer, lines) = mpsc::channel::<String>();
1687        let exit = Arc::new(Exit::default());
1688        let known_descendants: Arc<Mutex<HashMap<i32, u64>>> = Arc::default();
1689        state.generation = Some(Generation {
1690            id,
1691            pid,
1692            tree_backend,
1693            cgroup_dir: None,
1694            oom_kills_at_start: 0,
1695            known_descendants: known_descendants.clone(),
1696            writer,
1697            exit: exit.clone(),
1698            raw: Default::default(),
1699            output: None,
1700            pending: HashMap::new(),
1701            active_cell: None,
1702            host_requests: HashMap::new(),
1703            ready_received: false,
1704            initialized: false,
1705            startup_deadline: Instant::now() + options.startup_timeout,
1706            cell_cpu_start: None,
1707            cpu_pid: mechanism.is_none().then_some(pid),
1708            sandbox: mechanism,
1709            buffer: Vec::new(),
1710            restore_path,
1711            host_completes_startup,
1712            completing: false,
1713        });
1714        state.stderr.clear();
1715
1716        let shared = self.clone();
1717        let mut protocol_in = protocol_in;
1718        std::thread::spawn(move || {
1719            for line in lines {
1720                if let Err(error) = protocol_in
1721                    .write_all(line.as_bytes())
1722                    .and_then(|()| protocol_in.flush())
1723                {
1724                    shared.with_state(|state| shared.fail(state, id, KernelError::new(error.to_string())));
1725                    return;
1726                }
1727            }
1728        });
1729        let shared = self.clone();
1730        let channel_exit = exit.clone();
1731        std::thread::spawn(move || {
1732            if let Err(error) = read_chunks(protocol_out, |chunk| {
1733                shared.with_state(|state| shared.handle_data(state, id, chunk));
1734            }) {
1735                shared.with_state(|state| shared.fail(state, id, KernelError::new(error.to_string())));
1736                return;
1737            }
1738            if !channel_exit.wait_reaped(CLOSED_CHANNEL_GRACE) {
1739                shared.with_state(|state| {
1740                    let error = KernelError::new(format!(
1741                        "RLM kernel protocol channel (fd 3) closed{}",
1742                        state.diagnostics()
1743                    ));
1744                    shared.fail(state, id, error);
1745                });
1746            }
1747        });
1748        if let Some(stdout) = raw_stdout {
1749            let shared = self.clone();
1750            std::thread::spawn(move || {
1751                let _ = read_chunks(stdout, |chunk| {
1752                    shared.with_state(|state| handle_raw(state, id, 0, chunk));
1753                });
1754            });
1755        }
1756        if let Some(stderr) = stderr {
1757            let shared = self.clone();
1758            std::thread::spawn(move || {
1759                let _ = read_chunks(stderr, |chunk| {
1760                    shared.with_state(|state| {
1761                        if state.current(id).is_none() {
1762                            return;
1763                        }
1764                        handle_raw(state, id, 1, chunk);
1765                    });
1766                });
1767            });
1768        }
1769        if tree_backend != TreeMemoryBackend::Off {
1770            self.start_tree_memory_watchdog(id, pid, tree_backend, known_descendants, exit.clone());
1771        }
1772        let shared = self.clone();
1773        std::thread::spawn(move || {
1774            if let Some(process) = rustix::process::Pid::from_raw(pid) {
1775                while let Err(rustix::io::Errno::INTR) =
1776                    rustix::process::waitid(WaitId::Pid(process), WaitIdOptions::EXITED | WaitIdOptions::NOWAIT)
1777                {
1778                }
1779            }
1780            shared.with_state(|_| jsrt::sync::lock(&exit.state).0 = true);
1781            let status = child.wait();
1782            jsrt::sync::lock(&exit.state).1 =
1783                Some(status.as_ref().copied().unwrap_or_else(|_| ExitStatus::from_raw(0)));
1784            exit.changed.notify_all();
1785            shared.with_state(|state| {
1786                if state.current(id).is_some() {
1787                    // A cgroup OOM kill of the kernel itself arrives as SIGKILL; the scope still exists while its
1788                    // other processes live, so its memory.events says whether the tree cap did it.
1789                    let killed = status
1790                        .as_ref()
1791                        .ok()
1792                        .and_then(|status| status.signal())
1793                        .is_some_and(|signal| signal == SIGKILL || signal == SIGTERM);
1794                    let tree_kill = match state.current(id) {
1795                        Some(current) if killed => shared.tree_memory_exceeded(current).then_some(current.tree_backend),
1796                        _ => None,
1797                    };
1798                    let error = match tree_kill {
1799                        Some(backend) => {
1800                            KernelError::new(tree_memory::tree_memory_limit_message(shared.tree_memory_mb, backend))
1801                        }
1802                        None => shared.exit_error(state, status.ok()),
1803                    };
1804                    shared.fail(state, id, error);
1805                }
1806            });
1807            // Continuity: a kernel that died while serving is replaced and restored now, not at the next call.
1808            let mut state = jsrt::sync::lock(&shared.state);
1809            if std::mem::take(&mut state.continuity.restart)
1810                && !state.closed
1811                && state.generation.is_none()
1812                && let Ok(generation) = shared.ensure_started(&mut state, true)
1813            {
1814                drop(shared.wait_started(state, generation, None));
1815            }
1816        });
1817        Ok(id)
1818    }
1819
1820    fn exit_error(&self, state: &State, status: Option<ExitStatus>) -> KernelError {
1821        let mut code = status.and_then(|status| status.code());
1822        let mut signal = status.and_then(|status| status.signal());
1823        // bubblewrap reports a command killed by a signal as exit code 128 + the signal.
1824        if state.generation.as_ref().and_then(|generation| generation.sandbox) == Some(sandbox::Mechanism::Bwrap)
1825            && let Some(number) = code.filter(|code| (129..=192).contains(code))
1826        {
1827            (code, signal) = (None, Some(number - 128));
1828        }
1829        if code == Some(EXIT_MEMORY) {
1830            return KernelError::new(memory_limit_message(&self.limits));
1831        }
1832        if code == Some(EXIT_CPU) || signal == Some(SIGXCPU) {
1833            return KernelError::new(cpu_limit_message(&self.limits));
1834        }
1835        // The host's own kills happen after fail(), so a SIGKILL seen here came from elsewhere.
1836        let hint = if signal == Some(SIGKILL) {
1837            "; the system out-of-memory killer may have stopped it"
1838        } else {
1839            ""
1840        };
1841        let how = match (signal, code) {
1842            (Some(signal), _) => jsrt::process::signal_name(signal),
1843            (None, Some(code)) => code.to_string(),
1844            (None, None) => "unknown".into(),
1845        };
1846        KernelError::new(format!("RLM kernel exited ({how}){hint}{}", state.diagnostics()))
1847    }
1848
1849    /// Ends `generation` with `error`: rejects every call, aborts its cells' host work, and kills its process tree.
1850    fn fail(&self, state: &mut State, generation: u64, error: KernelError) {
1851        if state.current(generation).is_none() {
1852            return;
1853        }
1854        let Some(failed) = state.generation.take() else {
1855            return;
1856        };
1857        state.failure = Some((generation, error.clone()));
1858        if self.options.continuity.is_some() {
1859            if !state.closed {
1860                let continuity = &mut state.continuity;
1861                let unsaved_cells = continuity.cells - continuity.checkpoint_cells;
1862                let interrupted_cell = failed.active_cell.is_some();
1863                let restoring = failed.completing && !failed.initialized;
1864                match continuity.crash.as_mut() {
1865                    // Died again before reporting: keep the first cause, add what this failure adds.
1866                    Some(crash) => {
1867                        crash.interrupted_cell |= interrupted_cell;
1868                        crash.unsaved_cells = unsaved_cells;
1869                    }
1870                    None => {
1871                        continuity.crash = Some(Crash {
1872                            reason: error.message().to_string(),
1873                            interrupted_cell,
1874                            unsaved_cells,
1875                            restore_error: None,
1876                        });
1877                    }
1878                }
1879                if restoring && failed.restore_path.is_some() {
1880                    // Restoring the checkpoint killed the kernel: do not try it again.
1881                    state.no_auto_restore = true;
1882                    if let Some(crash) = state.continuity.crash.as_mut() {
1883                        crash.restore_error = Some(format!("the kernel failed while restoring: {}", error.message()));
1884                    }
1885                }
1886                state.continuity.restart = failed.initialized;
1887            }
1888        } else if failed.initialized {
1889            // A failed startup keeps reporting a bad configured snapshot on retry; once user code could have run,
1890            // an older checkpoint is never silently resurrected.
1891            state.no_auto_restore = true;
1892        }
1893        for waiter in failed.pending.values() {
1894            waiter.controller.abort(error.message());
1895        }
1896        if let Some(waiter) = failed.output.as_ref().and_then(|output| output.finished.as_ref()) {
1897            waiter.controller.abort(error.message());
1898        }
1899        state.reject_ops(&error);
1900        let known = jsrt::sync::lock(&failed.known_descendants).clone();
1901        process::kill_process_tree(
1902            failed.pid,
1903            true,
1904            !failed.exit.exited(),
1905            process::KillExtras {
1906                known: Some(&known),
1907                cgroup_dir: failed.cgroup_dir.as_deref(),
1908            },
1909        );
1910        // Dropping the generation closes the writer channel, which closes the kernel's input.
1911    }
1912
1913    fn send(&self, state: &mut State, generation: u64, line: String) -> Result<(), KernelError> {
1914        let closed = state.closed;
1915        let failure = state.failure.clone();
1916        match state.current(generation) {
1917            Some(current) if !closed => current.writer.send(line).map_err(|_| KernelError::new(NOT_RUNNING)),
1918            _ => Err(match failure {
1919                Some((failed, error)) if failed == generation => error,
1920                _ => KernelError::new(NOT_RUNNING),
1921            }),
1922        }
1923    }
1924
1925    fn handle_data(self: &Arc<Self>, state: &mut State, generation: u64, chunk: &[u8]) {
1926        let mut offset = 0;
1927        while offset < chunk.len() {
1928            let Some(current) = state.current(generation) else {
1929                return;
1930            };
1931            let newline = chunk[offset..]
1932                .iter()
1933                .position(|&byte| byte == b'\n')
1934                .map(|index| offset + index);
1935            let end = newline.unwrap_or(chunk.len());
1936            // Bounded before decoding, including unterminated frames split across many chunks.
1937            if current.buffer.len() + (end - offset) > MAX_FRAME_BYTES {
1938                let error = format!("RLM kernel protocol frame exceeds 1 MiB ({MAX_FRAME_BYTES} bytes)");
1939                self.fail(state, generation, KernelError::new(error));
1940                return;
1941            }
1942            current.buffer.extend_from_slice(&chunk[offset..end]);
1943            if newline.is_none() {
1944                return;
1945            }
1946            let line = std::mem::take(&mut current.buffer);
1947            let handled = match std::str::from_utf8(&line) {
1948                Ok(text) => self.handle_line(state, generation, text),
1949                Err(_) => Err("The encoded data was not valid for encoding utf-8".to_string()),
1950            };
1951            if let Err(message) = handled {
1952                let error = KernelError::new(format!("RLM kernel invalid protocol frame: {message}"));
1953                self.fail(state, generation, error);
1954            }
1955            offset = end + 1;
1956        }
1957    }
1958
1959    fn handle_line(self: &Arc<Self>, state: &mut State, generation: u64, line: &str) -> Result<(), String> {
1960        let parsed: Value =
1961            serde_json::from_str(&replace_lone_surrogates(line)).map_err(|error| format!("invalid JSON: {error}"))?;
1962        let Value::Object(frame) = parsed else {
1963            return Err("protocol frame must be a JSON object".into());
1964        };
1965        let Some(Value::String(event)) = frame.get("event") else {
1966            return Err("protocol frame is missing event".into());
1967        };
1968        let Some(current) = state.current(generation) else {
1969            return Ok(());
1970        };
1971        match event.as_str() {
1972            "ready" => {
1973                if current.ready_received {
1974                    return Err("duplicate ready event".into());
1975                }
1976                if frame.contains_key("id") {
1977                    return Err("ready event cannot have an id".into());
1978                }
1979                current.ready_received = true;
1980                current.initialized = !current.host_completes_startup;
1981                // Resolve the scope and its OOM baseline now that systemd-run has exec'd the runtime.
1982                if current.tree_backend == TreeMemoryBackend::Cgroup {
1983                    self.tree_memory_exceeded(current);
1984                }
1985                Ok(())
1986            }
1987            "host_request" => {
1988                let id = match frame.get("id") {
1989                    Some(Value::String(id)) if !id.is_empty() => id.clone(),
1990                    _ => return Err("host_request requires an id".into()),
1991                };
1992                let kind = match frame.get("type") {
1993                    Some(Value::String(kind)) if !kind.is_empty() => kind.clone(),
1994                    _ => return Err("host_request requires a type".into()),
1995                };
1996                let payload = match frame.get("payload") {
1997                    None => Map::new(),
1998                    Some(Value::Object(payload)) => payload.clone(),
1999                    Some(_) => return Err("host_request payload must be an object".into()),
2000                };
2001                let cell = match frame.get("cell") {
2002                    None => None,
2003                    Some(Value::String(cell)) if !cell.is_empty() => Some(cell.clone()),
2004                    Some(_) => return Err("host_request cell must be a request id".into()),
2005                };
2006                self.handle_host_request(
2007                    state,
2008                    generation,
2009                    HostRequest {
2010                        id,
2011                        kind,
2012                        payload,
2013                        cell,
2014                    },
2015                );
2016                Ok(())
2017            }
2018            "host_cancel" => {
2019                let Some(Value::String(id)) = frame.get("id") else {
2020                    return Err("host_cancel requires an id".into());
2021                };
2022                // A request that already finished (or was refused) has nothing to cancel.
2023                if let Some(request) = current.host_requests.get(id) {
2024                    request.abort("RLM host request cancelled");
2025                }
2026                Ok(())
2027            }
2028            "stdout" | "stderr" | "result" | "error" | "done" => {
2029                let id = match frame.get("id") {
2030                    Some(Value::String(id)) if !id.is_empty() => id.clone(),
2031                    _ => return Err(format!("{event} event requires an id")),
2032                };
2033                let status = frame.get("status").map(Value::as_str);
2034                if event == "done" && status.is_some_and(|status| status != Some("ok") && status != Some("error")) {
2035                    return Err("done event has invalid status".into());
2036                }
2037                // A subprocess killed by the scope's OOM killer may let the cell finish; report the cap, not a stray
2038                // error.
2039                if event == "done"
2040                    && current.tree_backend == TreeMemoryBackend::Cgroup
2041                    && current.pending.contains_key(&id)
2042                    && self.tree_memory_exceeded(current)
2043                {
2044                    let message = tree_memory::tree_memory_limit_message(self.tree_memory_mb, current.tree_backend);
2045                    self.fail(state, generation, KernelError::new(message));
2046                    return Ok(());
2047                }
2048                route_execution_frame(state, generation, &id, event, &frame)
2049            }
2050            other => Err(format!("unknown protocol event: {other}")),
2051        }
2052    }
2053
2054    fn handle_host_request(self: &Arc<Self>, state: &mut State, generation: u64, request: HostRequest) {
2055        let HostRequest {
2056            id,
2057            kind,
2058            payload,
2059            cell,
2060        } = request;
2061        let Some(current) = state.current(generation) else {
2062            return;
2063        };
2064        if current.host_requests.contains_key(&id) {
2065            let error = KernelError::new(format!("RLM kernel protocol duplicate host request id: {id}"));
2066            self.fail(state, generation, error);
2067            return;
2068        }
2069        // A request names the cell that made it; one from a task a finished cell left running is refused, so it never
2070        // runs under the next cell's handler, signal and output. A request without a cell belongs to the active one.
2071        let waiter = match &cell {
2072            Some(cell) if current.active_cell.as_ref() != Some(cell) => {
2073                self.send_host_error(state, generation, &id, "RLM host request from a cell that has finished");
2074                return;
2075            }
2076            Some(cell) => current.pending.get(cell),
2077            None => current.active_cell.as_ref().and_then(|cell| current.pending.get(cell)),
2078        };
2079        let Some(waiter) = waiter else {
2080            self.send_host_error(state, generation, &id, "RLM host request has no active cell");
2081            return;
2082        };
2083        if current.host_requests.len() >= MAX_HOST_REQUESTS {
2084            let message = format!("RLM host request concurrency exceeds {MAX_HOST_REQUESTS}");
2085            self.send_host_error(state, generation, &id, &message);
2086            return;
2087        }
2088        let handler = waiter.handler.clone().unwrap_or_else(|| self.handler.clone());
2089        let cell_signal = waiter.controller.clone();
2090        // The request's signal ends with its cell for good, not only while the handler runs: handlers keep it
2091        // (a shell job remembers the signal of the request that started it, to tell its cell ending from a cancel).
2092        let signal = Signal::new();
2093        let request = signal.clone();
2094        if cell_signal.on_abort(move |reason| request.abort(reason)).is_none() {
2095            signal.abort(cell_signal.reason().unwrap_or_default());
2096        }
2097        current.host_requests.insert(id.clone(), signal.clone());
2098        let shared = self.clone();
2099        std::thread::spawn(move || {
2100            let result =
2101                std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| handler.handle(&kind, payload, &signal)))
2102                    .unwrap_or_else(|_| Err(format!("RLM host handler for {kind} panicked")));
2103            // Every request is answered while its kernel lives, also after its cell completed or it was cancelled:
2104            // whatever still awaits it gets the outcome instead of waiting forever, and Python drops a reply nobody
2105            // waits for.
2106            shared.with_state(|state| {
2107                if state.current(generation).is_some() {
2108                    match result {
2109                        Ok(value) => {
2110                            let reply = object([
2111                                ("request", Value::from("host_reply")),
2112                                ("id", Value::from(id.clone())),
2113                                ("payload", value),
2114                            ]);
2115                            shared.send_host_reply(state, generation, &id, &reply);
2116                        }
2117                        Err(message) => shared.send_host_error(state, generation, &id, &message),
2118                    }
2119                }
2120                if let Some(current) = state.current(generation) {
2121                    current.host_requests.remove(&id);
2122                }
2123            });
2124        });
2125    }
2126
2127    fn send_host_reply(&self, state: &mut State, generation: u64, id: &str, reply: &Map<String, Value>) {
2128        let sent = encode_frame(reply).and_then(|line| self.send(state, generation, line));
2129        if let Err(error) = sent {
2130            if state.current(generation).is_none() {
2131                return;
2132            }
2133            self.send_host_error(state, generation, id, error.message());
2134        }
2135    }
2136
2137    fn send_host_error(&self, state: &mut State, generation: u64, id: &str, message: &str) {
2138        if state.current(generation).is_none() {
2139            return;
2140        }
2141        let reply = object([
2142            ("request", Value::from("host_reply")),
2143            ("id", Value::from(id)),
2144            ("error", Value::from(message)),
2145        ]);
2146        if let Err(error) = encode_frame(&reply).and_then(|line| self.send(state, generation, line)) {
2147            self.fail(state, generation, error);
2148        }
2149    }
2150}
2151
2152/// The watchdog's measure of `root`'s tree; remembers the descendants it saw (pid to start time) so they are killed
2153/// with the tree even after they left it.
2154fn watchdog_tree_bytes(root: i32, known: &Mutex<HashMap<i32, u64>>) -> u64 {
2155    let (bytes, descendants) = tree_memory::tree_memory_usage(root);
2156    let mut known = jsrt::sync::lock(known);
2157    known.retain(|pid, start| process::process_stat(*pid).is_some_and(|stat| stat.start_time == *start));
2158    for pid in descendants {
2159        if let Some(stat) = process::process_stat(pid) {
2160            known.insert(pid, stat.start_time);
2161        }
2162    }
2163    bytes
2164}
2165
2166fn entry_reject(state: &mut State, op: u64, error: KernelError) {
2167    if let Some(entry) = state.ops.get_mut(&op) {
2168        entry.outcome.get_or_insert(Err(error));
2169    }
2170}
2171
2172fn read_chunks(mut stream: impl Read, mut handle: impl FnMut(&[u8])) -> std::io::Result<()> {
2173    let mut buffer = vec![0u8; 64 * 1024];
2174    loop {
2175        match stream.read(&mut buffer) {
2176            Ok(0) => return Ok(()),
2177            Ok(read) => handle(&buffer[..read]),
2178            Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {}
2179            Err(error) => return Err(error),
2180        }
2181    }
2182}
2183
2184fn route_execution_frame(
2185    state: &mut State,
2186    generation: u64,
2187    request_id: &str,
2188    event: &str,
2189    frame: &Map<String, Value>,
2190) -> Result<(), String> {
2191    let Some(current) = state.current(generation) else {
2192        return Ok(());
2193    };
2194    let Some(waiter) = current.pending.get_mut(request_id) else {
2195        return Err(format!("protocol frame references unknown request id: {request_id}"));
2196    };
2197    let result = &mut waiter.state.result;
2198    match event {
2199        "stdout" => append_bounded(&mut result.stdout, &frame_string(frame.get("text"))),
2200        "stderr" => append_bounded(&mut result.stderr, &frame_string(frame.get("text"))),
2201        "result" => result.result = Some(frame_string(frame.get("result"))),
2202        "error" => {
2203            result.status = Status::Error;
2204            let traceback = match frame.get("traceback") {
2205                Some(Value::Array(lines)) => lines
2206                    .iter()
2207                    .filter_map(|line| line.as_str().map(String::from))
2208                    .collect(),
2209                _ => Vec::new(),
2210            };
2211            result.error = Some(CellError {
2212                ename: frame_string(frame.get("ename")),
2213                evalue: frame_string(frame.get("evalue")),
2214                traceback,
2215            });
2216        }
2217        _ => {
2218            if frame.get("status") == Some(&Value::from("error")) {
2219                result.status = Status::Error;
2220            }
2221            // Snapshot and restore failures arrive only on the done frame.
2222            if result.status == Status::Error
2223                && result.error.is_none()
2224                && let Some(error) = frame.get("error").filter(|error| !error.is_null())
2225            {
2226                result.error = Some(CellError {
2227                    ename: "RuntimeError".into(),
2228                    evalue: frame_string(Some(error)),
2229                    traceback: Vec::new(),
2230                });
2231            }
2232            let Some(mut waiter) = current.pending.remove(request_id) else {
2233                return Ok(());
2234            };
2235            if let Some(report) = snapshot::snapshot_report(frame.get("snapshot")) {
2236                waiter.state.snapshot = Some(report);
2237            }
2238            if let Some(report) = snapshot::restore_report(frame.get("restore")) {
2239                waiter.state.restore = Some(report);
2240            }
2241            if current.active_cell.as_deref() == Some(request_id) {
2242                current.active_cell = None;
2243            }
2244            waiter.controller.abort("RLM cell completed");
2245            let Some(output) = current.output.as_mut().filter(|output| output.request_id == request_id) else {
2246                state.resolve(waiter);
2247                return Ok(());
2248            };
2249            // fd 1 and fd 2 are separate pipes from the protocol: wait (bounded) for the flush markers the runtime
2250            // wrote after the cell's raw output, so the result is complete.
2251            let flush = frame.get("flush").and_then(Value::as_object);
2252            let flushed = |name: &str| flush.and_then(|flush| flush.get(name)) == Some(&Value::Bool(true));
2253            output.awaiting = Some([flushed("stdout"), flushed("stderr")]);
2254            output.deadline = Some(Instant::now() + RAW_FLUSH_TIMEOUT);
2255            output.finished = Some(waiter);
2256            finish_cell_if_flushed(state, generation);
2257        }
2258    }
2259    Ok(())
2260}
2261
2262/// The cell owns fd 1 and fd 2 output from now on, and reports the output from outside a cell that came before it.
2263fn take_raw_output(generation: &mut Generation, request_id: &str) {
2264    let orphan = generation.raw.each_mut().map(|stream| {
2265        let text = orphan_text(&stream.orphan, stream.orphan_dropped);
2266        stream.orphan.clear();
2267        stream.orphan_dropped = false;
2268        text
2269    });
2270    generation.output = Some(CellOutput {
2271        request_id: request_id.to_string(),
2272        marker: flush_marker(request_id),
2273        captures: Default::default(),
2274        orphan,
2275        awaiting: None,
2276        deadline: None,
2277        finished: None,
2278    });
2279}
2280
2281/// Raw bytes from fd 1 (`index` 0) or fd 2 (1). They belong to the cell that owns the output until that cell's flush
2282/// marker arrives on the stream; anything else is output from outside a cell.
2283fn handle_raw(state: &mut State, generation: u64, index: usize, chunk: &[u8]) {
2284    let State {
2285        generation: current,
2286        stderr,
2287        ..
2288    } = state;
2289    let Some(current) = current.as_mut().filter(|current| current.id == generation) else {
2290        return;
2291    };
2292    let Generation { raw, output, .. } = current;
2293    let stream = &mut raw[index];
2294    let mut data = std::mem::take(&mut stream.held);
2295    data.extend_from_slice(chunk);
2296    let mut data = &data[..];
2297    if !stream.started {
2298        // Before the startup marker: stderr only feeds the diagnostic tail, stdout is dropped.
2299        let startup = flush_marker("ready");
2300        let found = find(data, &startup);
2301        let end = found.unwrap_or_else(|| data.len() - marker_prefix_len(data, &startup));
2302        if index == 1 {
2303            keep_tail(stderr, &String::from_utf8_lossy(&data[..end]));
2304        }
2305        let Some(found) = found else {
2306            stream.held = data[end..].to_vec();
2307            return;
2308        };
2309        stream.started = true;
2310        data = &data[found + startup.len()..];
2311    }
2312    let Some(cell) = output.as_mut().filter(|cell| !cell.captures[index].marker_seen) else {
2313        route_raw(stream, stderr, index, data, None);
2314        return;
2315    };
2316    if let Some(found) = find(data, &cell.marker) {
2317        route_raw(stream, stderr, index, &data[..found], Some(&mut cell.captures[index]));
2318        cell.captures[index].marker_seen = true;
2319        route_raw(stream, stderr, index, &data[found + cell.marker.len()..], None);
2320        finish_cell_if_flushed(state, generation);
2321        return;
2322    }
2323    let held = marker_prefix_len(data, &cell.marker);
2324    stream.held = data[data.len() - held..].to_vec();
2325    route_raw(
2326        stream,
2327        stderr,
2328        index,
2329        &data[..data.len() - held],
2330        Some(&mut cell.captures[index]),
2331    );
2332}
2333
2334fn route_raw(
2335    stream: &mut RawStream,
2336    stderr: &mut String,
2337    index: usize,
2338    bytes: &[u8],
2339    capture: Option<&mut RawCapture>,
2340) {
2341    if bytes.is_empty() {
2342        return;
2343    }
2344    let text = stream.decoder.decode(bytes);
2345    if index == 1 {
2346        keep_tail(stderr, &text);
2347    }
2348    if let Some(capture) = capture {
2349        capture.push(&text);
2350        return;
2351    }
2352    let used = jsrt::utf16::len(&stream.orphan);
2353    let room = MAX_ORPHAN_CHARS.saturating_sub(used);
2354    if jsrt::utf16::len(&text) > room {
2355        stream.orphan_dropped = true;
2356    }
2357    if room > 0 {
2358        let kept = jsrt::utf16::head(&text, room);
2359        stream.orphan.push_str(kept);
2360    }
2361}
2362
2363/// Resolves the finished cell once every flush marker its done frame announced has arrived.
2364fn finish_cell_if_flushed(state: &mut State, generation: u64) {
2365    let Some(output) = state.current(generation).and_then(|current| current.output.as_ref()) else {
2366        return;
2367    };
2368    let (Some(_), Some(awaiting)) = (&output.finished, output.awaiting) else {
2369        return;
2370    };
2371    if (0..2).any(|index| awaiting[index] && !output.captures[index].marker_seen) {
2372        return;
2373    }
2374    finish_cell(state, generation);
2375}
2376
2377/// Combines the output from outside a cell, the cell's captured frames and its raw output, and resolves the cell.
2378fn finish_cell(state: &mut State, generation: u64) {
2379    let State {
2380        generation: current,
2381        stderr,
2382        ..
2383    } = state;
2384    let Some(current) = current.as_mut().filter(|current| current.id == generation) else {
2385        return;
2386    };
2387    let Some(output) = current.output.take() else {
2388        return;
2389    };
2390    // A held partial marker was output after all.
2391    for (index, stream) in current.raw.iter_mut().enumerate() {
2392        if stream.started {
2393            let held = std::mem::take(&mut stream.held);
2394            route_raw(stream, stderr, index, &held, None);
2395        }
2396    }
2397    let Some(mut waiter) = output.finished else {
2398        return;
2399    };
2400    let [orphan_out, orphan_err] = &output.orphan;
2401    waiter.state.result.stdout = format!(
2402        "{orphan_out}{}{}",
2403        waiter.state.result.stdout,
2404        output.captures[0].text(1)
2405    );
2406    waiter.state.result.stderr = format!(
2407        "{orphan_err}{}{}",
2408        waiter.state.result.stderr,
2409        output.captures[1].text(2)
2410    );
2411    state.resolve(waiter);
2412}