mod config;
pub mod continuity;
pub mod env;
pub mod ops;
mod output;
mod process;
mod sandboxed;
pub mod snapshot;
pub mod snapshot_store;
pub mod store;
pub mod tree_memory;
pub use continuity::{ContinuityPolicy, ContinuityReport, LostName};
pub use process::kill_command_group;
pub use snapshot::{
RestoreReport, SnapshotReport, load_or_create_snapshot_key, process_snapshot_key, sign_snapshot, verify_snapshot,
};
pub use tree_memory::TreeMemoryBackend;
pub use config::{KernelResourceLimits, LimitOverrides, kernel_environment, kernel_resource_limits, rlm_output_budget};
pub const ROOT_LANE: &str = "main";
pub fn sees(viewer: &str, owner: &str) -> bool {
viewer == ROOT_LANE || viewer == owner
}
use command_fds::{CommandFdExt, FdMapping};
use continuity::{Crash, recovery_report};
use output::{
MAX_FRAME_BYTES, MAX_ORPHAN_CHARS, RawCapture, Utf8Stream, append_bounded, find, flush_marker, keep_tail,
marker_prefix_len, orphan_text,
};
use rustix::process::{WaitId, WaitIdOptions};
use serde::Serialize;
use serde_json::{Map, Value};
use std::collections::{HashMap, VecDeque};
use std::io::{Read, Write};
use std::os::unix::fs::OpenOptionsExt;
use std::os::unix::process::{CommandExt, ExitStatusExt};
use std::path::{Path, PathBuf};
use std::process::{Command, ExitStatus, Stdio};
use std::sync::mpsc;
use std::sync::{Arc, Condvar, Mutex, MutexGuard, PoisonError, Weak};
use std::time::{Duration, Instant};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct KernelError {
message: String,
}
impl KernelError {
pub fn new(message: impl Into<String>) -> Self {
Self {
message: message.into(),
}
}
pub fn message(&self) -> &str {
&self.message
}
}
impl std::fmt::Display for KernelError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(&self.message)
}
}
impl std::error::Error for KernelError {}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum Status {
Ok,
Error,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct CellError {
pub ename: String,
pub evalue: String,
pub traceback: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct ExecutionResult {
pub status: Status,
pub stdout: String,
pub stderr: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub result: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<CellError>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct KernelReply {
#[serde(flatten)]
pub result: ExecutionResult,
#[serde(skip_serializing_if = "Option::is_none")]
pub snapshot: Option<SnapshotReport>,
#[serde(skip_serializing_if = "Option::is_none")]
pub restore: Option<RestoreReport>,
#[serde(skip_serializing_if = "Option::is_none")]
pub recovery: Option<ContinuityReport>,
}
impl KernelReply {
fn from_result(result: ExecutionResult) -> Self {
Self {
result,
snapshot: None,
restore: None,
recovery: None,
}
}
}
type Listener = Box<dyn FnOnce(String) + Send>;
#[derive(Default)]
struct SignalState {
reason: Option<String>,
listeners: Vec<(u64, Listener)>,
next: u64,
}
#[derive(Clone, Default)]
pub struct Signal {
inner: Arc<(Mutex<SignalState>, Condvar)>,
}
impl Signal {
pub fn new() -> Self {
Self::default()
}
pub fn abort(&self, reason: impl Into<String>) {
let reason = reason.into();
let listeners = {
let mut state = jsrt::sync::lock(&self.inner.0);
if state.reason.is_some() {
return;
}
state.reason = Some(reason.clone());
std::mem::take(&mut state.listeners)
};
self.inner.1.notify_all();
for (_, listener) in listeners {
listener(reason.clone());
}
}
pub fn reason(&self) -> Option<String> {
jsrt::sync::lock(&self.inner.0).reason.clone()
}
pub fn is_aborted(&self) -> bool {
jsrt::sync::lock(&self.inner.0).reason.is_some()
}
pub fn wait(&self, timeout: Duration) -> bool {
let state = jsrt::sync::lock(&self.inner.0);
let (state, _) = self
.inner
.1
.wait_timeout_while(state, timeout, |state| state.reason.is_none())
.unwrap_or_else(PoisonError::into_inner);
state.reason.is_some()
}
pub fn on_abort(&self, listener: impl FnOnce(String) + Send + 'static) -> Option<u64> {
self.listen(Box::new(listener)).ok()
}
pub fn forget(&self, id: u64) {
jsrt::sync::lock(&self.inner.0)
.listeners
.retain(|(listener, _)| *listener != id);
}
pub fn link(&self, listener: impl FnOnce(String) + Send + 'static) -> Link {
let id = match self.listen(Box::new(listener)) {
Ok(id) => Some(id),
Err((listener, reason)) => {
listener(reason);
None
}
};
Link {
signal: self.clone(),
id,
}
}
fn listen(&self, listener: Listener) -> Result<u64, (Listener, String)> {
let mut state = jsrt::sync::lock(&self.inner.0);
if let Some(reason) = state.reason.clone() {
return Err((listener, reason));
}
state.next += 1;
let id = state.next;
state.listeners.push((id, listener));
Ok(id)
}
pub fn run_child(
&self,
command: &mut std::process::Command,
input: Option<&[u8]>,
bound: signals::child::Bound,
) -> std::io::Result<signals::child::Output> {
let running = signals::child::start(command, input, bound)?;
let watched = running.watched().clone();
let _link = self.link(move |_| watched.cancel());
Ok(running.finish())
}
}
pub struct Link {
signal: Signal,
id: Option<u64>,
}
impl Drop for Link {
fn drop(&mut self) {
if let Some(id) = self.id {
self.signal.forget(id);
}
}
}
pub trait HostHandler: Send + Sync {
fn handle(&self, kind: &str, payload: Map<String, Value>, signal: &Signal) -> Result<Value, String>;
}
impl<F> HostHandler for F
where
F: Fn(&str, Map<String, Value>, &Signal) -> Result<Value, String> + Send + Sync,
{
fn handle(&self, kind: &str, payload: Map<String, Value>, signal: &Signal) -> Result<Value, String> {
self(kind, payload, signal)
}
}
pub struct EchoHandler;
impl HostHandler for EchoHandler {
fn handle(&self, kind: &str, payload: Map<String, Value>, _signal: &Signal) -> Result<Value, String> {
if kind == "test.fail" {
return Err(format!("refused {kind}"));
}
let mut reply = Map::new();
reply.insert("type".into(), Value::String(kind.into()));
reply.insert("payload".into(), Value::Object(payload));
Ok(Value::Object(reply))
}
}
pub struct KernelOptions {
pub cwd: PathBuf,
pub runtime_path: PathBuf,
pub python: Option<PathBuf>,
pub startup_timeout: Duration,
pub limits: LimitOverrides,
pub env: Vec<(String, String)>,
pub snapshot_path: Option<PathBuf>,
pub snapshot_key: Option<Vec<u8>>,
pub continuity: Option<ContinuityPolicy>,
pub sandbox: Option<Arc<sandbox::Launcher>>,
pub tree_memory: Option<TreeMemoryOptions>,
pub base_env: Option<Vec<(String, String)>>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TreeMemoryOptions {
pub max_mb: Option<u64>,
pub backend: Option<String>,
pub launcher: PathBuf,
}
impl Default for TreeMemoryOptions {
fn default() -> Self {
Self {
max_mb: None,
backend: None,
launcher: "systemd-run".into(),
}
}
}
impl KernelOptions {
pub fn new(cwd: impl Into<PathBuf>, runtime_path: impl Into<PathBuf>) -> Self {
Self {
cwd: cwd.into(),
runtime_path: runtime_path.into(),
python: None,
startup_timeout: Duration::from_secs(10),
limits: LimitOverrides::default(),
env: Vec::new(),
snapshot_path: None,
snapshot_key: None,
continuity: None,
sandbox: None,
tree_memory: None,
base_env: None,
}
}
pub fn var(&self, name: &str) -> Option<String> {
match &self.base_env {
Some(env) => env.iter().find(|(key, _)| key == name).map(|(_, value)| value.clone()),
None => std::env::var(name).ok(),
}
}
fn base_environment(&self) -> Vec<(std::ffi::OsString, std::ffi::OsString)> {
match &self.base_env {
Some(env) => env.iter().map(|(key, value)| (key.into(), value.into())).collect(),
None => std::env::vars_os().collect(),
}
}
}
const EXIT_MEMORY: i32 = 86;
const EXIT_CPU: i32 = 87;
const SIGKILL: i32 = 9;
const SIGTERM: i32 = 15;
const SIGXCPU: i32 = 24;
const MAX_HOST_REQUESTS: usize = 16;
const RAW_FLUSH_TIMEOUT: Duration = Duration::from_secs(5);
const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(1);
const CLOSED_CHANNEL_GRACE: Duration = Duration::from_millis(200);
const WATCHDOG_INTERVAL: Duration = Duration::from_millis(500);
const PLACEHOLDER_ID: &str = "00000000-0000-0000-0000-000000000000";
const SHUT_DOWN: &str = "RLM kernel is shut down";
const NOT_RUNNING: &str = "RLM kernel is not running";
fn memory_limit_message(limits: &KernelResourceLimits) -> String {
format!(
"RLM kernel exceeded its memory limit ({} MiB per process, ULTRON_RLM_MAX_MEMORY_MB) and was stopped; the next cell starts a fresh kernel without earlier Python state",
limits.max_memory_mb
)
}
fn cpu_limit_message(limits: &KernelResourceLimits) -> String {
format!(
"RLM kernel exceeded its CPU limit ({} CPU-seconds per cell, ULTRON_RLM_MAX_CPU_SECONDS) and was stopped; the next cell starts a fresh kernel without earlier Python state",
limits.max_cpu_seconds
)
}
fn frame_string(value: Option<&Value>) -> String {
match value {
None | Some(Value::Null) => String::new(),
Some(Value::String(text)) => text.clone(),
Some(other) => other.to_string(),
}
}
fn encode_frame(frame: &Map<String, Value>) -> Result<String, KernelError> {
let line = serde_json::to_string(frame)
.map_err(|error| KernelError::new(format!("RLM kernel cannot encode outbound frame: {error}")))?;
if line.len() > MAX_FRAME_BYTES {
return Err(KernelError::new(format!(
"RLM kernel outbound frame exceeds 1 MiB ({MAX_FRAME_BYTES} bytes)"
)));
}
Ok(line + "\n")
}
fn object(entries: impl IntoIterator<Item = (&'static str, Value)>) -> Map<String, Value> {
entries
.into_iter()
.map(|(key, value)| (key.to_string(), value))
.collect()
}
fn random_uuid() -> Result<String, KernelError> {
let mut bytes = [0u8; 16];
getrandom::fill(&mut bytes).map_err(|error| KernelError::new(format!("RLM kernel request id: {error}")))?;
Ok(jsrt::random::uuid_v4_from(bytes))
}
fn replace_lone_surrogates(line: &str) -> std::borrow::Cow<'_, str> {
if !line.contains("\\u") {
return line.into();
}
let hex = |index: usize| -> Option<u16> {
let digits = line.get(index..index + 6)?.strip_prefix("\\u")?;
u16::from_str_radix(digits, 16).ok()
};
let bytes = line.as_bytes();
let mut text = String::with_capacity(line.len());
let (mut index, mut copied) = (0, 0);
while index < bytes.len() {
if bytes[index] != b'\\' {
index += 1;
continue;
}
match hex(index) {
Some(0xd800..=0xdbff) if matches!(hex(index + 6), Some(0xdc00..=0xdfff)) => index += 12,
Some(0xd800..=0xdfff) => {
text.push_str(&line[copied..index]);
text.push_str("\\ufffd");
index += 6;
copied = index;
}
_ => index += 2,
}
}
if copied == 0 {
return line.into();
}
text.push_str(&line[copied..]);
text.into()
}
#[derive(Default)]
struct Exit {
state: Mutex<(bool, Option<ExitStatus>)>,
changed: Condvar,
}
impl Exit {
fn exited(&self) -> bool {
jsrt::sync::lock(&self.state).0
}
fn wait_reaped(&self, timeout: Duration) -> bool {
let state = jsrt::sync::lock(&self.state);
let (state, _) = self
.changed
.wait_timeout_while(state, timeout, |state| state.1.is_none())
.unwrap_or_else(PoisonError::into_inner);
state.1.is_some()
}
}
#[derive(Default)]
struct RawStream {
decoder: Utf8Stream,
held: Vec<u8>,
orphan: String,
orphan_dropped: bool,
started: bool,
}
struct Waiter {
op: u64,
state: KernelReply,
controller: Signal,
handler: Option<Arc<dyn HostHandler>>,
}
struct CellOutput {
request_id: String,
marker: Vec<u8>,
captures: [RawCapture; 2],
orphan: [String; 2],
awaiting: Option<[bool; 2]>,
deadline: Option<Instant>,
finished: Option<Waiter>,
}
struct Generation {
id: u64,
pid: i32,
tree_backend: TreeMemoryBackend,
cgroup_dir: Option<PathBuf>,
oom_kills_at_start: u64,
known_descendants: Arc<Mutex<HashMap<i32, u64>>>,
writer: mpsc::Sender<String>,
exit: Arc<Exit>,
raw: [RawStream; 2],
output: Option<CellOutput>,
pending: HashMap<String, Waiter>,
active_cell: Option<String>,
host_requests: HashMap<String, Signal>,
ready_received: bool,
initialized: bool,
startup_deadline: Instant,
cell_cpu_start: Option<f64>,
cpu_pid: Option<i32>,
sandbox: Option<sandbox::Mechanism>,
buffer: Vec<u8>,
restore_path: Option<PathBuf>,
host_completes_startup: bool,
completing: bool,
}
struct HostRequest {
id: String,
kind: String,
payload: Map<String, Value>,
cell: Option<String>,
}
struct Op {
running: bool,
outcome: Option<Result<KernelReply, KernelError>>,
cell: Option<Signal>,
handler: Option<Arc<dyn HostHandler>>,
recovery: Option<ContinuityReport>,
}
#[derive(Default)]
struct Continuity {
checkpoint: Option<SnapshotReport>,
cells: u64,
checkpoint_cells: u64,
last_checkpoint: Option<Instant>,
dirty: bool,
snapshot_queued: bool,
idle_flush_scheduled: bool,
crash: Option<Crash>,
notice: Option<ContinuityReport>,
last_recovery: Option<ContinuityReport>,
restart: bool,
}
#[derive(Default)]
struct State {
generation: Option<Generation>,
next_generation: u64,
failure: Option<(u64, KernelError)>,
closed: bool,
stderr: String,
ops: HashMap<u64, Op>,
queue: VecDeque<u64>,
next_op: u64,
no_auto_restore: bool,
continuity: Continuity,
}
impl State {
fn current(&mut self, generation: u64) -> Option<&mut Generation> {
self.generation.as_mut().filter(|current| current.id == generation)
}
fn diagnostics(&self) -> String {
if self.stderr.is_empty() {
String::new()
} else {
format!(": {}", self.stderr)
}
}
fn resolve(&mut self, waiter: Waiter) {
if let Some(op) = self.ops.get_mut(&waiter.op)
&& op.outcome.is_none()
{
op.outcome = Some(Ok(waiter.state));
}
}
fn reject_ops(&mut self, error: &KernelError) {
for op in self.ops.values_mut() {
op.outcome.get_or_insert_with(|| Err(error.clone()));
}
self.queue.clear();
}
}
struct Shared {
options: KernelOptions,
handler: Arc<dyn HostHandler>,
limits: KernelResourceLimits,
tree_memory_mb: u64,
snapshot_key: Vec<u8>,
state: Mutex<State>,
changed: Condvar,
}
pub struct Kernel {
shared: Arc<Shared>,
}
impl Kernel {
pub fn new(options: KernelOptions, handler: Arc<dyn HostHandler>) -> Result<Self, KernelError> {
let limits = kernel_resource_limits(|name| options.var(name), options.limits)?;
if options.startup_timeout.is_zero() || options.startup_timeout > Duration::from_millis(2_147_483_647) {
return Err(KernelError::new(
"RLM startupTimeoutMs must be a positive, finite timer duration",
));
}
if options.continuity.is_some() && options.snapshot_path.is_none() {
return Err(KernelError::new("RLM kernel continuity requires a snapshot path"));
}
let snapshot_key = options.snapshot_key.clone().unwrap_or_else(process_snapshot_key);
let tree_memory_mb = match &options.tree_memory {
Some(tree) => tree_memory::kernel_tree_memory_limit(
options.var(crate::env::RLM_MAX_TREE_MEMORY_MB_ENV).as_deref(),
limits.max_memory_mb,
tree.max_mb,
)?,
None => 0,
};
Ok(Self {
shared: Arc::new(Shared {
options,
handler,
limits,
tree_memory_mb,
snapshot_key,
state: Mutex::new(State::default()),
changed: Condvar::new(),
}),
})
}
pub fn limits(&self) -> KernelResourceLimits {
self.shared.limits
}
pub fn is_running(&self) -> bool {
let state = jsrt::sync::lock(&self.shared.state);
!state.closed
&& state
.generation
.as_ref()
.is_some_and(|generation| generation.initialized)
}
pub fn stderr_tail(&self) -> String {
jsrt::sync::lock(&self.shared.state).stderr.clone()
}
pub fn pending_calls(&self) -> usize {
jsrt::sync::lock(&self.shared.state).queue.len()
}
pub fn pid(&self) -> Option<u32> {
let state = jsrt::sync::lock(&self.shared.state);
state
.generation
.as_ref()
.and_then(|generation| u32::try_from(generation.pid).ok())
}
pub fn memory_usage(&self) -> Option<(u64, Option<u64>)> {
let pid = {
let state = jsrt::sync::lock(&self.shared.state);
if state.closed {
return None;
}
state.generation.as_ref()?.pid
};
if !cfg!(target_os = "linux") {
return None;
}
let cap = self.shared.tree_memory_mb;
Some((
tree_memory::tree_memory_usage(pid).0,
(cap > 0).then(|| cap * 1024 * 1024),
))
}
pub fn tree_memory_backend(&self) -> Option<TreeMemoryBackend> {
let state = jsrt::sync::lock(&self.shared.state);
state.generation.as_ref().map(|generation| generation.tree_backend)
}
pub fn sandbox(&self) -> Option<sandbox::Mechanism> {
let state = jsrt::sync::lock(&self.shared.state);
state.generation.as_ref().and_then(|generation| generation.sandbox)
}
pub fn start(&self) -> Result<(), KernelError> {
let mut state = jsrt::sync::lock(&self.shared.state);
let generation = self.shared.ensure_started(&mut state, true)?;
self.shared.wait_started(state, generation, None).map(drop)
}
pub fn execute(
&self,
code: &str,
signal: Option<&Signal>,
handler: Option<Arc<dyn HostHandler>>,
) -> Result<ExecutionResult, KernelError> {
self.execute_cell(code, signal, handler).map(|reply| reply.result)
}
pub fn execute_cell(
&self,
code: &str,
signal: Option<&Signal>,
handler: Option<Arc<dyn HostHandler>>,
) -> Result<KernelReply, KernelError> {
let frame = object([("request", Value::from("execute")), ("code", Value::from(code))]);
self.shared.enqueue(frame, signal, handler)
}
pub fn reset_scratch(&self) -> Result<ExecutionResult, KernelError> {
self.shared
.enqueue(object([("request", Value::from("reset_scratch"))]), None, None)
.map(|reply| reply.result)
}
pub fn snapshot(&self, path: Option<&Path>) -> Result<KernelReply, KernelError> {
let path = self.snapshot_target(path)?;
self.shared.enqueue(
object([("request", Value::from("snapshot")), ("path", Value::from(path))]),
None,
None,
)
}
pub fn restore(&self, path: Option<&Path>) -> Result<KernelReply, KernelError> {
let path = self.snapshot_target(path)?;
self.shared.enqueue(
object([("request", Value::from("restore")), ("path", Value::from(path))]),
None,
None,
)
}
fn snapshot_target(&self, path: Option<&Path>) -> Result<String, KernelError> {
path.or(self.shared.options.snapshot_path.as_deref())
.map(|path| path.to_string_lossy().into_owned())
.ok_or_else(|| KernelError::new("RLM snapshot path is not configured"))
}
pub fn last_recovery(&self) -> Option<ContinuityReport> {
jsrt::sync::lock(&self.shared.state).continuity.last_recovery.clone()
}
pub fn checkpoint(&self) -> Option<SnapshotReport> {
jsrt::sync::lock(&self.shared.state).continuity.checkpoint.clone()
}
pub fn shutdown(&self) {
let exit = {
let mut state = jsrt::sync::lock(&self.shared.state);
state.closed = true;
let error = KernelError::new(SHUT_DOWN);
let exit = match state
.generation
.as_ref()
.map(|generation| (generation.id, generation.exit.clone()))
{
Some((id, exit)) => {
self.shared.fail(&mut state, id, error);
Some(exit)
}
None => {
state.reject_ops(&error);
None
}
};
self.shared.changed.notify_all();
exit
};
if let Some(exit) = exit {
exit.wait_reaped(SHUTDOWN_TIMEOUT);
}
}
}
impl Drop for Kernel {
fn drop(&mut self) {
self.shutdown();
}
}
impl Shared {
fn with_state(&self, run: impl FnOnce(&mut State)) {
let mut state = jsrt::sync::lock(&self.state);
run(&mut state);
drop(state);
self.changed.notify_all();
}
fn enqueue(
self: &Arc<Self>,
frame: Map<String, Value>,
signal: Option<&Signal>,
handler: Option<Arc<dyn HostHandler>>,
) -> Result<KernelReply, KernelError> {
if jsrt::sync::lock(&self.state).closed {
return Err(KernelError::new(SHUT_DOWN));
}
if let Some(reason) = signal.and_then(Signal::reason) {
return Err(KernelError::new(reason));
}
let mut probe = frame.clone();
probe.insert("id".into(), Value::from(PLACEHOLDER_ID));
encode_frame(&probe)?;
let op = {
let mut state = jsrt::sync::lock(&self.state);
if state.closed {
return Err(KernelError::new(SHUT_DOWN));
}
state.next_op += 1;
let op = state.next_op;
state.ops.insert(
op,
Op {
running: false,
outcome: None,
cell: None,
handler,
recovery: None,
},
);
state.queue.push_back(op);
op
};
let kind = frame
.get("request")
.and_then(Value::as_str)
.unwrap_or_default()
.to_string();
let link = signal.map(|signal| {
let shared: Weak<Self> = Arc::downgrade(self);
signal.link(move |reason| {
if let Some(shared) = shared.upgrade() {
shared.abort_op(op, reason);
}
})
});
let mut result = self.run(op, frame);
drop(link);
let mut state = jsrt::sync::lock(&self.state);
let recovery = state.ops.remove(&op).and_then(|entry| entry.recovery);
if state.queue.front() == Some(&op) {
state.queue.pop_front();
}
match &mut result {
Ok(reply) => {
if let Some(report) = recovery {
reply.result.stderr = format!("{}{}", report.render(), reply.result.stderr);
reply.recovery = Some(report);
}
if kind == "execute" || kind == "reset_scratch" {
self.cell_completed(&mut state);
}
}
Err(_) => {
if state.continuity.notice.is_none() {
state.continuity.notice = recovery;
}
}
}
drop(state);
self.changed.notify_all();
result
}
fn cell_completed(self: &Arc<Self>, state: &mut State) {
let Some(policy) = self.options.continuity else {
return;
};
let continuity = &mut state.continuity;
continuity.cells += 1;
continuity.dirty = true;
if continuity.snapshot_queued || state.closed {
return;
}
let due = continuity
.last_checkpoint
.map(|last| last + policy.min_interval)
.filter(|due| *due > Instant::now());
match due {
None => self.queue_checkpoint(state, true),
Some(due) if !continuity.idle_flush_scheduled => {
continuity.idle_flush_scheduled = true;
let shared = Arc::downgrade(self);
std::thread::spawn(move || {
std::thread::sleep(due.saturating_duration_since(Instant::now()));
let Some(shared) = shared.upgrade() else {
return;
};
let mut state = jsrt::sync::lock(&shared.state);
state.continuity.idle_flush_scheduled = false;
if state.continuity.dirty && !state.continuity.snapshot_queued && !state.closed {
shared.queue_checkpoint(&mut state, false);
}
});
}
Some(_) => {}
}
}
fn queue_checkpoint(self: &Arc<Self>, state: &mut State, front: bool) {
let Some(path) = self.options.snapshot_path.as_ref() else {
return;
};
state.next_op += 1;
let op = state.next_op;
state.ops.insert(
op,
Op {
running: false,
outcome: None,
cell: None,
handler: None,
recovery: None,
},
);
if front {
state.queue.push_front(op);
} else {
state.queue.push_back(op);
}
state.continuity.snapshot_queued = true;
let frame = object([
("request", Value::from("snapshot")),
("path", Value::from(path.to_string_lossy().into_owned())),
]);
let shared = self.clone();
std::thread::spawn(move || {
let _ = shared.run(op, frame);
let mut state = jsrt::sync::lock(&shared.state);
state.ops.remove(&op);
if state.queue.front() == Some(&op) {
state.queue.pop_front();
}
state.continuity.snapshot_queued = false;
drop(state);
shared.changed.notify_all();
});
}
fn abort_op(&self, op: u64, reason: String) {
self.with_state(|state| {
let error = KernelError::new(reason);
let Some(entry) = state.ops.get_mut(&op) else {
return;
};
if entry.outcome.is_some() {
return;
}
if !entry.running {
entry.outcome = Some(Err(error));
state.queue.retain(|queued| *queued != op);
return;
}
if let Some(cell) = &entry.cell {
cell.abort(error.message());
}
if self.options.continuity.is_none() {
state.no_auto_restore = true;
}
match state.generation.as_ref().map(|generation| generation.id) {
Some(generation) => self.fail(state, generation, error),
None => entry_reject(state, op, error),
}
});
}
fn take_outcome(state: &mut State, op: u64) -> Option<Result<KernelReply, KernelError>> {
state.ops.get_mut(&op).and_then(|entry| entry.outcome.take())
}
fn run(self: &Arc<Self>, op: u64, frame: Map<String, Value>) -> Result<KernelReply, KernelError> {
let mut state = jsrt::sync::lock(&self.state);
loop {
if let Some(outcome) = Self::take_outcome(&mut state, op) {
return outcome;
}
if state.queue.front() == Some(&op) {
break;
}
state = self.changed.wait(state).unwrap_or_else(PoisonError::into_inner);
}
if let Some(entry) = state.ops.get_mut(&op) {
entry.running = true;
}
let kind = frame
.get("request")
.and_then(Value::as_str)
.unwrap_or_default()
.to_string();
let generation = self.ensure_started(&mut state, kind != "restore")?;
let state = self.wait_started(state, generation, Some(op))?;
drop(state);
let path = frame.get("path").and_then(Value::as_str).unwrap_or_default();
match kind.as_str() {
"snapshot" => self.signed_snapshot(op, generation, Path::new(path)),
"restore" => self.verified_restore(op, generation, Path::new(path)),
_ => self.request(op, generation, frame),
}
}
fn request(
self: &Arc<Self>,
op: u64,
generation: u64,
mut frame: Map<String, Value>,
) -> Result<KernelReply, KernelError> {
let mut state = jsrt::sync::lock(&self.state);
let execute = frame.get("request") == Some(&Value::from("execute"));
let request_id = random_uuid()?;
frame.insert("id".into(), Value::from(request_id.clone()));
let line = encode_frame(&frame)?;
let handler = state.ops.get(&op).and_then(|entry| entry.handler.clone());
let controller = Signal::new();
let Some(current) = state.current(generation) else {
return Err(Self::take_outcome(&mut state, op)
.and_then(Result::err)
.unwrap_or_else(|| KernelError::new(NOT_RUNNING)));
};
let state_ok = KernelReply::from_result(ExecutionResult {
status: Status::Ok,
stdout: String::new(),
stderr: String::new(),
result: None,
error: None,
});
current.pending.insert(
request_id.clone(),
Waiter {
op,
state: state_ok,
controller: controller.clone(),
handler,
},
);
if execute {
if current.cpu_pid.is_none() {
current.cpu_pid = sandboxed::python_pid(current.pid, &self.python(), &self.options.runtime_path);
}
current.cell_cpu_start = current.cpu_pid.and_then(process::process_cpu_seconds);
current.active_cell = Some(request_id.clone());
take_raw_output(current, &request_id);
let notice = state.continuity.notice.take();
if let Some(entry) = state.ops.get_mut(&op) {
entry.cell = Some(controller);
entry.recovery = notice;
}
}
if let Err(error) = self.send(&mut state, generation, line) {
if let Some(current) = state.current(generation) {
current.pending.remove(&request_id);
if current.active_cell.as_deref() == Some(request_id.as_str()) {
current.active_cell = None;
}
}
entry_reject(&mut state, op, error.clone());
self.fail(&mut state, generation, error);
}
let mut cpu_checked = Instant::now();
loop {
if let Some(outcome) = Self::take_outcome(&mut state, op) {
return outcome;
}
let now = Instant::now();
let mut timeout = WATCHDOG_INTERVAL;
if let Some(current) = state.current(generation) {
if !current.initialized {
let deadline = current.startup_deadline;
if deadline <= now {
let error = self.startup_timeout(&state);
self.fail(&mut state, generation, error);
continue;
}
timeout = timeout.min(deadline - now);
}
let Some(current) = state.current(generation) else {
continue;
};
if let Some(deadline) = current.output.as_ref().and_then(|output| output.deadline) {
if deadline <= now {
finish_cell(&mut state, generation);
continue;
}
timeout = timeout.min(deadline - now);
}
if now.duration_since(cpu_checked) >= WATCHDOG_INTERVAL {
cpu_checked = now;
if self.cpu_exceeded(current) {
let error = KernelError::new(cpu_limit_message(&self.limits));
self.fail(&mut state, generation, error);
continue;
}
}
}
state = self
.changed
.wait_timeout(state, timeout)
.unwrap_or_else(PoisonError::into_inner)
.0;
}
}
fn cpu_exceeded(&self, generation: &Generation) -> bool {
let limit = self.limits.max_cpu_seconds as f64;
if limit <= 0.0 || generation.active_cell.is_none() {
return false;
}
let Some(start) = generation.cell_cpu_start else {
return false;
};
let grace = (limit * 0.1).clamp(2.0, 10.0);
generation
.cpu_pid
.and_then(process::process_cpu_seconds)
.is_some_and(|used| used - start > limit + grace)
}
fn tree_memory_exceeded(&self, generation: &mut Generation) -> bool {
match generation.tree_backend {
TreeMemoryBackend::Off => false,
TreeMemoryBackend::Watchdog => {
watchdog_tree_bytes(generation.pid, &generation.known_descendants) > self.tree_memory_mb * 1024 * 1024
}
TreeMemoryBackend::Cgroup => {
if generation.cgroup_dir.is_none() {
let dir = tree_memory::process_cgroup_dir(generation.pid)
.filter(|dir| tree_memory::is_foreign_scope(dir));
let Some(dir) = dir.filter(|_| generation.ready_received) else {
return false;
};
generation.oom_kills_at_start = tree_memory::cgroup_oom_kills(&dir).unwrap_or(0);
generation.cgroup_dir = Some(dir);
}
generation
.cgroup_dir
.as_deref()
.and_then(tree_memory::cgroup_oom_kills)
.is_some_and(|kills| kills > generation.oom_kills_at_start)
}
}
}
fn start_tree_memory_watchdog(
self: &Arc<Self>,
id: u64,
pid: i32,
backend: TreeMemoryBackend,
known: Arc<Mutex<HashMap<i32, u64>>>,
exit: Arc<Exit>,
) {
let shared = Arc::downgrade(self);
std::thread::spawn(move || {
let mut delay = Duration::from_millis(500);
loop {
if exit.wait_reaped(delay) {
return;
}
let Some(shared) = shared.upgrade() else {
return;
};
let cap_bytes = shared.tree_memory_mb * 1024 * 1024;
let bytes = (backend == TreeMemoryBackend::Watchdog).then(|| watchdog_tree_bytes(pid, &known));
let mut state = jsrt::sync::lock(&shared.state);
let Some(current) = state.current(id) else {
return;
};
let exceeded = match bytes {
Some(bytes) => {
delay = Duration::from_millis(tree_memory::watchdog_poll_ms(bytes, cap_bytes));
bytes > cap_bytes
}
None => shared.tree_memory_exceeded(current),
};
if exceeded {
let message = tree_memory::tree_memory_limit_message(shared.tree_memory_mb, backend);
shared.fail(&mut state, id, KernelError::new(message));
drop(state);
shared.changed.notify_all();
return;
}
}
});
}
fn wait_started<'a>(
self: &'a Arc<Self>,
mut state: MutexGuard<'a, State>,
generation: u64,
op: Option<u64>,
) -> Result<MutexGuard<'a, State>, KernelError> {
loop {
if let Some(op) = op
&& let Some(outcome) = Self::take_outcome(&mut state, op)
{
return Err(outcome.err().unwrap_or_else(|| KernelError::new(NOT_RUNNING)));
}
let Some(current) = state.current(generation) else {
return Err(match &state.failure {
Some((failed, error)) if *failed == generation => error.clone(),
_ => KernelError::new(NOT_RUNNING),
});
};
if current.initialized {
return Ok(state);
}
if current.ready_received && !current.completing {
current.completing = true;
let restore_path = current.restore_path.clone();
drop(state);
self.complete_startup(generation, restore_path);
state = jsrt::sync::lock(&self.state);
continue;
}
let deadline = current.startup_deadline;
let now = Instant::now();
if now >= deadline {
let error = self.startup_timeout(&state);
self.fail(&mut state, generation, error);
continue;
}
state = self
.changed
.wait_timeout(state, deadline - now)
.unwrap_or_else(PoisonError::into_inner)
.0;
}
}
fn startup_timeout(&self, state: &State) -> KernelError {
KernelError::new(format!(
"RLM kernel startup timed out after {}ms{}",
self.options.startup_timeout.as_millis(),
state.diagnostics()
))
}
fn ensure_started(self: &Arc<Self>, state: &mut State, restore: bool) -> Result<u64, KernelError> {
if state.closed {
return Err(KernelError::new(SHUT_DOWN));
}
if let Some(generation) = &state.generation {
return Ok(generation.id);
}
self.spawn(state, restore)
}
fn complete_startup(self: &Arc<Self>, generation: u64, restore_path: Option<PathBuf>) {
let restored = restore_path.as_ref().map(|path| {
let op = {
let mut state = jsrt::sync::lock(&self.state);
state.next_op += 1;
let op = state.next_op;
state.ops.insert(
op,
Op {
running: true,
outcome: None,
cell: None,
handler: None,
recovery: None,
},
);
op
};
let outcome = self.verified_restore(op, generation, path);
jsrt::sync::lock(&self.state).ops.remove(&op);
outcome
});
let mut state = jsrt::sync::lock(&self.state);
if state.current(generation).is_none() {
return;
}
let continuity = self.options.continuity.is_some();
let skipped_restore = state
.continuity
.crash
.as_ref()
.and_then(|crash| crash.restore_error.clone());
let restore: Result<Option<RestoreReport>, String> = match restored {
None => skipped_restore.map_or(Ok(None), Err),
Some(Ok(reply)) if reply.result.status == Status::Error => Err(reply
.result
.error
.map_or_else(|| "unknown restore error".to_string(), |error| error.evalue)),
Some(Ok(reply)) => Ok(reply.restore),
Some(Err(error)) => {
self.fail(&mut state, generation, error);
drop(state);
self.changed.notify_all();
return;
}
};
if let Err(message) = &restore
&& !continuity
{
let error = KernelError::new(format!("RLM kernel startup restore failed: {message}"));
self.fail(&mut state, generation, error);
drop(state);
self.changed.notify_all();
return;
}
if continuity {
let crash = state.continuity.crash.take();
let restored_something = matches!(&restore, Ok(Some(report)) if !report.missing);
if crash.is_some() || restored_something || restore.is_err() {
let crash = crash.unwrap_or_else(|| Crash {
reason: "kernel started from its checkpoint".into(),
interrupted_cell: false,
unsaved_cells: 0,
restore_error: None,
});
let path = self
.options
.snapshot_path
.as_ref()
.map(|path| path.display().to_string())
.unwrap_or_default();
let report = recovery_report(crash, path, restore, state.continuity.checkpoint.as_ref());
state.continuity.notice = Some(report.clone());
state.continuity.last_recovery = Some(report);
}
}
if let Some(current) = state.current(generation) {
current.initialized = true;
}
drop(state);
self.changed.notify_all();
}
fn signed_snapshot(self: &Arc<Self>, op: u64, generation: u64, path: &Path) -> Result<KernelReply, KernelError> {
let directory = path.parent().unwrap_or(Path::new(""));
let name = path
.file_name()
.map(|name| name.to_string_lossy().into_owned())
.unwrap_or_default();
let unsigned = directory.join(format!(".{name}.{}.unsigned", random_uuid()?));
let frame = object([
("request", Value::from("snapshot")),
("path", Value::from(unsigned.to_string_lossy().into_owned())),
]);
let outcome = self.request(op, generation, frame).and_then(|reply| {
if reply.result.status != Status::Ok {
return Ok(reply);
}
let failed = |error: String| KernelReply::from_result(snapshot::snapshot_error(error));
let content = match std::fs::read(&unsigned) {
Ok(content) => content,
Err(error) => return Ok(failed(format!("snapshot signing failed: {error}"))),
};
let reported = reply
.snapshot
.as_ref()
.and_then(|report| report.content_sha256.as_deref());
if reported != Some(jsrt::hash::sha256_hex(&content).as_str()) {
return Ok(failed(
"snapshot changed after the kernel wrote it; refusing to sign it".into(),
));
}
let temporary = directory.join(format!(".{name}.{}.tmp", random_uuid()?));
let written = std::fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.mode(0o600)
.open(&temporary)
.and_then(|mut file| file.write_all(&sign_snapshot(&self.snapshot_key, &content)))
.and_then(|()| std::fs::rename(&temporary, path));
let _ = std::fs::remove_file(&temporary);
if let Err(error) = written {
return Ok(failed(format!("snapshot signing failed: {error}")));
}
if self.options.snapshot_path.as_deref() == Some(path) {
let mut state = jsrt::sync::lock(&self.state);
let continuity = &mut state.continuity;
continuity.checkpoint = reply.snapshot.clone();
continuity.checkpoint_cells = continuity.cells;
continuity.last_checkpoint = Some(Instant::now());
continuity.dirty = false;
if self.options.continuity.is_some() {
state.no_auto_restore = false;
}
}
Ok(reply)
});
let _ = std::fs::remove_file(&unsigned);
outcome
}
fn verified_restore(self: &Arc<Self>, op: u64, generation: u64, path: &Path) -> Result<KernelReply, KernelError> {
let shown = path.to_string_lossy().into_owned();
let raw = match std::fs::read(path) {
Ok(raw) => raw,
Err(error)
if matches!(
error.kind(),
std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory
) =>
{
let frame = object([("request", Value::from("restore")), ("path", Value::from(shown))]);
return self.request(op, generation, frame);
}
Err(error) => {
return Ok(KernelReply::from_result(snapshot::snapshot_error(format!(
"snapshot cannot be read: {error}"
))));
}
};
match verify_snapshot(&self.snapshot_key, &raw) {
Err(error) => Ok(KernelReply::from_result(snapshot::snapshot_error(error.0))),
Ok(content_sha256) => {
let frame = object([
("request", Value::from("restore")),
("path", Value::from(shown)),
("content_sha256", Value::from(content_sha256)),
]);
self.request(op, generation, frame)
}
}
}
fn python(&self) -> PathBuf {
self.options
.python
.clone()
.unwrap_or_else(|| crate::env::python(|name| self.options.var(name)).into())
}
fn spawn(self: &Arc<Self>, state: &mut State, restore: bool) -> Result<u64, KernelError> {
let options = &self.options;
let python = self.python();
let spawn_error = |error: std::io::Error| KernelError::new(format!("RLM kernel spawn failed: {error}"));
let mechanism = match &options.sandbox {
Some(launcher) => sandboxed::mechanism(launcher)?,
None => None,
};
let tree_backend = match &options.tree_memory {
None => TreeMemoryBackend::Off,
Some(tree) => {
let requested = tree
.backend
.clone()
.or_else(|| options.var(crate::env::RLM_TREE_MEMORY_BACKEND_ENV));
tree_memory::tree_memory_backend(self.tree_memory_mb, requested.as_deref(), || {
mechanism.is_none() && tree_memory::cgroup_scope_available(tree.launcher.as_os_str())
})
}
};
let argv: Vec<std::ffi::OsString> = vec![
python.clone().into(),
options.runtime_path.clone().into(),
"--protocol-fds".into(),
];
let budget = rlm_output_budget(options.var(crate::env::RLM_OUTPUT_BYTES_ENV).as_deref());
let fixed_env: Vec<(String, String)> = vec![
("NO_COLOR".into(), "1".into()),
("PYTHONDONTWRITEBYTECODE".into(), "1".into()),
(crate::env::IN_KERNEL_ENV.into(), "1".into()),
(
crate::env::RLM_MAX_MEMORY_MB_ENV.into(),
self.limits.max_memory_mb.to_string(),
),
(
crate::env::RLM_MAX_CPU_SECONDS_ENV.into(),
self.limits.max_cpu_seconds.to_string(),
),
(crate::env::RLM_OUTPUT_BYTES_ENV.into(), budget.to_string()),
];
let mut command = match (mechanism, &options.sandbox) {
(Some(mechanism), Some(launcher)) => sandboxed::command(sandboxed::Launch {
launcher,
mechanism,
python: &python,
argv,
cwd: &options.cwd,
runtime: &options.runtime_path,
snapshot_path: options.snapshot_path.as_deref(),
fixed_env,
extra_env: &options.env,
base_env: options.base_environment(),
}),
_ => {
let mut command = match (&options.tree_memory, tree_backend) {
(Some(tree), TreeMemoryBackend::Cgroup) => {
let mut command = Command::new(&tree.launcher);
command
.args(&tree_memory::cgroup_scope_command(self.tree_memory_mb)[1..])
.args(&argv);
command
}
_ => {
let mut command = Command::new(&argv[0]);
command.args(&argv[1..]);
command
}
};
command
.current_dir(&options.cwd)
.env_clear()
.envs(kernel_environment(options.base_environment()))
.envs(fixed_env)
.envs(options.env.iter().map(|(name, value)| (name, value)));
command
}
};
command.process_group(0).stdout(Stdio::piped()).stderr(Stdio::piped());
let (protocol_out, kernel_out) = std::io::pipe().map_err(spawn_error)?;
let (kernel_in, protocol_in) = std::io::pipe().map_err(spawn_error)?;
command.stdin(Stdio::null());
command
.fd_mappings(vec![
FdMapping {
parent_fd: kernel_out.into(),
child_fd: 3,
},
FdMapping {
parent_fd: kernel_in.into(),
child_fd: 4,
},
])
.map_err(|error| KernelError::new(error.to_string()))?;
let mut child = if mechanism.is_some() {
sandboxed::spawn(command).map_err(spawn_error)?
} else {
let child = command.spawn().map_err(spawn_error)?;
drop(command);
child
};
let pid = i32::try_from(child.id()).map_err(|error| KernelError::new(error.to_string()))?;
let stderr = child.stderr.take();
let raw_stdout = child.stdout.take();
state.next_generation += 1;
let id = state.next_generation;
let restore_path = if restore && !state.no_auto_restore {
options.snapshot_path.clone()
} else {
None
};
let host_completes_startup =
restore_path.is_some() || (options.continuity.is_some() && state.continuity.crash.is_some());
let (writer, lines) = mpsc::channel::<String>();
let exit = Arc::new(Exit::default());
let known_descendants: Arc<Mutex<HashMap<i32, u64>>> = Arc::default();
state.generation = Some(Generation {
id,
pid,
tree_backend,
cgroup_dir: None,
oom_kills_at_start: 0,
known_descendants: known_descendants.clone(),
writer,
exit: exit.clone(),
raw: Default::default(),
output: None,
pending: HashMap::new(),
active_cell: None,
host_requests: HashMap::new(),
ready_received: false,
initialized: false,
startup_deadline: Instant::now() + options.startup_timeout,
cell_cpu_start: None,
cpu_pid: mechanism.is_none().then_some(pid),
sandbox: mechanism,
buffer: Vec::new(),
restore_path,
host_completes_startup,
completing: false,
});
state.stderr.clear();
let shared = self.clone();
let mut protocol_in = protocol_in;
std::thread::spawn(move || {
for line in lines {
if let Err(error) = protocol_in
.write_all(line.as_bytes())
.and_then(|()| protocol_in.flush())
{
shared.with_state(|state| shared.fail(state, id, KernelError::new(error.to_string())));
return;
}
}
});
let shared = self.clone();
let channel_exit = exit.clone();
std::thread::spawn(move || {
if let Err(error) = read_chunks(protocol_out, |chunk| {
shared.with_state(|state| shared.handle_data(state, id, chunk));
}) {
shared.with_state(|state| shared.fail(state, id, KernelError::new(error.to_string())));
return;
}
if !channel_exit.wait_reaped(CLOSED_CHANNEL_GRACE) {
shared.with_state(|state| {
let error = KernelError::new(format!(
"RLM kernel protocol channel (fd 3) closed{}",
state.diagnostics()
));
shared.fail(state, id, error);
});
}
});
if let Some(stdout) = raw_stdout {
let shared = self.clone();
std::thread::spawn(move || {
let _ = read_chunks(stdout, |chunk| {
shared.with_state(|state| handle_raw(state, id, 0, chunk));
});
});
}
if let Some(stderr) = stderr {
let shared = self.clone();
std::thread::spawn(move || {
let _ = read_chunks(stderr, |chunk| {
shared.with_state(|state| {
if state.current(id).is_none() {
return;
}
handle_raw(state, id, 1, chunk);
});
});
});
}
if tree_backend != TreeMemoryBackend::Off {
self.start_tree_memory_watchdog(id, pid, tree_backend, known_descendants, exit.clone());
}
let shared = self.clone();
std::thread::spawn(move || {
if let Some(process) = rustix::process::Pid::from_raw(pid) {
while let Err(rustix::io::Errno::INTR) =
rustix::process::waitid(WaitId::Pid(process), WaitIdOptions::EXITED | WaitIdOptions::NOWAIT)
{
}
}
shared.with_state(|_| jsrt::sync::lock(&exit.state).0 = true);
let status = child.wait();
jsrt::sync::lock(&exit.state).1 =
Some(status.as_ref().copied().unwrap_or_else(|_| ExitStatus::from_raw(0)));
exit.changed.notify_all();
shared.with_state(|state| {
if state.current(id).is_some() {
let killed = status
.as_ref()
.ok()
.and_then(|status| status.signal())
.is_some_and(|signal| signal == SIGKILL || signal == SIGTERM);
let tree_kill = match state.current(id) {
Some(current) if killed => shared.tree_memory_exceeded(current).then_some(current.tree_backend),
_ => None,
};
let error = match tree_kill {
Some(backend) => {
KernelError::new(tree_memory::tree_memory_limit_message(shared.tree_memory_mb, backend))
}
None => shared.exit_error(state, status.ok()),
};
shared.fail(state, id, error);
}
});
let mut state = jsrt::sync::lock(&shared.state);
if std::mem::take(&mut state.continuity.restart)
&& !state.closed
&& state.generation.is_none()
&& let Ok(generation) = shared.ensure_started(&mut state, true)
{
drop(shared.wait_started(state, generation, None));
}
});
Ok(id)
}
fn exit_error(&self, state: &State, status: Option<ExitStatus>) -> KernelError {
let mut code = status.and_then(|status| status.code());
let mut signal = status.and_then(|status| status.signal());
if state.generation.as_ref().and_then(|generation| generation.sandbox) == Some(sandbox::Mechanism::Bwrap)
&& let Some(number) = code.filter(|code| (129..=192).contains(code))
{
(code, signal) = (None, Some(number - 128));
}
if code == Some(EXIT_MEMORY) {
return KernelError::new(memory_limit_message(&self.limits));
}
if code == Some(EXIT_CPU) || signal == Some(SIGXCPU) {
return KernelError::new(cpu_limit_message(&self.limits));
}
let hint = if signal == Some(SIGKILL) {
"; the system out-of-memory killer may have stopped it"
} else {
""
};
let how = match (signal, code) {
(Some(signal), _) => jsrt::process::signal_name(signal),
(None, Some(code)) => code.to_string(),
(None, None) => "unknown".into(),
};
KernelError::new(format!("RLM kernel exited ({how}){hint}{}", state.diagnostics()))
}
fn fail(&self, state: &mut State, generation: u64, error: KernelError) {
if state.current(generation).is_none() {
return;
}
let Some(failed) = state.generation.take() else {
return;
};
state.failure = Some((generation, error.clone()));
if self.options.continuity.is_some() {
if !state.closed {
let continuity = &mut state.continuity;
let unsaved_cells = continuity.cells - continuity.checkpoint_cells;
let interrupted_cell = failed.active_cell.is_some();
let restoring = failed.completing && !failed.initialized;
match continuity.crash.as_mut() {
Some(crash) => {
crash.interrupted_cell |= interrupted_cell;
crash.unsaved_cells = unsaved_cells;
}
None => {
continuity.crash = Some(Crash {
reason: error.message().to_string(),
interrupted_cell,
unsaved_cells,
restore_error: None,
});
}
}
if restoring && failed.restore_path.is_some() {
state.no_auto_restore = true;
if let Some(crash) = state.continuity.crash.as_mut() {
crash.restore_error = Some(format!("the kernel failed while restoring: {}", error.message()));
}
}
state.continuity.restart = failed.initialized;
}
} else if failed.initialized {
state.no_auto_restore = true;
}
for waiter in failed.pending.values() {
waiter.controller.abort(error.message());
}
if let Some(waiter) = failed.output.as_ref().and_then(|output| output.finished.as_ref()) {
waiter.controller.abort(error.message());
}
state.reject_ops(&error);
let known = jsrt::sync::lock(&failed.known_descendants).clone();
process::kill_process_tree(
failed.pid,
true,
!failed.exit.exited(),
process::KillExtras {
known: Some(&known),
cgroup_dir: failed.cgroup_dir.as_deref(),
},
);
}
fn send(&self, state: &mut State, generation: u64, line: String) -> Result<(), KernelError> {
let closed = state.closed;
let failure = state.failure.clone();
match state.current(generation) {
Some(current) if !closed => current.writer.send(line).map_err(|_| KernelError::new(NOT_RUNNING)),
_ => Err(match failure {
Some((failed, error)) if failed == generation => error,
_ => KernelError::new(NOT_RUNNING),
}),
}
}
fn handle_data(self: &Arc<Self>, state: &mut State, generation: u64, chunk: &[u8]) {
let mut offset = 0;
while offset < chunk.len() {
let Some(current) = state.current(generation) else {
return;
};
let newline = chunk[offset..]
.iter()
.position(|&byte| byte == b'\n')
.map(|index| offset + index);
let end = newline.unwrap_or(chunk.len());
if current.buffer.len() + (end - offset) > MAX_FRAME_BYTES {
let error = format!("RLM kernel protocol frame exceeds 1 MiB ({MAX_FRAME_BYTES} bytes)");
self.fail(state, generation, KernelError::new(error));
return;
}
current.buffer.extend_from_slice(&chunk[offset..end]);
if newline.is_none() {
return;
}
let line = std::mem::take(&mut current.buffer);
let handled = match std::str::from_utf8(&line) {
Ok(text) => self.handle_line(state, generation, text),
Err(_) => Err("The encoded data was not valid for encoding utf-8".to_string()),
};
if let Err(message) = handled {
let error = KernelError::new(format!("RLM kernel invalid protocol frame: {message}"));
self.fail(state, generation, error);
}
offset = end + 1;
}
}
fn handle_line(self: &Arc<Self>, state: &mut State, generation: u64, line: &str) -> Result<(), String> {
let parsed: Value =
serde_json::from_str(&replace_lone_surrogates(line)).map_err(|error| format!("invalid JSON: {error}"))?;
let Value::Object(frame) = parsed else {
return Err("protocol frame must be a JSON object".into());
};
let Some(Value::String(event)) = frame.get("event") else {
return Err("protocol frame is missing event".into());
};
let Some(current) = state.current(generation) else {
return Ok(());
};
match event.as_str() {
"ready" => {
if current.ready_received {
return Err("duplicate ready event".into());
}
if frame.contains_key("id") {
return Err("ready event cannot have an id".into());
}
current.ready_received = true;
current.initialized = !current.host_completes_startup;
if current.tree_backend == TreeMemoryBackend::Cgroup {
self.tree_memory_exceeded(current);
}
Ok(())
}
"host_request" => {
let id = match frame.get("id") {
Some(Value::String(id)) if !id.is_empty() => id.clone(),
_ => return Err("host_request requires an id".into()),
};
let kind = match frame.get("type") {
Some(Value::String(kind)) if !kind.is_empty() => kind.clone(),
_ => return Err("host_request requires a type".into()),
};
let payload = match frame.get("payload") {
None => Map::new(),
Some(Value::Object(payload)) => payload.clone(),
Some(_) => return Err("host_request payload must be an object".into()),
};
let cell = match frame.get("cell") {
None => None,
Some(Value::String(cell)) if !cell.is_empty() => Some(cell.clone()),
Some(_) => return Err("host_request cell must be a request id".into()),
};
self.handle_host_request(
state,
generation,
HostRequest {
id,
kind,
payload,
cell,
},
);
Ok(())
}
"host_cancel" => {
let Some(Value::String(id)) = frame.get("id") else {
return Err("host_cancel requires an id".into());
};
if let Some(request) = current.host_requests.get(id) {
request.abort("RLM host request cancelled");
}
Ok(())
}
"stdout" | "stderr" | "result" | "error" | "done" => {
let id = match frame.get("id") {
Some(Value::String(id)) if !id.is_empty() => id.clone(),
_ => return Err(format!("{event} event requires an id")),
};
let status = frame.get("status").map(Value::as_str);
if event == "done" && status.is_some_and(|status| status != Some("ok") && status != Some("error")) {
return Err("done event has invalid status".into());
}
if event == "done"
&& current.tree_backend == TreeMemoryBackend::Cgroup
&& current.pending.contains_key(&id)
&& self.tree_memory_exceeded(current)
{
let message = tree_memory::tree_memory_limit_message(self.tree_memory_mb, current.tree_backend);
self.fail(state, generation, KernelError::new(message));
return Ok(());
}
route_execution_frame(state, generation, &id, event, &frame)
}
other => Err(format!("unknown protocol event: {other}")),
}
}
fn handle_host_request(self: &Arc<Self>, state: &mut State, generation: u64, request: HostRequest) {
let HostRequest {
id,
kind,
payload,
cell,
} = request;
let Some(current) = state.current(generation) else {
return;
};
if current.host_requests.contains_key(&id) {
let error = KernelError::new(format!("RLM kernel protocol duplicate host request id: {id}"));
self.fail(state, generation, error);
return;
}
let waiter = match &cell {
Some(cell) if current.active_cell.as_ref() != Some(cell) => {
self.send_host_error(state, generation, &id, "RLM host request from a cell that has finished");
return;
}
Some(cell) => current.pending.get(cell),
None => current.active_cell.as_ref().and_then(|cell| current.pending.get(cell)),
};
let Some(waiter) = waiter else {
self.send_host_error(state, generation, &id, "RLM host request has no active cell");
return;
};
if current.host_requests.len() >= MAX_HOST_REQUESTS {
let message = format!("RLM host request concurrency exceeds {MAX_HOST_REQUESTS}");
self.send_host_error(state, generation, &id, &message);
return;
}
let handler = waiter.handler.clone().unwrap_or_else(|| self.handler.clone());
let cell_signal = waiter.controller.clone();
let signal = Signal::new();
let request = signal.clone();
if cell_signal.on_abort(move |reason| request.abort(reason)).is_none() {
signal.abort(cell_signal.reason().unwrap_or_default());
}
current.host_requests.insert(id.clone(), signal.clone());
let shared = self.clone();
std::thread::spawn(move || {
let result =
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| handler.handle(&kind, payload, &signal)))
.unwrap_or_else(|_| Err(format!("RLM host handler for {kind} panicked")));
shared.with_state(|state| {
if state.current(generation).is_some() {
match result {
Ok(value) => {
let reply = object([
("request", Value::from("host_reply")),
("id", Value::from(id.clone())),
("payload", value),
]);
shared.send_host_reply(state, generation, &id, &reply);
}
Err(message) => shared.send_host_error(state, generation, &id, &message),
}
}
if let Some(current) = state.current(generation) {
current.host_requests.remove(&id);
}
});
});
}
fn send_host_reply(&self, state: &mut State, generation: u64, id: &str, reply: &Map<String, Value>) {
let sent = encode_frame(reply).and_then(|line| self.send(state, generation, line));
if let Err(error) = sent {
if state.current(generation).is_none() {
return;
}
self.send_host_error(state, generation, id, error.message());
}
}
fn send_host_error(&self, state: &mut State, generation: u64, id: &str, message: &str) {
if state.current(generation).is_none() {
return;
}
let reply = object([
("request", Value::from("host_reply")),
("id", Value::from(id)),
("error", Value::from(message)),
]);
if let Err(error) = encode_frame(&reply).and_then(|line| self.send(state, generation, line)) {
self.fail(state, generation, error);
}
}
}
fn watchdog_tree_bytes(root: i32, known: &Mutex<HashMap<i32, u64>>) -> u64 {
let (bytes, descendants) = tree_memory::tree_memory_usage(root);
let mut known = jsrt::sync::lock(known);
known.retain(|pid, start| process::process_stat(*pid).is_some_and(|stat| stat.start_time == *start));
for pid in descendants {
if let Some(stat) = process::process_stat(pid) {
known.insert(pid, stat.start_time);
}
}
bytes
}
fn entry_reject(state: &mut State, op: u64, error: KernelError) {
if let Some(entry) = state.ops.get_mut(&op) {
entry.outcome.get_or_insert(Err(error));
}
}
fn read_chunks(mut stream: impl Read, mut handle: impl FnMut(&[u8])) -> std::io::Result<()> {
let mut buffer = vec![0u8; 64 * 1024];
loop {
match stream.read(&mut buffer) {
Ok(0) => return Ok(()),
Ok(read) => handle(&buffer[..read]),
Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {}
Err(error) => return Err(error),
}
}
}
fn route_execution_frame(
state: &mut State,
generation: u64,
request_id: &str,
event: &str,
frame: &Map<String, Value>,
) -> Result<(), String> {
let Some(current) = state.current(generation) else {
return Ok(());
};
let Some(waiter) = current.pending.get_mut(request_id) else {
return Err(format!("protocol frame references unknown request id: {request_id}"));
};
let result = &mut waiter.state.result;
match event {
"stdout" => append_bounded(&mut result.stdout, &frame_string(frame.get("text"))),
"stderr" => append_bounded(&mut result.stderr, &frame_string(frame.get("text"))),
"result" => result.result = Some(frame_string(frame.get("result"))),
"error" => {
result.status = Status::Error;
let traceback = match frame.get("traceback") {
Some(Value::Array(lines)) => lines
.iter()
.filter_map(|line| line.as_str().map(String::from))
.collect(),
_ => Vec::new(),
};
result.error = Some(CellError {
ename: frame_string(frame.get("ename")),
evalue: frame_string(frame.get("evalue")),
traceback,
});
}
_ => {
if frame.get("status") == Some(&Value::from("error")) {
result.status = Status::Error;
}
if result.status == Status::Error
&& result.error.is_none()
&& let Some(error) = frame.get("error").filter(|error| !error.is_null())
{
result.error = Some(CellError {
ename: "RuntimeError".into(),
evalue: frame_string(Some(error)),
traceback: Vec::new(),
});
}
let Some(mut waiter) = current.pending.remove(request_id) else {
return Ok(());
};
if let Some(report) = snapshot::snapshot_report(frame.get("snapshot")) {
waiter.state.snapshot = Some(report);
}
if let Some(report) = snapshot::restore_report(frame.get("restore")) {
waiter.state.restore = Some(report);
}
if current.active_cell.as_deref() == Some(request_id) {
current.active_cell = None;
}
waiter.controller.abort("RLM cell completed");
let Some(output) = current.output.as_mut().filter(|output| output.request_id == request_id) else {
state.resolve(waiter);
return Ok(());
};
let flush = frame.get("flush").and_then(Value::as_object);
let flushed = |name: &str| flush.and_then(|flush| flush.get(name)) == Some(&Value::Bool(true));
output.awaiting = Some([flushed("stdout"), flushed("stderr")]);
output.deadline = Some(Instant::now() + RAW_FLUSH_TIMEOUT);
output.finished = Some(waiter);
finish_cell_if_flushed(state, generation);
}
}
Ok(())
}
fn take_raw_output(generation: &mut Generation, request_id: &str) {
let orphan = generation.raw.each_mut().map(|stream| {
let text = orphan_text(&stream.orphan, stream.orphan_dropped);
stream.orphan.clear();
stream.orphan_dropped = false;
text
});
generation.output = Some(CellOutput {
request_id: request_id.to_string(),
marker: flush_marker(request_id),
captures: Default::default(),
orphan,
awaiting: None,
deadline: None,
finished: None,
});
}
fn handle_raw(state: &mut State, generation: u64, index: usize, chunk: &[u8]) {
let State {
generation: current,
stderr,
..
} = state;
let Some(current) = current.as_mut().filter(|current| current.id == generation) else {
return;
};
let Generation { raw, output, .. } = current;
let stream = &mut raw[index];
let mut data = std::mem::take(&mut stream.held);
data.extend_from_slice(chunk);
let mut data = &data[..];
if !stream.started {
let startup = flush_marker("ready");
let found = find(data, &startup);
let end = found.unwrap_or_else(|| data.len() - marker_prefix_len(data, &startup));
if index == 1 {
keep_tail(stderr, &String::from_utf8_lossy(&data[..end]));
}
let Some(found) = found else {
stream.held = data[end..].to_vec();
return;
};
stream.started = true;
data = &data[found + startup.len()..];
}
let Some(cell) = output.as_mut().filter(|cell| !cell.captures[index].marker_seen) else {
route_raw(stream, stderr, index, data, None);
return;
};
if let Some(found) = find(data, &cell.marker) {
route_raw(stream, stderr, index, &data[..found], Some(&mut cell.captures[index]));
cell.captures[index].marker_seen = true;
route_raw(stream, stderr, index, &data[found + cell.marker.len()..], None);
finish_cell_if_flushed(state, generation);
return;
}
let held = marker_prefix_len(data, &cell.marker);
stream.held = data[data.len() - held..].to_vec();
route_raw(
stream,
stderr,
index,
&data[..data.len() - held],
Some(&mut cell.captures[index]),
);
}
fn route_raw(
stream: &mut RawStream,
stderr: &mut String,
index: usize,
bytes: &[u8],
capture: Option<&mut RawCapture>,
) {
if bytes.is_empty() {
return;
}
let text = stream.decoder.decode(bytes);
if index == 1 {
keep_tail(stderr, &text);
}
if let Some(capture) = capture {
capture.push(&text);
return;
}
let used = jsrt::utf16::len(&stream.orphan);
let room = MAX_ORPHAN_CHARS.saturating_sub(used);
if jsrt::utf16::len(&text) > room {
stream.orphan_dropped = true;
}
if room > 0 {
let kept = jsrt::utf16::head(&text, room);
stream.orphan.push_str(kept);
}
}
fn finish_cell_if_flushed(state: &mut State, generation: u64) {
let Some(output) = state.current(generation).and_then(|current| current.output.as_ref()) else {
return;
};
let (Some(_), Some(awaiting)) = (&output.finished, output.awaiting) else {
return;
};
if (0..2).any(|index| awaiting[index] && !output.captures[index].marker_seen) {
return;
}
finish_cell(state, generation);
}
fn finish_cell(state: &mut State, generation: u64) {
let State {
generation: current,
stderr,
..
} = state;
let Some(current) = current.as_mut().filter(|current| current.id == generation) else {
return;
};
let Some(output) = current.output.take() else {
return;
};
for (index, stream) in current.raw.iter_mut().enumerate() {
if stream.started {
let held = std::mem::take(&mut stream.held);
route_raw(stream, stderr, index, &held, None);
}
}
let Some(mut waiter) = output.finished else {
return;
};
let [orphan_out, orphan_err] = &output.orphan;
waiter.state.result.stdout = format!(
"{orphan_out}{}{}",
waiter.state.result.stdout,
output.captures[0].text(1)
);
waiter.state.result.stderr = format!(
"{orphan_err}{}{}",
waiter.state.result.stderr,
output.captures[1].text(2)
);
state.resolve(waiter);
}