pub mod bash_notes;
pub mod cell;
pub mod cell_writes;
pub mod file_hooks;
pub mod journal;
pub mod loki_guard;
pub mod sandboxed;
pub mod secrets;
pub mod shell;
pub mod text;
pub mod view_image;
use journal::{JobRecord, Status};
use jsrt::sync;
use kernel::store::DocumentStore;
use kernel::{HostHandler, Signal};
use serde_json::{Map, Value, json};
use std::fs::File;
use std::io::{Read, Seek, SeekFrom, Write};
use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
use std::os::unix::process::CommandExt;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Condvar, Mutex, PoisonError, mpsc};
use std::time::{Duration, Instant, UNIX_EPOCH};
use text::{DEFAULT_MAX_BYTES, Decoder, clean, tail_bytes, truncate_tail};
pub const JOB_TEXT_BYTES: usize = 16 * 1024;
pub const JOB_RETAINED_BYTES: usize = 4 * 1024 * 1024;
pub const JOB_MAX_ACTIVE: usize = 32;
pub const JOB_MAX_RETAINED: usize = 64;
const JOB_MAX_WAIT_SECONDS: f64 = 3600.0;
const MAX_TIMEOUT_SECONDS: f64 = 86_400.0;
const POLL: Duration = Duration::from_millis(10);
pub trait Guard: Send + Sync {
fn before_shell(&self, _lane: &str, _command: &str) -> Option<String> {
None
}
fn before_write(&self, _lane: &str, _path: &str, _content: &str) -> Value {
json!({ "blocked": false })
}
fn guard_stats(&self) -> Vec<Value> {
Vec::new()
}
fn begin_turn(&self) {}
fn cell_started(&self, _lane: &str) {}
fn cell_ended(&self, _lane: &str) {}
fn take_pending(&self, _lane: &str) -> Option<String> {
None
}
}
pub struct AllowAll;
impl Guard for AllowAll {}
pub fn not_available(kind: &str) -> String {
format!("Ultron RLM host request is not available in this host yet: {kind}")
}
pub struct Host {
cwd: PathBuf,
jobs_dir: PathBuf,
spill_dir: PathBuf,
guard: Box<dyn Guard>,
launcher: Option<Arc<sandbox::Launcher>>,
shell_path: Option<String>,
env: Vec<(String, String)>,
base_env: Option<Arc<Vec<(String, String)>>>,
owns_jobs: bool,
shared: Arc<Shared>,
}
#[derive(Clone, Debug, PartialEq)]
pub struct JobEnd {
pub record: JobRecord,
pub awaited: bool,
pub root_aborted: bool,
}
impl JobEnd {
pub fn event(&self) -> Option<Value> {
journal::job_event(&self.record, self.awaited, self.root_aborted)
}
}
pub type JobEnds = Arc<dyn Fn(JobEnd) + Send + Sync>;
pub type RootOf = Arc<dyn Fn(&str) -> Option<String> + Send + Sync>;
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
enum StopReason {
Cancelled,
RootAborted,
Closed,
}
struct Shared {
jobs: Mutex<Vec<Arc<Job>>>,
journal: Mutex<Option<Arc<dyn DocumentStore>>>,
released: AtomicBool,
ends: Mutex<Option<JobEnds>>,
roots: Mutex<Option<RootOf>>,
writes: cell_writes::CellWrites,
}
impl Shared {
fn persist(&self) {
self.persist_locked(&sync::lock(&self.jobs));
}
fn persist_locked(&self, jobs: &[Arc<Job>]) {
let Some(journal) = sync::lock(&self.journal).clone() else {
return;
};
if self.released.load(Ordering::SeqCst) {
return;
}
let records: Vec<JobRecord> = jobs.iter().map(|job| job_record(job, false)).collect();
let _ = journal.write(&journal::persisted(&records));
}
}
struct Record {
status: Status,
exit_code: Option<i32>,
started_at: u64,
ended_at: Option<u64>,
output_bytes: usize,
retained_bytes: usize,
tail: String,
error: Option<String>,
}
struct Job {
id: String,
lane: String,
root_id: Option<String>,
command: String,
cwd: String,
timeout: Option<f64>,
output_path: PathBuf,
raw_path: PathBuf,
exit_path: PathBuf,
record: Mutex<Record>,
process: Mutex<Option<(i32, Option<u64>)>>,
exited: Mutex<Option<Option<i32>>>,
cancel: (Mutex<bool>, Condvar),
reason: Mutex<Option<StopReason>>,
tracking: (Mutex<Tracking>, Condvar),
waiters: AtomicUsize,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Tracking {
Following,
Released,
Ended,
}
impl Job {
fn new(
id: String,
(lane, root_id): (String, Option<String>),
(command, cwd): (String, String),
timeout: Option<f64>,
jobs_dir: &Path,
record: Record,
) -> Job {
Job {
output_path: jobs_dir.join(format!("{id}.log")),
raw_path: jobs_dir.join(format!("{id}.out")),
exit_path: jobs_dir.join(format!("{id}.exit")),
id,
lane,
root_id,
command,
cwd,
timeout,
record: Mutex::new(record),
process: Mutex::new(None),
exited: Mutex::new(None),
cancel: (Mutex::new(false), Condvar::new()),
reason: Mutex::new(None),
tracking: (Mutex::new(Tracking::Following), Condvar::new()),
waiters: AtomicUsize::new(0),
}
}
fn finished(&self) {
self.stop_following(Tracking::Ended);
}
fn stop_following(&self, to: Tracking) {
*sync::lock(&self.tracking.0) = to;
self.tracking.1.notify_all();
}
fn following(&self) -> bool {
*sync::lock(&self.tracking.0) == Tracking::Following
}
fn wait_settled(&self) {
let mut tracking = sync::lock(&self.tracking.0);
while *tracking == Tracking::Following {
tracking = self.tracking.1.wait(tracking).unwrap_or_else(PoisonError::into_inner);
}
}
}
fn job_record(job: &Job, tail: bool) -> JobRecord {
let record = sync::lock(&job.record);
let process = *sync::lock(&job.process);
JobRecord {
id: job.id.clone(),
lane: job.lane.clone(),
root_id: job.root_id.clone(),
command: job.command.clone(),
cwd: job.cwd.clone(),
timeout: job.timeout,
status: record.status,
exit_code: record.exit_code.map(i64::from),
started_at: record.started_at,
ended_at: record.ended_at,
output_path: job.output_path.to_string_lossy().into_owned(),
output_bytes: record.output_bytes as u64,
retained_bytes: record.retained_bytes as u64,
tail: if tail { record.tail.clone() } else { String::new() },
error: record.error.clone(),
pid: process.map(|(pid, _)| i64::from(pid)),
pid_start: process.and_then(|(_, start)| start),
}
}
fn process_start(pid: i32) -> Option<u64> {
let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
let fields: Vec<&str> = stat.rsplit_once(')')?.1.split_whitespace().collect();
if fields.first() == Some(&"Z") {
return None;
}
fields.get(19)?.parse().ok()
}
fn keeper_alive(process: Option<(i32, Option<u64>)>) -> bool {
match process {
Some((pid, start)) => process_start(pid).is_some_and(|now| start.is_none_or(|start| start == now)),
None => false,
}
}
fn exit_status(path: &Path) -> Option<i32> {
let text = std::fs::read_to_string(path).ok()?;
text.split_whitespace().last()?.parse().ok()
}
fn command_of(payload: &Map<String, Value>) -> Result<String, String> {
match payload.get("command") {
Some(Value::String(command)) if !command.trim().is_empty() => Ok(command.clone()),
_ => Err("bash command must be a non-empty string".into()),
}
}
fn timeout_of(value: Option<&Value>) -> Result<Option<f64>, String> {
match value {
None | Some(Value::Null) => Ok(None),
Some(value) => match value.as_f64() {
Some(seconds) if seconds > 0.0 => Ok(Some(seconds.min(MAX_TIMEOUT_SECONDS))),
_ => Err("bash timeout must be a positive number of seconds".into()),
},
}
}
fn wait_seconds(value: Option<&Value>, name: &str) -> Result<Option<f64>, String> {
match value {
None | Some(Value::Null) => Ok(None),
Some(value) => match value.as_f64() {
Some(seconds) if seconds >= 0.0 => Ok(Some(seconds.min(JOB_MAX_WAIT_SECONDS))),
_ => Err(format!("{name} must be a non-negative number of seconds")),
},
}
}
fn safe_integer(value: Option<&Value>, default: u64) -> Option<u64> {
match value {
None => Some(default),
Some(value) => jsrt::number::safe_integer(value).and_then(|n| u64::try_from(n).ok()),
}
}
pub const PARENT_TOKEN_ENV: &str = "ULTRON_PARENT_TOKEN";
pub fn sandboxed_argv(argv: Vec<String>, env: &[(String, String)]) -> Vec<String> {
if env.is_empty() {
return argv;
}
std::iter::once("env".to_string())
.chain(env.iter().map(|(key, value)| format!("{key}={value}")))
.chain(argv)
.collect()
}
#[derive(Clone, Copy)]
struct CommandEnv<'a> {
cwd: &'a Path,
base: Option<&'a [(String, String)]>,
env: &'a [(String, String)],
}
fn path_of(env: &[(String, String)]) -> Option<String> {
env.iter()
.find(|(key, _)| key == "PATH")
.map(|(_, value)| value.clone())
}
fn command_env(process: &mut Command, base: Option<&[(String, String)]>, env: &[(String, String)], isolated: bool) {
if let (Some(base), false) = (base, isolated) {
process.env_clear().envs(base.iter().map(|(key, value)| (key, value)));
}
process.envs(env.iter().map(|(key, value)| (key, value)));
}
fn run_shell(
shell: &shell::ShellConfig,
command: &str,
CommandEnv { cwd, base, env }: CommandEnv<'_>,
launcher: Option<&sandbox::Launcher>,
mut on_text: impl FnMut(&str, usize),
stop: impl Fn() -> bool,
) -> Result<Option<i32>, String> {
let argv = shell.argv(command);
let mut process = match launcher {
Some(launcher) => launcher.command(&sandboxed_argv(argv, env), cwd)?,
None => {
let mut process = Command::new(&argv[0]);
process.args(&argv[1..]).current_dir(cwd);
process
}
};
let isolated = launcher.is_some_and(|launcher| launcher.mechanism().is_some());
command_env(&mut process, base, env, isolated);
process.env_remove(PARENT_TOKEN_ENV);
let (reader, writer) = std::io::pipe().map_err(|error| error.to_string())?;
let stderr = writer.try_clone().map_err(|error| error.to_string())?;
let mut child = process
.stdin(Stdio::null())
.stdout(writer)
.stderr(stderr)
.process_group(0)
.spawn()
.map_err(|error| error.to_string())?;
drop(process);
let group = i32::try_from(child.id()).unwrap_or(0);
let (sender, chunks) = mpsc::channel::<Vec<u8>>();
let pump = std::thread::spawn(move || {
let mut reader = reader;
let mut buffer = vec![0u8; 64 * 1024];
while let Ok(read) = reader.read(&mut buffer) {
if read == 0 || sender.send(buffer[..read].to_vec()).is_err() {
break;
}
}
});
let mut decoder = Decoder::default();
let mut killed = false;
loop {
match chunks.recv_timeout(POLL) {
Ok(data) => on_text(&clean(&decoder.decode(&data, true)), data.len()),
Err(mpsc::RecvTimeoutError::Disconnected) => break,
Err(mpsc::RecvTimeoutError::Timeout) => {}
}
if !killed && stop() {
killed = true;
kernel::kill_command_group(group);
}
}
let _ = pump.join();
on_text(&clean(&decoder.decode(&[], false)), 0);
let status = child.wait().map_err(|error| error.to_string())?;
Ok(status.code())
}
impl Host {
pub fn guard_stats(&self) -> Vec<Value> {
self.guard.guard_stats()
}
pub fn begin_turn(&self) {
self.guard.begin_turn();
}
pub fn cell_started(&self, lane: &str) {
self.shared.writes.begin(lane);
self.guard.cell_started(lane);
}
pub fn recording_cell_writes(self, on: bool) -> Self {
self.shared.writes.set_enabled(on);
self
}
pub fn cwd(&self) -> &Path {
&self.cwd
}
pub fn take_cell_writes(&self, lane: &str) -> Vec<cell_writes::CellWrite> {
self.shared.writes.take(lane)
}
pub fn cell_ended(&self, lane: &str) {
self.guard.cell_ended(lane);
}
pub fn take_pending(&self, lane: &str) -> Option<String> {
self.guard.take_pending(lane)
}
pub fn new(
cwd: impl Into<PathBuf>,
jobs_dir: impl Into<PathBuf>,
spill_dir: impl Into<PathBuf>,
guard: Box<dyn Guard>,
) -> Self {
let jobs_dir = jobs_dir.into();
let _ = std::fs::DirBuilder::new().recursive(true).mode(0o700).create(&jobs_dir);
let cwd = cwd.into();
Host {
shared: Arc::new(Shared {
jobs: Mutex::new(Vec::new()),
journal: Mutex::new(None),
released: AtomicBool::new(false),
ends: Mutex::new(None),
roots: Mutex::new(None),
writes: cell_writes::CellWrites::default(),
}),
cwd,
jobs_dir,
spill_dir: spill_dir.into(),
guard,
launcher: None,
shell_path: None,
env: Vec::new(),
base_env: None,
owns_jobs: true,
}
}
pub fn with_base_env(mut self, env: Vec<(String, String)>) -> Self {
self.base_env = Some(Arc::new(env));
self
}
pub fn with_env(mut self, env: Vec<(String, String)>) -> Self {
self.env = env;
self
}
pub fn sharing_jobs(mut self, session: &Host) -> Self {
self.shared = session.shared.clone();
self.owns_jobs = false;
self
}
pub fn with_shell_path(mut self, shell_path: Option<String>) -> Self {
self.shell_path = shell_path;
self
}
fn shell(&self) -> Result<shell::ShellConfig, String> {
self.shell_named(self.shell_path.as_deref())
}
fn shell_named(&self, custom: Option<&str>) -> Result<shell::ShellConfig, String> {
match &self.base_env {
Some(env) => shell::shell_config_on_path(custom, path_of(env).as_deref()),
None => shell::system_shell_config(custom),
}
}
pub fn with_journal(self, journal: Arc<dyn DocumentStore>) -> Result<Host, String> {
let now = jsrt::date::now_ms() as u64;
let restored = journal::restore_durable(journal.read()?.as_ref(), now)?;
let mut adopted = Vec::new();
{
let mut jobs = sync::lock(&self.shared.jobs);
for saved in restored.records {
let job = Arc::new(Job::new(
saved.id.clone(),
(saved.lane.clone(), saved.root_id.clone()),
(saved.command.clone(), saved.cwd.clone()),
saved.timeout,
Path::new(&saved.output_path).parent().unwrap_or(&self.jobs_dir),
Record {
status: saved.status,
exit_code: saved.exit_code.and_then(|code| i32::try_from(code).ok()),
started_at: saved.started_at,
ended_at: saved.ended_at,
output_bytes: saved.output_bytes as usize,
retained_bytes: saved.retained_bytes as usize,
tail: saved.tail.clone(),
error: saved.error.clone(),
},
));
if saved.status == Status::Running {
let pid = saved.pid.and_then(|pid| i32::try_from(pid).ok());
*sync::lock(&job.process) = pid.map(|pid| (pid, saved.pid_start));
adopted.push(job.clone());
} else {
job.finished();
}
jobs.push(job);
}
}
*sync::lock(&self.shared.journal) = Some(journal);
for job in adopted {
if keeper_alive(*sync::lock(&job.process)) {
let (runner, shared) = (job.clone(), self.shared.clone());
std::thread::spawn(move || follow(&runner, None, &shared));
} else {
follow(&job, None, &self.shared);
}
}
self.shared.persist();
Ok(self)
}
pub fn release_jobs(&self) {
loop {
let jobs = sync::lock(&self.shared.jobs);
let stopping: Vec<Arc<Job>> = jobs
.iter()
.filter(|job| *sync::lock(&job.cancel.0) && job.following())
.cloned()
.collect();
if stopping.is_empty() {
self.shared.persist_locked(&jobs);
self.shared.released.store(true, Ordering::SeqCst);
break;
}
drop(jobs);
for job in stopping {
job.wait_settled();
}
}
for job in sync::lock(&self.shared.jobs).iter() {
job.cancel.1.notify_all();
}
}
pub fn stop_jobs(&self) {
let jobs = sync::lock(&self.shared.jobs).clone();
for job in jobs {
cancel(&job, StopReason::Closed, &self.shared);
}
}
pub fn on_job_end(&self, ends: JobEnds) {
*sync::lock(&self.shared.ends) = Some(ends);
}
pub fn set_roots(&self, roots: RootOf) {
*sync::lock(&self.shared.roots) = Some(roots);
}
pub fn cancel_root(&self, root: &str) {
let jobs: Vec<Arc<Job>> = sync::lock(&self.shared.jobs)
.iter()
.filter(|job| job.root_id.as_deref() == Some(root))
.cloned()
.collect();
for job in jobs {
cancel(&job, StopReason::RootAborted, &self.shared);
}
}
pub fn job_waiters(&self, id: &str) -> usize {
sync::lock(&self.shared.jobs)
.iter()
.find(|job| job.id == id)
.map_or(0, |job| job.waiters.load(Ordering::SeqCst))
}
pub fn job_records(&self, lane: &str) -> Vec<JobRecord> {
sync::lock(&self.shared.jobs)
.iter()
.rev()
.filter(|job| visible(job, lane))
.map(|job| job_record(job, true))
.collect()
}
pub fn running_jobs(&self) -> usize {
sync::lock(&self.shared.jobs)
.iter()
.filter(|job| sync::lock(&job.record).status == Status::Running)
.count()
}
pub fn running(&self, lane: &str, excluded: &dyn Fn(Option<&str>) -> bool) -> usize {
sync::lock(&self.shared.jobs)
.iter()
.filter(|job| job.lane == lane && !excluded(job.root_id.as_deref()))
.filter(|job| sync::lock(&job.record).status == Status::Running)
.count()
}
pub fn handle_lane(
&self,
lane: &str,
kind: &str,
payload: Map<String, Value>,
signal: &Signal,
) -> Result<Value, String> {
if kind == "files.before_write" {
let writes = match payload.get("writes") {
Some(Value::Array(writes)) if (1..=64).contains(&writes.len()) => writes,
_ => return Err("files.before_write needs 1 to 64 writes".into()),
};
let mut proposed = Vec::new();
for write in writes {
match (write.get("path"), write.get("content")) {
(Some(Value::String(path)), Some(Value::String(content))) if !path.is_empty() => {
proposed.push((path.as_str(), content.as_str()));
}
_ => return Err("files.before_write writes need a path and text content".into()),
}
}
let results: Vec<Value> = proposed
.into_iter()
.map(|(path, content)| {
let result = self.guard.before_write(lane, path, content);
if result["blocked"] != true {
let path = file_hooks::resolve_lexical(&self.cwd, path);
self.shared.writes.record(lane, &path, content);
}
result
})
.collect();
return Ok(json!({ "results": results }));
}
if matches!(kind, "bash" | "shell.run")
&& let Some(Value::String(command)) = payload.get("command")
&& let Some(reason) = self.guard.before_shell(lane, command)
{
return Err(format!("bash command was not run: {reason}"));
}
let requested = payload.get("command").and_then(Value::as_str).map(str::to_string);
let mut result = self.dispatch_shell(lane, kind, payload, signal)?;
if matches!(
kind,
"bash" | "shell.run" | "shell.result" | "shell.get" | "shell.cancel"
) {
let command = requested.unwrap_or_else(|| result["command"].as_str().unwrap_or_default().to_string());
let policy = self
.launcher
.as_deref()
.filter(|launcher| launcher.mechanism().is_some())
.map(|launcher| &launcher.policy);
bash_notes::annotate(&mut result, &command, policy);
}
Ok(result)
}
fn dispatch_shell(
&self,
lane: &str,
kind: &str,
payload: Map<String, Value>,
signal: &Signal,
) -> Result<Value, String> {
match kind {
"bash" if payload.contains_key("yield_after") => self.shell_bash(lane, &payload, signal),
"bash" => self.blocking_bash(&payload, signal, &mut |_| {}),
"shell.run" => {
kernel::ops::fields(kind, &payload)?;
let command = command_of(&payload)?;
let timeout = timeout_of(payload.get("timeout"))?;
let yield_after = wait_seconds(payload.get("yield_after"), "yield_after")?.unwrap_or(0.0);
let job = self.start(lane, command, timeout)?;
wait(&job, Some(yield_after), signal);
Ok(snapshot(&job))
}
"shell.result" => {
kernel::ops::fields(kind, &payload)?;
let job = self.find(lane, payload.get("id"))?;
if sync::lock(&job.record).status == Status::Running {
wait(&job, wait_seconds(payload.get("wait"), "wait")?, signal);
}
Ok(snapshot(&job))
}
"shell.get" => {
kernel::ops::fields(kind, &payload)?;
let job = self.find(lane, payload.get("id"))?;
Ok(snapshot(&job))
}
"shell.cancel" => {
kernel::ops::fields(kind, &payload)?;
let job = self.find(lane, payload.get("id"))?;
cancel(&job, StopReason::Cancelled, &self.shared);
Ok(snapshot(&job))
}
"shell.read" => {
kernel::ops::fields(kind, &payload)?;
let job = self.find(lane, payload.get("id"))?;
read_output(&job, payload.get("cursor"), payload.get("max_bytes"))
}
"shell.list" => {
kernel::ops::fields(kind, &payload)?;
let jobs = sync::lock(&self.shared.jobs).clone();
Ok(Value::Array(
jobs.iter()
.rev()
.filter(|job| visible(job, lane))
.map(|job| {
let mut snapshot = snapshot(job);
snapshot["text"] = Value::Null;
snapshot
})
.collect(),
))
}
other if other.starts_with("shell.") => Err(kernel::ops::not_wired(other)),
other => Err(not_available(other)),
}
}
pub fn user_bash(&self, command: &str, signal: &Signal, on_chunk: &mut dyn FnMut(&str)) -> Result<Value, String> {
if let Some(reason) = self.guard.before_shell(kernel::ROOT_LANE, command) {
return Err(format!("bash command was not run: {reason}"));
}
let mut payload = Map::new();
payload.insert("command".into(), command.into());
self.blocking_bash(&payload, signal, on_chunk)
}
fn blocking_bash(
&self,
payload: &Map<String, Value>,
signal: &Signal,
on_chunk: &mut dyn FnMut(&str),
) -> Result<Value, String> {
let command = command_of(payload)?;
let timeout = timeout_of(payload.get("timeout"))?;
let deadline = timeout.map(|seconds| Instant::now() + Duration::from_secs_f64(seconds));
let timed_out = std::cell::Cell::new(false);
let mut chunks: std::collections::VecDeque<String> = Default::default();
let mut kept_units = 0usize;
let mut total_bytes = 0usize;
let mut spill: Option<(PathBuf, File)> = None;
let open_spill =
|spill: &mut Option<(PathBuf, File)>, chunks: &std::collections::VecDeque<String>| -> Result<(), String> {
if spill.is_none() {
let path = self
.spill_dir
.join(format!("pi-bash-{}.log", jsrt::random::random_hex(8)));
let mut file = File::create(&path).map_err(|error| error.to_string())?;
for chunk in chunks {
let _ = file.write_all(chunk.as_bytes());
}
*spill = Some((path, file));
}
Ok(())
};
let shell = self.shell()?;
let mut failure = None;
let exit = run_shell(
&shell,
&command,
CommandEnv {
cwd: &self.cwd,
base: self.base_env.as_deref().map(Vec::as_slice),
env: &self.env,
},
self.launcher.as_deref(),
|text, raw| {
if raw > 0 || !text.is_empty() {
on_chunk(text);
}
total_bytes += raw;
if total_bytes > DEFAULT_MAX_BYTES
&& let Err(error) = open_spill(&mut spill, &chunks)
{
failure = Some(error);
}
if let Some((_, file)) = spill.as_mut() {
let _ = file.write_all(text.as_bytes());
}
chunks.push_back(text.to_owned());
kept_units += text.encode_utf16().count();
while kept_units > DEFAULT_MAX_BYTES * 2 && chunks.len() > 1 {
if let Some(removed) = chunks.pop_front() {
kept_units -= removed.encode_utf16().count();
}
}
},
|| {
if deadline.is_some_and(|deadline| Instant::now() >= deadline) {
timed_out.set(true);
}
timed_out.get() || signal.is_aborted()
},
)?;
if let Some(error) = failure {
return Err(error);
}
let full: String = chunks.iter().map(String::as_str).collect();
let truncated = truncate_tail(&full);
if truncated.is_some() {
open_spill(&mut spill, &chunks)?;
}
let stopped = timed_out.get() || signal.is_aborted();
Ok(json!({
"output": truncated.clone().unwrap_or(full),
"exit_code": if stopped { None } else { exit },
"cancelled": stopped && !timed_out.get(),
"timed_out": timed_out.get(),
"truncated": truncated.is_some(),
"full_output_path": spill.map(|(path, _)| path.to_string_lossy().into_owned()),
}))
}
fn shell_bash(&self, lane: &str, payload: &Map<String, Value>, signal: &Signal) -> Result<Value, String> {
kernel::ops::fields("bash", payload)?;
let command = command_of(payload)?;
let timeout = timeout_of(payload.get("timeout"))?;
let yield_after = wait_seconds(payload.get("yield_after"), "yield_after")?;
let job = self.start(lane, command, timeout)?;
wait(&job, yield_after, signal);
if sync::lock(&job.record).status == Status::Running && signal.is_aborted() {
job.waiters.fetch_add(1, Ordering::SeqCst);
cancel(&job, StopReason::Cancelled, &self.shared);
job.waiters.fetch_sub(1, Ordering::SeqCst);
}
bash_result(&job)
}
fn find(&self, lane: &str, id: Option<&Value>) -> Result<Arc<Job>, String> {
let Some(Value::String(id)) = id.filter(|id| id.as_str().is_some_and(|id| !id.is_empty())) else {
return Err("job id must be a nonempty string".into());
};
sync::lock(&self.shared.jobs)
.iter()
.find(|job| &job.id == id && visible(job, lane))
.cloned()
.ok_or_else(|| format!("Unknown shell job {id}"))
}
fn start(&self, lane: &str, command: String, timeout: Option<f64>) -> Result<Arc<Job>, String> {
let root = sync::lock(&self.shared.roots).clone().and_then(|roots| roots(lane));
let mut jobs = sync::lock(&self.shared.jobs);
let running = |job: &Arc<Job>| sync::lock(&job.record).status == Status::Running;
if jobs.iter().filter(|job| running(job)).count() >= JOB_MAX_ACTIVE {
return Err(format!("At most {JOB_MAX_ACTIVE} shell jobs can run at once"));
}
let id = format!("job-{}", jsrt::random::random_hex(4));
let job = Arc::new(Job::new(
id,
(lane.to_string(), root),
(command, self.cwd.to_string_lossy().into_owned()),
timeout,
&self.jobs_dir,
Record {
status: Status::Running,
exit_code: None,
started_at: jsrt::date::now_ms() as u64,
ended_at: None,
output_bytes: 0,
retained_bytes: 0,
tail: String::new(),
error: None,
},
));
jobs.push(job.clone());
let excess = jobs.len().saturating_sub(JOB_MAX_RETAINED);
let pruned: Vec<String> = jobs
.iter()
.filter(|job| !running(job))
.take(excess)
.map(|job| job.id.clone())
.collect();
jobs.retain(|job| {
let keep = !pruned.contains(&job.id);
if !keep {
for path in [&job.output_path, &job.raw_path, &job.exit_path] {
let _ = std::fs::remove_file(path);
}
}
keep
});
drop(jobs);
let (started, spawned) = mpsc::channel();
let (runner, shared, cwd, launcher, shell) = (
job.clone(),
self.shared.clone(),
(self.cwd.clone(), self.base_env.clone(), self.env.clone()),
self.launcher.clone(),
self.shell()
.and_then(|shell| Ok((shell, self.shell_named(None)?.shell))),
);
std::thread::spawn(move || {
let env = CommandEnv {
cwd: &cwd.0,
base: cwd.1.as_deref().map(Vec::as_slice),
env: &cwd.2,
};
match shell.and_then(|(shell, keeper)| spawn_keeper(&runner, (&shell, &keeper), env, launcher.as_deref())) {
Ok(child) => {
let pid = child.id() as i32;
*sync::lock(&runner.process) = Some((pid, process_start(pid)));
let _ = started.send(());
follow(&runner, Some(child), &shared);
}
Err(error) => {
let mut record = sync::lock(&runner.record);
record.status = Status::Failed;
record.ended_at = Some(jsrt::date::now_ms() as u64);
record.error = Some(error);
drop(record);
let _ = started.send(());
runner.finished();
}
}
});
let _ = spawned.recv();
self.shared.persist();
Ok(job)
}
}
const KEEPER: &str = r#""$1" "$2" "$0" 2>&1 </dev/null; printf '%s\n' "$?" >&2"#;
fn spawn_keeper(
job: &Job,
(shell, keeper): (&shell::ShellConfig, &str),
CommandEnv { cwd, base, env }: CommandEnv<'_>,
launcher: Option<&sandbox::Launcher>,
) -> Result<std::process::Child, String> {
let create = |path: &Path| {
std::fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.mode(0o600)
.open(path)
.map_err(|error| format!("{}: {error}", path.display()))
};
let raw = create(&job.raw_path)?;
let exit = create(&job.exit_path)?;
let mut argv = vec![keeper.to_string(), "-c".into(), KEEPER.into(), job.command.clone()];
argv.push(shell.shell.clone());
argv.extend(shell.args.iter().cloned());
let mut process = match launcher {
Some(launcher) => launcher.command_persistent(&sandboxed_argv(argv, env), cwd)?,
None => {
let mut process = Command::new(&argv[0]);
process.args(&argv[1..]).current_dir(cwd);
process
}
};
let isolated = launcher.is_some_and(|launcher| launcher.mechanism().is_some());
command_env(&mut process, base, env, isolated);
process
.env_remove(PARENT_TOKEN_ENV)
.stdin(Stdio::null())
.stdout(raw)
.stderr(exit)
.process_group(0)
.spawn()
.map_err(|error| error.to_string())
}
struct Digest {
decoder: Decoder,
log: Option<File>,
}
impl Digest {
fn new(job: &Job) -> Digest {
let log = std::fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.mode(0o600)
.open(&job.output_path)
.ok();
let mut record = sync::lock(&job.record);
record.output_bytes = 0;
record.retained_bytes = 0;
record.tail.clear();
Digest {
decoder: Decoder::default(),
log,
}
}
fn text(&mut self, job: &Job, text: &str) {
if text.is_empty() {
return;
}
let mut record = sync::lock(&job.record);
record.output_bytes += text.len();
let room = JOB_RETAINED_BYTES.saturating_sub(record.retained_bytes);
if room > 0 {
let kept = &text.as_bytes()[..room.min(text.len())];
if let Some(log) = self.log.as_mut() {
let _ = log.write_all(kept);
}
record.retained_bytes += kept.len();
}
let tail = format!("{}{text}", record.tail);
record.tail = tail_bytes(&tail, JOB_TEXT_BYTES / 2).to_owned();
}
fn pump(&mut self, job: &Job, raw: &mut Option<File>) {
let Some(file) = raw.as_mut() else {
return;
};
let mut buffer = vec![0u8; 64 * 1024];
while let Ok(read) = file.read(&mut buffer) {
if read == 0 {
break;
}
let text = clean(&self.decoder.decode(&buffer[..read], true));
self.text(job, &text);
}
}
fn finish(&mut self, job: &Job) {
let text = clean(&self.decoder.decode(&[], false));
self.text(job, &text);
}
}
fn follow(job: &Arc<Job>, child: Option<std::process::Child>, shared: &Shared) {
let own = child.is_some();
if let Some(mut child) = child {
let waiter = job.clone();
std::thread::spawn(move || {
let code = child.wait().ok().and_then(|status| status.code());
let guard = sync::lock(&waiter.cancel.0);
*sync::lock(&waiter.exited) = Some(code);
drop(guard);
waiter.cancel.1.notify_all();
});
}
let mut digest = Digest::new(job);
let mut raw = File::open(&job.raw_path).ok();
let started_at = sync::lock(&job.record).started_at;
let deadline = job
.timeout
.map(|seconds| started_at.saturating_add((seconds * 1000.0) as u64));
let process = *sync::lock(&job.process);
let mut stopped: Option<Status> = None;
let ended = loop {
let ended = match own {
true => *sync::lock(&job.exited),
false => {
let code = exit_status(&job.exit_path);
(code.is_some() || !keeper_alive(process)).then_some(code)
}
};
digest.pump(job, &mut raw);
if let Some(ended) = ended {
break ended;
}
if shared.released.load(Ordering::SeqCst) {
job.stop_following(Tracking::Released);
return;
}
if stopped.is_none() {
if deadline.is_some_and(|deadline| jsrt::date::now_ms() as u64 >= deadline) {
stopped = Some(Status::TimedOut);
} else if *sync::lock(&job.cancel.0) {
stopped = Some(Status::Cancelled);
}
if stopped.is_some()
&& let Some((pid, _)) = process
{
kernel::kill_command_group(pid);
}
}
let cancel = sync::lock(&job.cancel.0);
if sync::lock(&job.exited).is_none() {
drop(
job.cancel
.1
.wait_timeout(cancel, POLL)
.unwrap_or_else(PoisonError::into_inner),
);
}
};
digest.finish(job);
let code = exit_status(&job.exit_path);
let jobs = sync::lock(&shared.jobs);
if shared.released.load(Ordering::SeqCst) {
drop(jobs);
job.stop_following(Tracking::Released);
return;
}
{
let mut record = sync::lock(&job.record);
record.ended_at = Some(match own {
true => jsrt::date::now_ms() as u64,
false => std::fs::metadata(&job.exit_path)
.and_then(|meta| meta.modified())
.ok()
.filter(|_| code.is_some())
.and_then(|time| time.duration_since(UNIX_EPOCH).ok())
.map_or_else(|| jsrt::date::now_ms() as u64, |d| d.as_millis() as u64),
});
match stopped {
Some(status) => {
record.status = status;
record.exit_code = None;
if status == Status::TimedOut {
record.error = Some(format!(
"timed out after {}s; process tree killed",
jsrt::number::to_string(job.timeout.unwrap_or(0.0))
));
}
}
None => match (code, own) {
(Some(code), _) => {
record.status = Status::Completed;
record.exit_code = Some(code);
}
(None, true) => {
record.status = Status::Completed;
record.exit_code = ended;
}
(None, false) => {
record.status = Status::Lost;
record.exit_code = None;
record.error =
Some("its process ended while no Ultron worker was running; the exit status is unknown".into());
}
},
}
}
shared.persist_locked(&jobs);
drop(jobs);
let reason = *sync::lock(&job.reason);
let end = JobEnd {
record: job_record(job, true),
awaited: job.waiters.load(Ordering::SeqCst) > 0,
root_aborted: reason == Some(StopReason::RootAborted),
};
let ends = sync::lock(&shared.ends).clone();
job.finished();
if reason != Some(StopReason::Closed)
&& let Some(ends) = ends
{
ends(end);
}
}
fn wait(job: &Arc<Job>, seconds: Option<f64>, signal: &Signal) {
job.waiters.fetch_add(1, Ordering::SeqCst);
let deadline = seconds.map(|seconds| Instant::now() + Duration::from_secs_f64(seconds.max(0.02)));
let woken = job.clone();
let link = signal.link(move |_| {
let _tracking = sync::lock(&woken.tracking.0);
woken.tracking.1.notify_all();
});
let mut done = sync::lock(&job.tracking.0);
while *done == Tracking::Following && !signal.is_aborted() {
done = match deadline {
None => job.tracking.1.wait(done).unwrap_or_else(PoisonError::into_inner),
Some(deadline) => {
let now = Instant::now();
if now >= deadline {
break;
}
job.tracking
.1
.wait_timeout(done, deadline - now)
.unwrap_or_else(PoisonError::into_inner)
.0
}
};
}
drop(done);
drop(link);
job.waiters.fetch_sub(1, Ordering::SeqCst);
}
fn cancel(job: &Job, reason: StopReason, shared: &Shared) {
{
let _jobs = sync::lock(&shared.jobs);
if shared.released.load(Ordering::SeqCst) || !job.following() {
return;
}
sync::lock(&job.reason).get_or_insert(reason);
*sync::lock(&job.cancel.0) = true;
}
job.cancel.1.notify_all();
job.wait_settled();
}
fn read_file(path: &Path, cursor: usize, max: usize) -> String {
let mut buffer = Vec::new();
if let Ok(mut file) = File::open(path)
&& file.seek(SeekFrom::Start(cursor as u64)).is_ok()
{
let _ = file.take(max as u64).read_to_end(&mut buffer);
}
String::from_utf8_lossy(&buffer).into_owned()
}
fn job_text(job: &Job, record: &Record) -> (String, bool) {
let half = JOB_TEXT_BYTES / 2;
if record.output_bytes <= JOB_TEXT_BYTES && record.retained_bytes == record.output_bytes {
return (read_file(&job.output_path, 0, JOB_TEXT_BYTES), false);
}
let head = read_file(&job.output_path, 0, half);
let omitted = record.output_bytes.saturating_sub(half + record.tail.len());
let marker = format!(
"\n[... {omitted} bytes omitted; {} bytes kept in {}: await job.read(cursor={half}, max_bytes=65536) ...]\n",
record.retained_bytes,
job.output_path.display()
);
(format!("{head}{marker}{}", record.tail), true)
}
fn snapshot(job: &Job) -> Value {
let record = sync::lock(&job.record);
let (text, truncated) = job_text(job, &record);
let elapsed = record
.ended_at
.unwrap_or_else(|| jsrt::date::now_ms() as u64)
.saturating_sub(record.started_at);
json!({
"id": job.id,
"lane": job.lane,
"command": job.command,
"status": record.status.as_str(),
"running": record.status == Status::Running,
"exit_code": record.exit_code,
"ok": record.status == Status::Completed && record.exit_code == Some(0),
"timed_out": record.status == Status::TimedOut,
"cancelled": record.status == Status::Cancelled,
"text": text,
"truncated": truncated,
"output_path": job.output_path.to_string_lossy(),
"output_bytes": record.output_bytes,
"elapsed_seconds": jsrt::json::number((elapsed as f64 / 100.0).round() / 10.0),
"error": record.error,
})
}
fn bash_result(job: &Job) -> Result<Value, String> {
let (status, exit_code, error, partial) = {
let record = sync::lock(&job.record);
(
record.status,
record.exit_code,
record.error.clone(),
record.retained_bytes < record.output_bytes,
)
};
if status == Status::Failed {
return Err(error.unwrap_or_else(|| "bash command failed".into()));
}
let snapshot = snapshot(job);
let truncated = snapshot["truncated"].as_bool() == Some(true);
let running = status == Status::Running;
Ok(json!({
"output": snapshot["text"],
"exit_code": exit_code,
"cancelled": status == Status::Cancelled,
"timed_out": status == Status::TimedOut,
"truncated": truncated,
"full_output_path": if truncated { snapshot["output_path"].clone() } else { Value::Null },
"partial_file": partial,
"running": running,
"job": if running { snapshot } else { Value::Null },
}))
}
fn read_output(job: &Job, cursor: Option<&Value>, max_bytes: Option<&Value>) -> Result<Value, String> {
let cursor = safe_integer(cursor, 0).ok_or("cursor must be a non-negative integer")? as usize;
let max = safe_integer(max_bytes, 65_536)
.filter(|max| (1..=1_048_576).contains(max))
.ok_or("max_bytes must be an integer between 1 and 1048576")? as usize;
let record = sync::lock(&job.record);
let start = cursor.min(record.retained_bytes);
let text = read_file(&job.output_path, start, max.min(record.retained_bytes - start));
let next = start + text.len();
Ok(json!({
"text": text,
"next_cursor": next,
"done": record.status != Status::Running && next >= record.retained_bytes,
"truncated": record.retained_bytes < record.output_bytes,
}))
}
impl HostHandler for Host {
fn handle(&self, kind: &str, payload: Map<String, Value>, signal: &Signal) -> Result<Value, String> {
self.handle_lane(kernel::ROOT_LANE, kind, payload, signal)
}
}
fn visible(job: &Job, lane: &str) -> bool {
kernel::sees(lane, &job.lane)
}
impl Drop for Host {
fn drop(&mut self) {
if self.owns_jobs && !self.shared.released.load(Ordering::SeqCst) {
self.stop_jobs();
}
}
}