Skip to main content

host/
lib.rs

1//! Host operations the Python kernel API calls (RLM-first core), served as a [`kernel::HostHandler`].
2//!
3//! Ported from the TypeScript session worker's host handler (experimental/session-worker.ts), its blocking
4//! `bash` (ultron/rlm/host-bash.ts over core/bash-executor.ts) and host-owned shell jobs (ultron/rlm/shell-jobs.ts).
5//! `read`, `write` and `edit` run inside runtime.py; the host only answers their `files.before_write` check.
6//! Requests that need a model, subagents or durable stores are refused with [`not_available`].
7//!
8//! Shell jobs are durable (`Host::with_journal`): each runs under a small keeper shell in its own process group that
9//! writes the raw output to `<id>.out` and the exit status to `<id>.exit`, so the job does not depend on the worker
10//! that started it. The host follows those files into the cleaned, bounded spill file `<id>.log` and journals every
11//! start and end. A reopened worker settles each job the journal lists as running from what really happened: still
12//! running (the keeper is alive: it is followed again), completed with the status the keeper wrote, or `lost` (the
13//! process ended without one); never silently gone.
14
15pub mod bash_notes;
16pub mod cell;
17pub mod cell_writes;
18pub mod file_hooks;
19pub mod journal;
20pub mod loki_guard;
21pub mod sandboxed;
22pub mod secrets;
23pub mod shell;
24pub mod text;
25pub mod view_image;
26
27use journal::{JobRecord, Status};
28use jsrt::sync;
29use kernel::store::DocumentStore;
30use kernel::{HostHandler, Signal};
31use serde_json::{Map, Value, json};
32use std::fs::File;
33use std::io::{Read, Seek, SeekFrom, Write};
34use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
35use std::os::unix::process::CommandExt;
36use std::path::{Path, PathBuf};
37use std::process::{Command, Stdio};
38use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
39use std::sync::{Arc, Condvar, Mutex, PoisonError, mpsc};
40use std::time::{Duration, Instant, UNIX_EPOCH};
41use text::{DEFAULT_MAX_BYTES, Decoder, clean, tail_bytes, truncate_tail};
42
43pub const JOB_TEXT_BYTES: usize = 16 * 1024;
44pub const JOB_RETAINED_BYTES: usize = 4 * 1024 * 1024;
45pub const JOB_MAX_ACTIVE: usize = 32;
46pub const JOB_MAX_RETAINED: usize = 64;
47const JOB_MAX_WAIT_SECONDS: f64 = 3600.0;
48const MAX_TIMEOUT_SECONDS: f64 = 86_400.0;
49/// How often a waiting thread re-checks its cell's abort signal and a running command its stop condition.
50// ponytail: 10 ms polling instead of wiring the kernel Signal into a condvar; fine for a handful of waiters.
51const POLL: Duration = Duration::from_millis(10);
52
53/// The before-write and before-shell guards (Loki). The default allows everything.
54pub trait Guard: Send + Sync {
55    /// A command of the cell on `lane`: `Some(reason)` blocks it, "bash command was not run: {reason}".
56    fn before_shell(&self, _lane: &str, _command: &str) -> Option<String> {
57        None
58    }
59    /// A write of the cell on `lane`: `{blocked, reason?, notes?}` as runtime.py reads it.
60    fn before_write(&self, _lane: &str, _path: &str, _content: &str) -> Value {
61        json!({ "blocked": false })
62    }
63    /// Each file-hook guard's statistics (`fileHooks.stats()`), for `rlm.pool`.
64    fn guard_stats(&self) -> Vec<Value> {
65        Vec::new()
66    }
67    /// A new root turn began: per-turn guard times start from zero.
68    fn begin_turn(&self) {}
69    /// A cell on `lane` is starting (after-cell checks take their baseline).
70    fn cell_started(&self, _lane: &str) {}
71    /// A cell on `lane` ended (after-cell checks start in the background).
72    fn cell_ended(&self, _lane: &str) {}
73    /// What after-cell checks found for `lane` and it has not been shown yet.
74    fn take_pending(&self, _lane: &str) -> Option<String> {
75        None
76    }
77}
78
79pub struct AllowAll;
80impl Guard for AllowAll {}
81
82pub fn not_available(kind: &str) -> String {
83    format!("Ultron RLM host request is not available in this host yet: {kind}")
84}
85
86pub struct Host {
87    cwd: PathBuf,
88    jobs_dir: PathBuf,
89    spill_dir: PathBuf,
90    guard: Box<dyn Guard>,
91    /// Runs shell commands isolated (see [`Host::sandboxed`]); None runs them as before.
92    launcher: Option<Arc<sandbox::Launcher>>,
93    /// The `shellPath` setting (`~` expanded); None or empty runs the default shell ([`shell::shell_config`]).
94    shell_path: Option<String>,
95    /// Extra environment of every command (a worktree subagent's `ULTRON_WORKTREE`, `ULTRON_PARENT_REPO`,
96    /// `ULTRON_WORKTREE_BRANCH`, `PWD`), over the inherited one.
97    env: Vec<(String, String)>,
98    /// The environment every command starts from (the session's own, [`Host::with_base_env`]); None: this
99    /// process's.
100    base_env: Option<Arc<Vec<(String, String)>>>,
101    /// Whether this host owns its jobs table: a host sharing another's ([`Host::sharing_jobs`]) never stops them.
102    owns_jobs: bool,
103    shared: Arc<Shared>,
104}
105
106/// A job that ended, as `ShellJobs` `onEnd` reports it: its record, whether a caller was waiting on it (the result
107/// came inline, so nothing is announced) and whether its root turn's abort stopped it.
108#[derive(Clone, Debug, PartialEq)]
109pub struct JobEnd {
110    pub record: JobRecord,
111    pub awaited: bool,
112    pub root_aborted: bool,
113}
114
115impl JobEnd {
116    /// The `job_done` event (`jobEvent`), unless the end was awaited or its root turn aborted.
117    pub fn event(&self) -> Option<Value> {
118        journal::job_event(&self.record, self.awaited, self.root_aborted)
119    }
120}
121
122/// Sees every job end (not those stopped by the host's shutdown).
123pub type JobEnds = Arc<dyn Fn(JobEnd) + Send + Sync>;
124/// The usage root a job started by `lane` belongs to (`host.rootOf(caller)`): the root turn running on the root
125/// lane, or the root of the task that owns another lane.
126pub type RootOf = Arc<dyn Fn(&str) -> Option<String> + Send + Sync>;
127
128#[derive(Clone, Copy, PartialEq, Eq, Debug)]
129enum StopReason {
130    /// `job.cancel()`.
131    Cancelled,
132    /// Its root turn was aborted (Esc): stopped, and its end is not announced.
133    RootAborted,
134    /// The host shut down: nothing is reported.
135    Closed,
136}
137
138/// What job threads share with the host: the jobs, the journal they are written to, whether the host released
139/// its jobs to a next owner, and who hears of their ends.
140struct Shared {
141    jobs: Mutex<Vec<Arc<Job>>>,
142    journal: Mutex<Option<Arc<dyn DocumentStore>>>,
143    released: AtomicBool,
144    ends: Mutex<Option<JobEnds>>,
145    roots: Mutex<Option<RootOf>>,
146    /// What each lane's running cell wrote through checked writes ([`cell_writes`]).
147    writes: cell_writes::CellWrites,
148}
149
150impl Shared {
151    /// Writes the journal (every job, tails left out: they are not even copied), serialized under the jobs lock so the
152    /// newest state wins.
153    fn persist(&self) {
154        self.persist_locked(&sync::lock(&self.jobs));
155    }
156
157    /// [`Shared::persist`] with the jobs lock held: a job's end and the release to a next owner are decided under it,
158    /// so exactly one of the two workers settles each job.
159    fn persist_locked(&self, jobs: &[Arc<Job>]) {
160        let Some(journal) = sync::lock(&self.journal).clone() else {
161            return;
162        };
163        if self.released.load(Ordering::SeqCst) {
164            return;
165        }
166        let records: Vec<JobRecord> = jobs.iter().map(|job| job_record(job, false)).collect();
167        let _ = journal.write(&journal::persisted(&records));
168    }
169}
170
171struct Record {
172    status: Status,
173    exit_code: Option<i32>,
174    started_at: u64,
175    ended_at: Option<u64>,
176    output_bytes: usize,
177    retained_bytes: usize,
178    tail: String,
179    error: Option<String>,
180}
181
182struct Job {
183    id: String,
184    /// The lane that started the job (it and the root lane see it) and that lane's root turn.
185    lane: String,
186    root_id: Option<String>,
187    command: String,
188    /// Where the command runs (`laneWorkspace(lane).cwd`, else the session's).
189    cwd: String,
190    timeout: Option<f64>,
191    /// The cleaned, bounded output the kernel reads.
192    output_path: PathBuf,
193    /// The keeper's raw output and the exit status it writes when the command ends.
194    raw_path: PathBuf,
195    exit_path: PathBuf,
196    record: Mutex<Record>,
197    /// The keeper process (process group leader) and its start time.
198    process: Mutex<Option<(i32, Option<u64>)>>,
199    /// Set by the waiter of this worker's own keeper when it exits (its status code); guarded by `cancel.0`'s lock
200    /// for wake-ups.
201    exited: Mutex<Option<Option<i32>>>,
202    /// Cancel requested; its condvar wakes the follower (cancel, keeper exit, release).
203    cancel: (Mutex<bool>, Condvar),
204    /// Why the first stop asked for it.
205    reason: Mutex<Option<StopReason>>,
206    /// Whether this worker still follows the job; its condvar wakes waiters and cancellers when that ends.
207    tracking: (Mutex<Tracking>, Condvar),
208    waiters: AtomicUsize,
209}
210
211/// Where this worker is with a job. Waiters and cancellers wait while it is `Following`.
212#[derive(Clone, Copy, Debug, PartialEq, Eq)]
213enum Tracking {
214    /// Its follower runs (or will run) and settles it.
215    Following,
216    /// Handed to the next owner of the journal ([`Host::release_jobs`]): this worker never settles it.
217    Released,
218    /// Settled: its record holds its end.
219    Ended,
220}
221
222impl Job {
223    fn new(
224        id: String,
225        (lane, root_id): (String, Option<String>),
226        (command, cwd): (String, String),
227        timeout: Option<f64>,
228        jobs_dir: &Path,
229        record: Record,
230    ) -> Job {
231        Job {
232            output_path: jobs_dir.join(format!("{id}.log")),
233            raw_path: jobs_dir.join(format!("{id}.out")),
234            exit_path: jobs_dir.join(format!("{id}.exit")),
235            id,
236            lane,
237            root_id,
238            command,
239            cwd,
240            timeout,
241            record: Mutex::new(record),
242            process: Mutex::new(None),
243            exited: Mutex::new(None),
244            cancel: (Mutex::new(false), Condvar::new()),
245            reason: Mutex::new(None),
246            tracking: (Mutex::new(Tracking::Following), Condvar::new()),
247            waiters: AtomicUsize::new(0),
248        }
249    }
250
251    fn finished(&self) {
252        self.stop_following(Tracking::Ended);
253    }
254
255    fn stop_following(&self, to: Tracking) {
256        *sync::lock(&self.tracking.0) = to;
257        self.tracking.1.notify_all();
258    }
259
260    fn following(&self) -> bool {
261        *sync::lock(&self.tracking.0) == Tracking::Following
262    }
263
264    /// Waits until this worker settled the job or released it.
265    fn wait_settled(&self) {
266        let mut tracking = sync::lock(&self.tracking.0);
267        while *tracking == Tracking::Following {
268            tracking = self.tracking.1.wait(tracking).unwrap_or_else(PoisonError::into_inner);
269        }
270    }
271}
272
273/// The job's record; `tail`: with its output tail (the journal leaves it out).
274fn job_record(job: &Job, tail: bool) -> JobRecord {
275    let record = sync::lock(&job.record);
276    let process = *sync::lock(&job.process);
277    JobRecord {
278        id: job.id.clone(),
279        lane: job.lane.clone(),
280        root_id: job.root_id.clone(),
281        command: job.command.clone(),
282        cwd: job.cwd.clone(),
283        timeout: job.timeout,
284        status: record.status,
285        exit_code: record.exit_code.map(i64::from),
286        started_at: record.started_at,
287        ended_at: record.ended_at,
288        output_path: job.output_path.to_string_lossy().into_owned(),
289        output_bytes: record.output_bytes as u64,
290        retained_bytes: record.retained_bytes as u64,
291        tail: if tail { record.tail.clone() } else { String::new() },
292        error: record.error.clone(),
293        pid: process.map(|(pid, _)| i64::from(pid)),
294        pid_start: process.and_then(|(_, start)| start),
295    }
296}
297
298/// Start time of a live (not zombie) process, in clock ticks since boot (`/proc/<pid>/stat` field 22).
299fn process_start(pid: i32) -> Option<u64> {
300    let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
301    let fields: Vec<&str> = stat.rsplit_once(')')?.1.split_whitespace().collect();
302    if fields.first() == Some(&"Z") {
303        return None;
304    }
305    fields.get(19)?.parse().ok()
306}
307
308/// Whether the keeper recorded for a job still runs (same pid and start time).
309fn keeper_alive(process: Option<(i32, Option<u64>)>) -> bool {
310    match process {
311        Some((pid, start)) => process_start(pid).is_some_and(|now| start.is_none_or(|start| start == now)),
312        None => false,
313    }
314}
315
316/// The exit status the keeper wrote, once the command has ended.
317fn exit_status(path: &Path) -> Option<i32> {
318    let text = std::fs::read_to_string(path).ok()?;
319    text.split_whitespace().last()?.parse().ok()
320}
321
322fn command_of(payload: &Map<String, Value>) -> Result<String, String> {
323    match payload.get("command") {
324        Some(Value::String(command)) if !command.trim().is_empty() => Ok(command.clone()),
325        _ => Err("bash command must be a non-empty string".into()),
326    }
327}
328
329/// bashTimeoutSeconds: absent or null is no limit; at most a day.
330fn timeout_of(value: Option<&Value>) -> Result<Option<f64>, String> {
331    match value {
332        None | Some(Value::Null) => Ok(None),
333        Some(value) => match value.as_f64() {
334            Some(seconds) if seconds > 0.0 => Ok(Some(seconds.min(MAX_TIMEOUT_SECONDS))),
335            _ => Err("bash timeout must be a positive number of seconds".into()),
336        },
337    }
338}
339
340fn wait_seconds(value: Option<&Value>, name: &str) -> Result<Option<f64>, String> {
341    match value {
342        None | Some(Value::Null) => Ok(None),
343        Some(value) => match value.as_f64() {
344            Some(seconds) if seconds >= 0.0 => Ok(Some(seconds.min(JOB_MAX_WAIT_SECONDS))),
345            _ => Err(format!("{name} must be a non-negative number of seconds")),
346        },
347    }
348}
349
350/// `Number.isSafeInteger(value) && value >= 0`, or `default` when absent.
351fn safe_integer(value: Option<&Value>, default: u64) -> Option<u64> {
352    match value {
353        None => Some(default),
354        Some(value) => jsrt::number::safe_integer(value).and_then(|n| u64::try_from(n).ok()),
355    }
356}
357
358/// The verdict token of an `ultron mcp --child` server (`ULTRON_PARENT_TOKEN`), removed from every command's
359/// environment.
360pub const PARENT_TOKEN_ENV: &str = "ULTRON_PARENT_TOKEN";
361
362/// `argv` as it runs inside a sandbox: every sandbox clears the environment to its allowlist (bwrap `--clearenv`,
363/// `env -i` under unshare and in a Docker container), so the command's own variables (a worktree subagent's
364/// `ULTRON_WORKTREE`, `ULTRON_PARENT_REPO`, `ULTRON_WORKTREE_BRANCH`, `PWD`) are set inside, by `env`.
365pub fn sandboxed_argv(argv: Vec<String>, env: &[(String, String)]) -> Vec<String> {
366    if env.is_empty() {
367        return argv;
368    }
369    std::iter::once("env".to_string())
370        .chain(env.iter().map(|(key, value)| format!("{key}={value}")))
371        .chain(argv)
372        .collect()
373}
374
375/// Where a command runs: its directory, the base environment (None: this process's) and its own variables.
376#[derive(Clone, Copy)]
377struct CommandEnv<'a> {
378    cwd: &'a Path,
379    base: Option<&'a [(String, String)]>,
380    env: &'a [(String, String)],
381}
382
383fn path_of(env: &[(String, String)]) -> Option<String> {
384    env.iter()
385        .find(|(key, _)| key == "PATH")
386        .map(|(_, value)| value.clone())
387}
388
389/// Gives `process` its environment: an unisolated command gets the base environment (when there is one) and then
390/// `env`. A sandboxed one keeps the launcher's own (the sandbox crate's `launch_env`, as its mechanism probe uses):
391/// inside, the command sees the policy's allowlist, valued from the base environment ([`Host::sandboxed`]), and its
392/// own variables ([`sandboxed_argv`]).
393fn command_env(process: &mut Command, base: Option<&[(String, String)]>, env: &[(String, String)], isolated: bool) {
394    if let (Some(base), false) = (base, isolated) {
395        process.env_clear().envs(base.iter().map(|(key, value)| (key, value)));
396    }
397    process.envs(env.iter().map(|(key, value)| (key, value)));
398}
399
400/// Runs `command` with `shell` in its own process group, stdout and stderr on one pipe. `on_text` gets each decoded,
401/// cleaned chunk and the raw byte count; `stop` is polled and kills the process group when it turns true.
402/// Returns the exit code (None when killed by a signal).
403fn run_shell(
404    shell: &shell::ShellConfig,
405    command: &str,
406    CommandEnv { cwd, base, env }: CommandEnv<'_>,
407    launcher: Option<&sandbox::Launcher>,
408    mut on_text: impl FnMut(&str, usize),
409    stop: impl Fn() -> bool,
410) -> Result<Option<i32>, String> {
411    let argv = shell.argv(command);
412    let mut process = match launcher {
413        Some(launcher) => launcher.command(&sandboxed_argv(argv, env), cwd)?,
414        None => {
415            let mut process = Command::new(&argv[0]);
416            process.args(&argv[1..]).current_dir(cwd);
417            process
418        }
419    };
420    let isolated = launcher.is_some_and(|launcher| launcher.mechanism().is_some());
421    command_env(&mut process, base, env, isolated);
422    // A child server's verdict token authenticates only its own `rlm.finish`; commands never see it.
423    process.env_remove(PARENT_TOKEN_ENV);
424    let (reader, writer) = std::io::pipe().map_err(|error| error.to_string())?;
425    let stderr = writer.try_clone().map_err(|error| error.to_string())?;
426    let mut child = process
427        .stdin(Stdio::null())
428        .stdout(writer)
429        .stderr(stderr)
430        .process_group(0)
431        .spawn()
432        .map_err(|error| error.to_string())?;
433    // The Command holds the pipe's write ends: drop it so the reader sees EOF when the command exits.
434    drop(process);
435    // The command's own group (`process_group(0)`): its pgid is its pid.
436    let group = i32::try_from(child.id()).unwrap_or(0);
437    let (sender, chunks) = mpsc::channel::<Vec<u8>>();
438    let pump = std::thread::spawn(move || {
439        let mut reader = reader;
440        let mut buffer = vec![0u8; 64 * 1024];
441        while let Ok(read) = reader.read(&mut buffer) {
442            if read == 0 || sender.send(buffer[..read].to_vec()).is_err() {
443                break;
444            }
445        }
446    });
447    let mut decoder = Decoder::default();
448    let mut killed = false;
449    loop {
450        match chunks.recv_timeout(POLL) {
451            Ok(data) => on_text(&clean(&decoder.decode(&data, true)), data.len()),
452            Err(mpsc::RecvTimeoutError::Disconnected) => break,
453            Err(mpsc::RecvTimeoutError::Timeout) => {}
454        }
455        if !killed && stop() {
456            killed = true;
457            kernel::kill_command_group(group);
458        }
459    }
460    let _ = pump.join();
461    on_text(&clean(&decoder.decode(&[], false)), 0);
462    let status = child.wait().map_err(|error| error.to_string())?;
463    Ok(status.code())
464}
465
466impl Host {
467    /// Each file-hook guard's statistics (`fileHooks.stats()`).
468    pub fn guard_stats(&self) -> Vec<Value> {
469        self.guard.guard_stats()
470    }
471
472    /// A new root turn began: per-turn guard times start from zero (`fileHooks.beginTurn()`).
473    pub fn begin_turn(&self) {
474        self.guard.begin_turn();
475    }
476
477    /// A cell on `lane` is starting: the after-cell file hooks take their baseline picture if they have none.
478    pub fn cell_started(&self, lane: &str) {
479        self.shared.writes.begin(lane);
480        self.guard.cell_started(lane);
481    }
482
483    /// Record what each cell writes through checked writes, for [`Host::take_cell_writes`] (off by default).
484    pub fn recording_cell_writes(self, on: bool) -> Self {
485        self.shared.writes.set_enabled(on);
486        self
487    }
488
489    /// Where commands run and relative paths resolve.
490    pub fn cwd(&self) -> &Path {
491        &self.cwd
492    }
493
494    /// The files `lane`'s cell wrote through checked writes, with their texts (and forget them).
495    pub fn take_cell_writes(&self, lane: &str) -> Vec<cell_writes::CellWrite> {
496        self.shared.writes.take(lane)
497    }
498
499    /// A cell on `lane` ended: the after-cell file hooks check what it changed, in the background.
500    pub fn cell_ended(&self, lane: &str) {
501        self.guard.cell_ended(lane);
502    }
503
504    /// Findings of after-cell checks not yet shown to `lane` (and forget them), for its next cell result.
505    pub fn take_pending(&self, lane: &str) -> Option<String> {
506        self.guard.take_pending(lane)
507    }
508
509    /// `cwd` is where commands run; job spill files go to `jobs_dir/<id>.log`, blocking-bash spill files to
510    /// `spill_dir/pi-bash-<hex>.log` (TS: the OS temp dir).
511    pub fn new(
512        cwd: impl Into<PathBuf>,
513        jobs_dir: impl Into<PathBuf>,
514        spill_dir: impl Into<PathBuf>,
515        guard: Box<dyn Guard>,
516    ) -> Self {
517        let jobs_dir = jobs_dir.into();
518        let _ = std::fs::DirBuilder::new().recursive(true).mode(0o700).create(&jobs_dir);
519        let cwd = cwd.into();
520        Host {
521            shared: Arc::new(Shared {
522                jobs: Mutex::new(Vec::new()),
523                journal: Mutex::new(None),
524                released: AtomicBool::new(false),
525                ends: Mutex::new(None),
526                roots: Mutex::new(None),
527                writes: cell_writes::CellWrites::default(),
528            }),
529            cwd,
530            jobs_dir,
531            spill_dir: spill_dir.into(),
532            guard,
533            launcher: None,
534            shell_path: None,
535            env: Vec::new(),
536            base_env: None,
537            owns_jobs: true,
538        }
539    }
540
541    /// Every command (blocking bash, shell jobs, the sandbox's allowlisted variables) starts from `env` instead of
542    /// this process's environment, and the shell and the sandbox launcher are found on its PATH: a session's
543    /// commands see the session's environment, never the server's. Call before [`Host::sandboxed`].
544    pub fn with_base_env(mut self, env: Vec<(String, String)>) -> Self {
545        self.base_env = Some(Arc::new(env));
546        self
547    }
548
549    /// Every command (blocking bash and shell jobs) runs with `env` over the inherited environment, as
550    /// `withShellEnv(operations, laneWorkspace(lane).env)` runs a worktree subagent's.
551    pub fn with_env(mut self, env: Vec<(String, String)>) -> Self {
552        self.env = env;
553        self
554    }
555
556    /// This host's shell jobs join `session`'s one table (TypeScript's single `ShellJobs` for every lane): the root
557    /// lane lists them, the root turn's abort cancels them, their ends are announced and journaled by the session's
558    /// hooks. The jobs keep running after this host is dropped (they are the session's).
559    pub fn sharing_jobs(mut self, session: &Host) -> Self {
560        self.shared = session.shared.clone();
561        self.owns_jobs = false;
562        self
563    }
564
565    /// Commands run under `shell_path` (the `shellPath` setting), resolved for each command as TypeScript's
566    /// `createLocalBashOperations({ shellPath })` does: a path that does not exist fails the command.
567    pub fn with_shell_path(mut self, shell_path: Option<String>) -> Self {
568        self.shell_path = shell_path;
569        self
570    }
571
572    fn shell(&self) -> Result<shell::ShellConfig, String> {
573        self.shell_named(self.shell_path.as_deref())
574    }
575
576    /// The shell `custom` names, else the default one, with `bash` looked up on the base environment's PATH.
577    fn shell_named(&self, custom: Option<&str>) -> Result<shell::ShellConfig, String> {
578        match &self.base_env {
579            Some(env) => shell::shell_config_on_path(custom, path_of(env).as_deref()),
580            None => shell::system_shell_config(custom),
581        }
582    }
583
584    /// Makes shell jobs durable: restores the jobs journal from `journal` and journals every start and end from now
585    /// on. A job the journal lists as running is settled from its process: followed again while its keeper runs,
586    /// completed with the exit status the keeper wrote, or `lost` when the process ended without one. Records
587    /// without a process (TypeScript journals) read as `interrupted`, as there. Call once, before the first request.
588    pub fn with_journal(self, journal: Arc<dyn DocumentStore>) -> Result<Host, String> {
589        let now = jsrt::date::now_ms() as u64;
590        let restored = journal::restore_durable(journal.read()?.as_ref(), now)?;
591        let mut adopted = Vec::new();
592        {
593            let mut jobs = sync::lock(&self.shared.jobs);
594            for saved in restored.records {
595                let job = Arc::new(Job::new(
596                    saved.id.clone(),
597                    (saved.lane.clone(), saved.root_id.clone()),
598                    (saved.command.clone(), saved.cwd.clone()),
599                    saved.timeout,
600                    // The record names its own spill file; the keeper's files sit next to it.
601                    Path::new(&saved.output_path).parent().unwrap_or(&self.jobs_dir),
602                    Record {
603                        status: saved.status,
604                        exit_code: saved.exit_code.and_then(|code| i32::try_from(code).ok()),
605                        started_at: saved.started_at,
606                        ended_at: saved.ended_at,
607                        output_bytes: saved.output_bytes as usize,
608                        retained_bytes: saved.retained_bytes as usize,
609                        tail: saved.tail.clone(),
610                        error: saved.error.clone(),
611                    },
612                ));
613                if saved.status == Status::Running {
614                    let pid = saved.pid.and_then(|pid| i32::try_from(pid).ok());
615                    *sync::lock(&job.process) = pid.map(|pid| (pid, saved.pid_start));
616                    adopted.push(job.clone());
617                } else {
618                    job.finished();
619                }
620                jobs.push(job);
621            }
622        }
623        *sync::lock(&self.shared.journal) = Some(journal);
624        for job in adopted {
625            if keeper_alive(*sync::lock(&job.process)) {
626                let (runner, shared) = (job.clone(), self.shared.clone());
627                std::thread::spawn(move || follow(&runner, None, &shared));
628            } else {
629                // Ended while no worker followed it: settle it now, so the first reply is already its real state.
630                follow(&job, None, &self.shared);
631            }
632        }
633        self.shared.persist();
634        Ok(self)
635    }
636
637    /// Hands the running jobs to the next owner of the journal: this host stops following and journaling them, and
638    /// dropping it no longer stops them (a worker handing over to its successor). Without a journal it changes
639    /// nothing but the drop.
640    ///
641    /// A stop already asked for is carried out first, so the successor does not keep following a job the person
642    /// cancelled. The release is decided under the jobs lock, as each job's end and each stop request are: a job
643    /// either ends here (and the journal says so) or is the successor's, and a later stop here returns at once.
644    pub fn release_jobs(&self) {
645        loop {
646            let jobs = sync::lock(&self.shared.jobs);
647            let stopping: Vec<Arc<Job>> = jobs
648                .iter()
649                .filter(|job| *sync::lock(&job.cancel.0) && job.following())
650                .cloned()
651                .collect();
652            if stopping.is_empty() {
653                self.shared.persist_locked(&jobs);
654                self.shared.released.store(true, Ordering::SeqCst);
655                break;
656            }
657            drop(jobs);
658            for job in stopping {
659                job.wait_settled();
660            }
661        }
662        for job in sync::lock(&self.shared.jobs).iter() {
663            job.cancel.1.notify_all();
664        }
665    }
666
667    /// Stops every running job and waits until each has ended (worker shutdown). Dropping the host does the same
668    /// unless [`Host::release_jobs`] was called. Nothing is reported of them.
669    pub fn stop_jobs(&self) {
670        let jobs = sync::lock(&self.shared.jobs).clone();
671        for job in jobs {
672            cancel(&job, StopReason::Closed, &self.shared);
673        }
674    }
675
676    /// Who hears of every job's end (`onEnd`): the completion events of jobs nobody waited on.
677    pub fn on_job_end(&self, ends: JobEnds) {
678        *sync::lock(&self.shared.ends) = Some(ends);
679    }
680
681    /// Where a new job's usage root comes from (`host.rootOf(caller)`).
682    pub fn set_roots(&self, roots: RootOf) {
683        *sync::lock(&self.shared.roots) = Some(roots);
684    }
685
686    /// `cancelRoot`: Esc aborted a root turn: stop every job started under it (their ends are not announced).
687    pub fn cancel_root(&self, root: &str) {
688        let jobs: Vec<Arc<Job>> = sync::lock(&self.shared.jobs)
689            .iter()
690            .filter(|job| job.root_id.as_deref() == Some(root))
691            .cloned()
692            .collect();
693        for job in jobs {
694            cancel(&job, StopReason::RootAborted, &self.shared);
695        }
696    }
697
698    /// How many callers are waiting on job `id` now (a waiter that sees the job end gets its result inline, so
699    /// the end is not announced).
700    pub fn job_waiters(&self, id: &str) -> usize {
701        sync::lock(&self.shared.jobs)
702            .iter()
703            .find(|job| job.id == id)
704            .map_or(0, |job| job.waiters.load(Ordering::SeqCst))
705    }
706
707    /// Every job's record, newest first (`ShellJobs.list()`), as `lane` may see them: the root lane every job, another
708    /// lane its own.
709    pub fn job_records(&self, lane: &str) -> Vec<JobRecord> {
710        sync::lock(&self.shared.jobs)
711            .iter()
712            .rev()
713            .filter(|job| visible(job, lane))
714            .map(|job| job_record(job, true))
715            .collect()
716    }
717
718    pub fn running_jobs(&self) -> usize {
719        sync::lock(&self.shared.jobs)
720            .iter()
721            .filter(|job| sync::lock(&job.record).status == Status::Running)
722            .count()
723    }
724
725    /// Running jobs of `lane` whose root `excluded` does not exclude (`running`).
726    pub fn running(&self, lane: &str, excluded: &dyn Fn(Option<&str>) -> bool) -> usize {
727        sync::lock(&self.shared.jobs)
728            .iter()
729            .filter(|job| job.lane == lane && !excluded(job.root_id.as_deref()))
730            .filter(|job| sync::lock(&job.record).status == Status::Running)
731            .count()
732    }
733
734    /// A request of `lane`'s kernel: its jobs belong to it (and to its root turn), and a lane other than the root's
735    /// sees only its own.
736    pub fn handle_lane(
737        &self,
738        lane: &str,
739        kind: &str,
740        payload: Map<String, Value>,
741        signal: &Signal,
742    ) -> Result<Value, String> {
743        if kind == "files.before_write" {
744            // session-worker.ts `proposedWrites`.
745            let writes = match payload.get("writes") {
746                Some(Value::Array(writes)) if (1..=64).contains(&writes.len()) => writes,
747                _ => return Err("files.before_write needs 1 to 64 writes".into()),
748            };
749            let mut proposed = Vec::new();
750            for write in writes {
751                match (write.get("path"), write.get("content")) {
752                    (Some(Value::String(path)), Some(Value::String(content))) if !path.is_empty() => {
753                        proposed.push((path.as_str(), content.as_str()));
754                    }
755                    _ => return Err("files.before_write writes need a path and text content".into()),
756                }
757            }
758            let results: Vec<Value> = proposed
759                .into_iter()
760                .map(|(path, content)| {
761                    let result = self.guard.before_write(lane, path, content);
762                    if result["blocked"] != true {
763                        let path = file_hooks::resolve_lexical(&self.cwd, path);
764                        self.shared.writes.record(lane, &path, content);
765                    }
766                    result
767                })
768                .collect();
769            return Ok(json!({ "results": results }));
770        }
771        if matches!(kind, "bash" | "shell.run")
772            && let Some(Value::String(command)) = payload.get("command")
773            && let Some(reason) = self.guard.before_shell(lane, command)
774        {
775            return Err(format!("bash command was not run: {reason}"));
776        }
777        let requested = payload.get("command").and_then(Value::as_str).map(str::to_string);
778        let mut result = self.dispatch_shell(lane, kind, payload, signal)?;
779        // The model's results: a non-error exit 1 and a sandbox refusal are noted (bash_notes).
780        if matches!(
781            kind,
782            "bash" | "shell.run" | "shell.result" | "shell.get" | "shell.cancel"
783        ) {
784            let command = requested.unwrap_or_else(|| result["command"].as_str().unwrap_or_default().to_string());
785            let policy = self
786                .launcher
787                .as_deref()
788                .filter(|launcher| launcher.mechanism().is_some())
789                .map(|launcher| &launcher.policy);
790            bash_notes::annotate(&mut result, &command, policy);
791        }
792        Ok(result)
793    }
794
795    fn dispatch_shell(
796        &self,
797        lane: &str,
798        kind: &str,
799        payload: Map<String, Value>,
800        signal: &Signal,
801    ) -> Result<Value, String> {
802        match kind {
803            "bash" if payload.contains_key("yield_after") => self.shell_bash(lane, &payload, signal),
804            "bash" => self.blocking_bash(&payload, signal, &mut |_| {}),
805            "shell.run" => {
806                kernel::ops::fields(kind, &payload)?;
807                let command = command_of(&payload)?;
808                let timeout = timeout_of(payload.get("timeout"))?;
809                let yield_after = wait_seconds(payload.get("yield_after"), "yield_after")?.unwrap_or(0.0);
810                let job = self.start(lane, command, timeout)?;
811                wait(&job, Some(yield_after), signal);
812                Ok(snapshot(&job))
813            }
814            "shell.result" => {
815                kernel::ops::fields(kind, &payload)?;
816                let job = self.find(lane, payload.get("id"))?;
817                // TS validates `wait` only while the job runs.
818                if sync::lock(&job.record).status == Status::Running {
819                    wait(&job, wait_seconds(payload.get("wait"), "wait")?, signal);
820                }
821                Ok(snapshot(&job))
822            }
823            "shell.get" => {
824                kernel::ops::fields(kind, &payload)?;
825                let job = self.find(lane, payload.get("id"))?;
826                Ok(snapshot(&job))
827            }
828            "shell.cancel" => {
829                kernel::ops::fields(kind, &payload)?;
830                let job = self.find(lane, payload.get("id"))?;
831                cancel(&job, StopReason::Cancelled, &self.shared);
832                Ok(snapshot(&job))
833            }
834            "shell.read" => {
835                kernel::ops::fields(kind, &payload)?;
836                let job = self.find(lane, payload.get("id"))?;
837                read_output(&job, payload.get("cursor"), payload.get("max_bytes"))
838            }
839            "shell.list" => {
840                kernel::ops::fields(kind, &payload)?;
841                let jobs = sync::lock(&self.shared.jobs).clone();
842                Ok(Value::Array(
843                    jobs.iter()
844                        .rev()
845                        .filter(|job| visible(job, lane))
846                        .map(|job| {
847                            let mut snapshot = snapshot(job);
848                            snapshot["text"] = Value::Null;
849                            snapshot
850                        })
851                        .collect(),
852                ))
853            }
854            other if other.starts_with("shell.") => Err(kernel::ops::not_wired(other)),
855            other => Err(not_available(other)),
856        }
857    }
858
859    /// A user's own command (RPC `bash`, the TUI's `!`; AgentSession `executeBash`): the shell guard, then the
860    /// blocking bash, each cleaned output chunk passed to `on_chunk` as it arrives (bash-executor.ts `onChunk`, which
861    /// AgentSession streams as `bash_execution_update`).
862    pub fn user_bash(&self, command: &str, signal: &Signal, on_chunk: &mut dyn FnMut(&str)) -> Result<Value, String> {
863        if let Some(reason) = self.guard.before_shell(kernel::ROOT_LANE, command) {
864            return Err(format!("bash command was not run: {reason}"));
865        }
866        let mut payload = Map::new();
867        payload.insert("command".into(), command.into());
868        self.blocking_bash(&payload, signal, on_chunk)
869    }
870
871    /// host-bash.ts runHostBash over bash-executor.ts: wait for the command, keep the tail, spill the whole output.
872    fn blocking_bash(
873        &self,
874        payload: &Map<String, Value>,
875        signal: &Signal,
876        on_chunk: &mut dyn FnMut(&str),
877    ) -> Result<Value, String> {
878        let command = command_of(payload)?;
879        let timeout = timeout_of(payload.get("timeout"))?;
880        let deadline = timeout.map(|seconds| Instant::now() + Duration::from_secs_f64(seconds));
881        let timed_out = std::cell::Cell::new(false);
882        let mut chunks: std::collections::VecDeque<String> = Default::default();
883        let mut kept_units = 0usize;
884        let mut total_bytes = 0usize;
885        let mut spill: Option<(PathBuf, File)> = None;
886        let open_spill =
887            |spill: &mut Option<(PathBuf, File)>, chunks: &std::collections::VecDeque<String>| -> Result<(), String> {
888                if spill.is_none() {
889                    let path = self
890                        .spill_dir
891                        .join(format!("pi-bash-{}.log", jsrt::random::random_hex(8)));
892                    let mut file = File::create(&path).map_err(|error| error.to_string())?;
893                    for chunk in chunks {
894                        let _ = file.write_all(chunk.as_bytes());
895                    }
896                    *spill = Some((path, file));
897                }
898                Ok(())
899            };
900        let shell = self.shell()?;
901        let mut failure = None;
902        let exit = run_shell(
903            &shell,
904            &command,
905            CommandEnv {
906                cwd: &self.cwd,
907                base: self.base_env.as_deref().map(Vec::as_slice),
908                env: &self.env,
909            },
910            self.launcher.as_deref(),
911            |text, raw| {
912                // The decoder's flush at the end (no bytes read) is not a chunk of the command's output.
913                if raw > 0 || !text.is_empty() {
914                    on_chunk(text);
915                }
916                total_bytes += raw;
917                if total_bytes > DEFAULT_MAX_BYTES
918                    && let Err(error) = open_spill(&mut spill, &chunks)
919                {
920                    failure = Some(error);
921                }
922                if let Some((_, file)) = spill.as_mut() {
923                    let _ = file.write_all(text.as_bytes());
924                }
925                chunks.push_back(text.to_owned());
926                kept_units += text.encode_utf16().count();
927                while kept_units > DEFAULT_MAX_BYTES * 2 && chunks.len() > 1 {
928                    if let Some(removed) = chunks.pop_front() {
929                        kept_units -= removed.encode_utf16().count();
930                    }
931                }
932            },
933            || {
934                if deadline.is_some_and(|deadline| Instant::now() >= deadline) {
935                    timed_out.set(true);
936                }
937                timed_out.get() || signal.is_aborted()
938            },
939        )?;
940        if let Some(error) = failure {
941            return Err(error);
942        }
943        let full: String = chunks.iter().map(String::as_str).collect();
944        let truncated = truncate_tail(&full);
945        if truncated.is_some() {
946            open_spill(&mut spill, &chunks)?;
947        }
948        let stopped = timed_out.get() || signal.is_aborted();
949        Ok(json!({
950            "output": truncated.clone().unwrap_or(full),
951            "exit_code": if stopped { None } else { exit },
952            "cancelled": stopped && !timed_out.get(),
953            "timed_out": timed_out.get(),
954            "truncated": truncated.is_some(),
955            "full_output_path": spill.map(|(path, _)| path.to_string_lossy().into_owned()),
956        }))
957    }
958
959    /// Plain `await bash(cmd)` with a yield_after: wait like the blocking bash, then leave a slow command running.
960    fn shell_bash(&self, lane: &str, payload: &Map<String, Value>, signal: &Signal) -> Result<Value, String> {
961        kernel::ops::fields("bash", payload)?;
962        let command = command_of(payload)?;
963        let timeout = timeout_of(payload.get("timeout"))?;
964        let yield_after = wait_seconds(payload.get("yield_after"), "yield_after")?;
965        let job = self.start(lane, command, timeout)?;
966        wait(&job, yield_after, signal);
967        if sync::lock(&job.record).status == Status::Running && signal.is_aborted() {
968            job.waiters.fetch_add(1, Ordering::SeqCst);
969            cancel(&job, StopReason::Cancelled, &self.shared);
970            job.waiters.fetch_sub(1, Ordering::SeqCst);
971        }
972        bash_result(&job)
973    }
974
975    fn find(&self, lane: &str, id: Option<&Value>) -> Result<Arc<Job>, String> {
976        let Some(Value::String(id)) = id.filter(|id| id.as_str().is_some_and(|id| !id.is_empty())) else {
977            return Err("job id must be a nonempty string".into());
978        };
979        sync::lock(&self.shared.jobs)
980            .iter()
981            .find(|job| &job.id == id && visible(job, lane))
982            .cloned()
983            .ok_or_else(|| format!("Unknown shell job {id}"))
984    }
985
986    fn start(&self, lane: &str, command: String, timeout: Option<f64>) -> Result<Arc<Job>, String> {
987        let root = sync::lock(&self.shared.roots).clone().and_then(|roots| roots(lane));
988        let mut jobs = sync::lock(&self.shared.jobs);
989        let running = |job: &Arc<Job>| sync::lock(&job.record).status == Status::Running;
990        if jobs.iter().filter(|job| running(job)).count() >= JOB_MAX_ACTIVE {
991            return Err(format!("At most {JOB_MAX_ACTIVE} shell jobs can run at once"));
992        }
993        let id = format!("job-{}", jsrt::random::random_hex(4));
994        let job = Arc::new(Job::new(
995            id,
996            (lane.to_string(), root),
997            (command, self.cwd.to_string_lossy().into_owned()),
998            timeout,
999            &self.jobs_dir,
1000            Record {
1001                status: Status::Running,
1002                exit_code: None,
1003                started_at: jsrt::date::now_ms() as u64,
1004                ended_at: None,
1005                output_bytes: 0,
1006                retained_bytes: 0,
1007                tail: String::new(),
1008                error: None,
1009            },
1010        ));
1011        jobs.push(job.clone());
1012        // Bounded retention: drop the oldest finished jobs and their files.
1013        let excess = jobs.len().saturating_sub(JOB_MAX_RETAINED);
1014        let pruned: Vec<String> = jobs
1015            .iter()
1016            .filter(|job| !running(job))
1017            .take(excess)
1018            .map(|job| job.id.clone())
1019            .collect();
1020        jobs.retain(|job| {
1021            let keep = !pruned.contains(&job.id);
1022            if !keep {
1023                for path in [&job.output_path, &job.raw_path, &job.exit_path] {
1024                    let _ = std::fs::remove_file(path);
1025                }
1026            }
1027            keep
1028        });
1029        drop(jobs);
1030        // The keeper is spawned on the thread that follows it (its waiter); its sandbox is persistent
1031        // (`Launcher::command_persistent`), so neither that thread nor this worker ending stops it.
1032        let (started, spawned) = mpsc::channel();
1033        let (runner, shared, cwd, launcher, shell) = (
1034            job.clone(),
1035            self.shared.clone(),
1036            (self.cwd.clone(), self.base_env.clone(), self.env.clone()),
1037            self.launcher.clone(),
1038            // The keeper is POSIX shell run by the default shell (/bin/bash, else bash on the session's PATH).
1039            self.shell()
1040                .and_then(|shell| Ok((shell, self.shell_named(None)?.shell))),
1041        );
1042        std::thread::spawn(move || {
1043            let env = CommandEnv {
1044                cwd: &cwd.0,
1045                base: cwd.1.as_deref().map(Vec::as_slice),
1046                env: &cwd.2,
1047            };
1048            match shell.and_then(|(shell, keeper)| spawn_keeper(&runner, (&shell, &keeper), env, launcher.as_deref())) {
1049                Ok(child) => {
1050                    let pid = child.id() as i32;
1051                    *sync::lock(&runner.process) = Some((pid, process_start(pid)));
1052                    let _ = started.send(());
1053                    follow(&runner, Some(child), &shared);
1054                }
1055                Err(error) => {
1056                    let mut record = sync::lock(&runner.record);
1057                    record.status = Status::Failed;
1058                    record.ended_at = Some(jsrt::date::now_ms() as u64);
1059                    record.error = Some(error);
1060                    drop(record);
1061                    let _ = started.send(());
1062                    runner.finished();
1063                }
1064            }
1065        });
1066        let _ = spawned.recv();
1067        self.shared.persist();
1068        Ok(job)
1069    }
1070}
1071
1072/// The keeper: runs the command with the session's shell (`$1 $2 "$0"`: the shell and its `-c`; stdout and stderr
1073/// together into the raw file, which is the keeper's stdout), then writes the command's exit status to its own
1074/// stderr, the exit file. Files, not pipes, so the command keeps running and its end is recorded when the worker
1075/// that started it is gone. The keeper itself is POSIX shell run by bash, whatever shell the command uses.
1076const KEEPER: &str = r#""$1" "$2" "$0" 2>&1 </dev/null; printf '%s\n' "$?" >&2"#;
1077
1078fn spawn_keeper(
1079    job: &Job,
1080    (shell, keeper): (&shell::ShellConfig, &str),
1081    CommandEnv { cwd, base, env }: CommandEnv<'_>,
1082    launcher: Option<&sandbox::Launcher>,
1083) -> Result<std::process::Child, String> {
1084    let create = |path: &Path| {
1085        std::fs::OpenOptions::new()
1086            .write(true)
1087            .create(true)
1088            .truncate(true)
1089            .mode(0o600)
1090            .open(path)
1091            .map_err(|error| format!("{}: {error}", path.display()))
1092    };
1093    let raw = create(&job.raw_path)?;
1094    let exit = create(&job.exit_path)?;
1095    let mut argv = vec![keeper.to_string(), "-c".into(), KEEPER.into(), job.command.clone()];
1096    argv.push(shell.shell.clone());
1097    argv.extend(shell.args.iter().cloned());
1098    let mut process = match launcher {
1099        // A job outlives the worker that started it (a successor adopts it): its sandbox is not tied to us.
1100        Some(launcher) => launcher.command_persistent(&sandboxed_argv(argv, env), cwd)?,
1101        None => {
1102            let mut process = Command::new(&argv[0]);
1103            process.args(&argv[1..]).current_dir(cwd);
1104            process
1105        }
1106    };
1107    let isolated = launcher.is_some_and(|launcher| launcher.mechanism().is_some());
1108    command_env(&mut process, base, env, isolated);
1109    process
1110        .env_remove(PARENT_TOKEN_ENV)
1111        .stdin(Stdio::null())
1112        .stdout(raw)
1113        .stderr(exit)
1114        .process_group(0)
1115        .spawn()
1116        .map_err(|error| error.to_string())
1117}
1118
1119/// Turns raw output into the cleaned, bounded spill file and the record's counts and tail, as the TypeScript job
1120/// does with its stream.
1121struct Digest {
1122    decoder: Decoder,
1123    log: Option<File>,
1124}
1125
1126impl Digest {
1127    /// Starts the spill file over: a job followed again after a restart is digested from its first raw byte.
1128    fn new(job: &Job) -> Digest {
1129        let log = std::fs::OpenOptions::new()
1130            .write(true)
1131            .create(true)
1132            .truncate(true)
1133            .mode(0o600)
1134            .open(&job.output_path)
1135            .ok();
1136        let mut record = sync::lock(&job.record);
1137        record.output_bytes = 0;
1138        record.retained_bytes = 0;
1139        record.tail.clear();
1140        Digest {
1141            decoder: Decoder::default(),
1142            log,
1143        }
1144    }
1145
1146    fn text(&mut self, job: &Job, text: &str) {
1147        if text.is_empty() {
1148            return;
1149        }
1150        let mut record = sync::lock(&job.record);
1151        record.output_bytes += text.len();
1152        let room = JOB_RETAINED_BYTES.saturating_sub(record.retained_bytes);
1153        if room > 0 {
1154            let kept = &text.as_bytes()[..room.min(text.len())];
1155            if let Some(log) = self.log.as_mut() {
1156                let _ = log.write_all(kept);
1157            }
1158            record.retained_bytes += kept.len();
1159        }
1160        let tail = format!("{}{text}", record.tail);
1161        record.tail = tail_bytes(&tail, JOB_TEXT_BYTES / 2).to_owned();
1162    }
1163
1164    /// Reads every raw byte written so far.
1165    fn pump(&mut self, job: &Job, raw: &mut Option<File>) {
1166        let Some(file) = raw.as_mut() else {
1167            return;
1168        };
1169        let mut buffer = vec![0u8; 64 * 1024];
1170        while let Ok(read) = file.read(&mut buffer) {
1171            if read == 0 {
1172                break;
1173            }
1174            let text = clean(&self.decoder.decode(&buffer[..read], true));
1175            self.text(job, &text);
1176        }
1177    }
1178
1179    fn finish(&mut self, job: &Job) {
1180        let text = clean(&self.decoder.decode(&[], false));
1181        self.text(job, &text);
1182    }
1183}
1184
1185/// Follows a job until it ends: its own child (`child`), or a keeper started by an earlier worker (`None`), whose
1186/// end is its exit file or, without one, its process going away (`lost`). Stops the process group on cancel or
1187/// timeout. Returns without settling the job when the host released its jobs.
1188fn follow(job: &Arc<Job>, child: Option<std::process::Child>, shared: &Shared) {
1189    let own = child.is_some();
1190    if let Some(mut child) = child {
1191        // Waits for the keeper on a thread of its own so its exit wakes the follower at once.
1192        let waiter = job.clone();
1193        std::thread::spawn(move || {
1194            let code = child.wait().ok().and_then(|status| status.code());
1195            let guard = sync::lock(&waiter.cancel.0);
1196            *sync::lock(&waiter.exited) = Some(code);
1197            drop(guard);
1198            waiter.cancel.1.notify_all();
1199        });
1200    }
1201    let mut digest = Digest::new(job);
1202    let mut raw = File::open(&job.raw_path).ok();
1203    let started_at = sync::lock(&job.record).started_at;
1204    let deadline = job
1205        .timeout
1206        .map(|seconds| started_at.saturating_add((seconds * 1000.0) as u64));
1207    let process = *sync::lock(&job.process);
1208    let mut stopped: Option<Status> = None;
1209    let ended = loop {
1210        let ended = match own {
1211            true => *sync::lock(&job.exited),
1212            false => {
1213                let code = exit_status(&job.exit_path);
1214                (code.is_some() || !keeper_alive(process)).then_some(code)
1215            }
1216        };
1217        digest.pump(job, &mut raw);
1218        if let Some(ended) = ended {
1219            break ended;
1220        }
1221        if shared.released.load(Ordering::SeqCst) {
1222            job.stop_following(Tracking::Released);
1223            return;
1224        }
1225        if stopped.is_none() {
1226            if deadline.is_some_and(|deadline| jsrt::date::now_ms() as u64 >= deadline) {
1227                stopped = Some(Status::TimedOut);
1228            } else if *sync::lock(&job.cancel.0) {
1229                stopped = Some(Status::Cancelled);
1230            }
1231            // The keeper's own group (`process_group(0)` in `spawn_keeper`): its pgid is its pid.
1232            if stopped.is_some()
1233                && let Some((pid, _)) = process
1234            {
1235                kernel::kill_command_group(pid);
1236            }
1237        }
1238        let cancel = sync::lock(&job.cancel.0);
1239        if sync::lock(&job.exited).is_none() {
1240            drop(
1241                job.cancel
1242                    .1
1243                    .wait_timeout(cancel, POLL)
1244                    .unwrap_or_else(PoisonError::into_inner),
1245            );
1246        }
1247    };
1248    digest.finish(job);
1249    // The keeper writes the command's status just before it exits.
1250    let code = exit_status(&job.exit_path);
1251    let jobs = sync::lock(&shared.jobs);
1252    if shared.released.load(Ordering::SeqCst) {
1253        // Released while it ended: the next owner settles it from its exit file.
1254        drop(jobs);
1255        job.stop_following(Tracking::Released);
1256        return;
1257    }
1258    {
1259        let mut record = sync::lock(&job.record);
1260        record.ended_at = Some(match own {
1261            true => jsrt::date::now_ms() as u64,
1262            // Ended while nobody followed it: the exit file's time, when there is one.
1263            false => std::fs::metadata(&job.exit_path)
1264                .and_then(|meta| meta.modified())
1265                .ok()
1266                .filter(|_| code.is_some())
1267                .and_then(|time| time.duration_since(UNIX_EPOCH).ok())
1268                .map_or_else(|| jsrt::date::now_ms() as u64, |d| d.as_millis() as u64),
1269        });
1270        match stopped {
1271            Some(status) => {
1272                record.status = status;
1273                record.exit_code = None;
1274                if status == Status::TimedOut {
1275                    record.error = Some(format!(
1276                        "timed out after {}s; process tree killed",
1277                        jsrt::number::to_string(job.timeout.unwrap_or(0.0))
1278                    ));
1279                }
1280            }
1281            None => match (code, own) {
1282                (Some(code), _) => {
1283                    record.status = Status::Completed;
1284                    record.exit_code = Some(code);
1285                }
1286                // The keeper itself was killed: as a command killed by a signal, no exit code.
1287                (None, true) => {
1288                    record.status = Status::Completed;
1289                    record.exit_code = ended;
1290                }
1291                (None, false) => {
1292                    record.status = Status::Lost;
1293                    record.exit_code = None;
1294                    record.error =
1295                        Some("its process ended while no Ultron worker was running; the exit status is unknown".into());
1296                }
1297            },
1298        }
1299    }
1300    shared.persist_locked(&jobs);
1301    drop(jobs);
1302    // `onEnd`, before the job's waiters wake: a caller still waiting gets the result inline, so nothing is announced.
1303    let reason = *sync::lock(&job.reason);
1304    let end = JobEnd {
1305        record: job_record(job, true),
1306        awaited: job.waiters.load(Ordering::SeqCst) > 0,
1307        root_aborted: reason == Some(StopReason::RootAborted),
1308    };
1309    let ends = sync::lock(&shared.ends).clone();
1310    job.finished();
1311    if reason != Some(StopReason::Closed)
1312        && let Some(ends) = ends
1313    {
1314        ends(end);
1315    }
1316}
1317
1318/// Wait for a job at most `seconds` (at least 20 ms; None: until it ends) or until the calling cell ends.
1319fn wait(job: &Arc<Job>, seconds: Option<f64>, signal: &Signal) {
1320    job.waiters.fetch_add(1, Ordering::SeqCst);
1321    let deadline = seconds.map(|seconds| Instant::now() + Duration::from_secs_f64(seconds.max(0.02)));
1322    let woken = job.clone();
1323    let link = signal.link(move |_| {
1324        let _tracking = sync::lock(&woken.tracking.0);
1325        woken.tracking.1.notify_all();
1326    });
1327    let mut done = sync::lock(&job.tracking.0);
1328    while *done == Tracking::Following && !signal.is_aborted() {
1329        done = match deadline {
1330            None => job.tracking.1.wait(done).unwrap_or_else(PoisonError::into_inner),
1331            Some(deadline) => {
1332                let now = Instant::now();
1333                if now >= deadline {
1334                    break;
1335                }
1336                job.tracking
1337                    .1
1338                    .wait_timeout(done, deadline - now)
1339                    .unwrap_or_else(PoisonError::into_inner)
1340                    .0
1341            }
1342        };
1343    }
1344    drop(done);
1345    drop(link);
1346    job.waiters.fetch_sub(1, Ordering::SeqCst);
1347}
1348
1349/// Stops a job this worker follows and waits until it is settled (or released). A job already handed to the next
1350/// owner is not this worker's to stop: it returns at once.
1351fn cancel(job: &Job, reason: StopReason, shared: &Shared) {
1352    {
1353        // Under the jobs lock, as the release is: a stop asked for before it is carried out by this worker.
1354        let _jobs = sync::lock(&shared.jobs);
1355        if shared.released.load(Ordering::SeqCst) || !job.following() {
1356            return;
1357        }
1358        sync::lock(&job.reason).get_or_insert(reason);
1359        *sync::lock(&job.cancel.0) = true;
1360    }
1361    job.cancel.1.notify_all();
1362    job.wait_settled();
1363}
1364
1365fn read_file(path: &Path, cursor: usize, max: usize) -> String {
1366    let mut buffer = Vec::new();
1367    if let Ok(mut file) = File::open(path)
1368        && file.seek(SeekFrom::Start(cursor as u64)).is_ok()
1369    {
1370        let _ = file.take(max as u64).read_to_end(&mut buffer);
1371    }
1372    String::from_utf8_lossy(&buffer).into_owned()
1373}
1374
1375/// Whole output when it fits, else the first and last halves around a marker naming how to read the rest.
1376fn job_text(job: &Job, record: &Record) -> (String, bool) {
1377    let half = JOB_TEXT_BYTES / 2;
1378    if record.output_bytes <= JOB_TEXT_BYTES && record.retained_bytes == record.output_bytes {
1379        return (read_file(&job.output_path, 0, JOB_TEXT_BYTES), false);
1380    }
1381    let head = read_file(&job.output_path, 0, half);
1382    let omitted = record.output_bytes.saturating_sub(half + record.tail.len());
1383    let marker = format!(
1384        "\n[... {omitted} bytes omitted; {} bytes kept in {}: await job.read(cursor={half}, max_bytes=65536) ...]\n",
1385        record.retained_bytes,
1386        job.output_path.display()
1387    );
1388    (format!("{head}{marker}{}", record.tail), true)
1389}
1390
1391fn snapshot(job: &Job) -> Value {
1392    let record = sync::lock(&job.record);
1393    let (text, truncated) = job_text(job, &record);
1394    let elapsed = record
1395        .ended_at
1396        .unwrap_or_else(|| jsrt::date::now_ms() as u64)
1397        .saturating_sub(record.started_at);
1398    json!({
1399        "id": job.id,
1400        "lane": job.lane,
1401        "command": job.command,
1402        "status": record.status.as_str(),
1403        "running": record.status == Status::Running,
1404        "exit_code": record.exit_code,
1405        "ok": record.status == Status::Completed && record.exit_code == Some(0),
1406        "timed_out": record.status == Status::TimedOut,
1407        "cancelled": record.status == Status::Cancelled,
1408        "text": text,
1409        "truncated": truncated,
1410        "output_path": job.output_path.to_string_lossy(),
1411        "output_bytes": record.output_bytes,
1412        "elapsed_seconds": jsrt::json::number((elapsed as f64 / 100.0).round() / 10.0),
1413        "error": record.error,
1414    })
1415}
1416
1417/// The blocking bash's reply shape for a command that ran as a job, plus `running` and the job while it runs.
1418fn bash_result(job: &Job) -> Result<Value, String> {
1419    let (status, exit_code, error, partial) = {
1420        let record = sync::lock(&job.record);
1421        (
1422            record.status,
1423            record.exit_code,
1424            record.error.clone(),
1425            record.retained_bytes < record.output_bytes,
1426        )
1427    };
1428    if status == Status::Failed {
1429        return Err(error.unwrap_or_else(|| "bash command failed".into()));
1430    }
1431    let snapshot = snapshot(job);
1432    let truncated = snapshot["truncated"].as_bool() == Some(true);
1433    let running = status == Status::Running;
1434    Ok(json!({
1435        "output": snapshot["text"],
1436        "exit_code": exit_code,
1437        "cancelled": status == Status::Cancelled,
1438        "timed_out": status == Status::TimedOut,
1439        "truncated": truncated,
1440        "full_output_path": if truncated { snapshot["output_path"].clone() } else { Value::Null },
1441        "partial_file": partial,
1442        "running": running,
1443        "job": if running { snapshot } else { Value::Null },
1444    }))
1445}
1446
1447fn read_output(job: &Job, cursor: Option<&Value>, max_bytes: Option<&Value>) -> Result<Value, String> {
1448    let cursor = safe_integer(cursor, 0).ok_or("cursor must be a non-negative integer")? as usize;
1449    let max = safe_integer(max_bytes, 65_536)
1450        .filter(|max| (1..=1_048_576).contains(max))
1451        .ok_or("max_bytes must be an integer between 1 and 1048576")? as usize;
1452    let record = sync::lock(&job.record);
1453    let start = cursor.min(record.retained_bytes);
1454    let text = read_file(&job.output_path, start, max.min(record.retained_bytes - start));
1455    let next = start + text.len();
1456    Ok(json!({
1457        "text": text,
1458        "next_cursor": next,
1459        "done": record.status != Status::Running && next >= record.retained_bytes,
1460        "truncated": record.retained_bytes < record.output_bytes,
1461    }))
1462}
1463
1464impl HostHandler for Host {
1465    fn handle(&self, kind: &str, payload: Map<String, Value>, signal: &Signal) -> Result<Value, String> {
1466        self.handle_lane(kernel::ROOT_LANE, kind, payload, signal)
1467    }
1468}
1469
1470fn visible(job: &Job, lane: &str) -> bool {
1471    kernel::sees(lane, &job.lane)
1472}
1473
1474impl Drop for Host {
1475    /// Worker shutdown: stop every running job, unless they were released to the next owner.
1476    fn drop(&mut self) {
1477        if self.owns_jobs && !self.shared.released.load(Ordering::SeqCst) {
1478            self.stop_jobs();
1479        }
1480    }
1481}