1pub mod bash_notes;
16pub mod cell;
17pub mod cell_writes;
18pub mod file_hooks;
19pub mod journal;
20pub mod loki_guard;
21pub mod sandboxed;
22pub mod secrets;
23pub mod shell;
24pub mod text;
25pub mod view_image;
26
27use journal::{JobRecord, Status};
28use jsrt::sync;
29use kernel::store::DocumentStore;
30use kernel::{HostHandler, Signal};
31use serde_json::{Map, Value, json};
32use std::fs::File;
33use std::io::{Read, Seek, SeekFrom, Write};
34use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
35use std::os::unix::process::CommandExt;
36use std::path::{Path, PathBuf};
37use std::process::{Command, Stdio};
38use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
39use std::sync::{Arc, Condvar, Mutex, PoisonError, mpsc};
40use std::time::{Duration, Instant, UNIX_EPOCH};
41use text::{DEFAULT_MAX_BYTES, Decoder, clean, tail_bytes, truncate_tail};
42
43pub const JOB_TEXT_BYTES: usize = 16 * 1024;
44pub const JOB_RETAINED_BYTES: usize = 4 * 1024 * 1024;
45pub const JOB_MAX_ACTIVE: usize = 32;
46pub const JOB_MAX_RETAINED: usize = 64;
47const JOB_MAX_WAIT_SECONDS: f64 = 3600.0;
48const MAX_TIMEOUT_SECONDS: f64 = 86_400.0;
49const POLL: Duration = Duration::from_millis(10);
52
53pub trait Guard: Send + Sync {
55 fn before_shell(&self, _lane: &str, _command: &str) -> Option<String> {
57 None
58 }
59 fn before_write(&self, _lane: &str, _path: &str, _content: &str) -> Value {
61 json!({ "blocked": false })
62 }
63 fn guard_stats(&self) -> Vec<Value> {
65 Vec::new()
66 }
67 fn begin_turn(&self) {}
69 fn cell_started(&self, _lane: &str) {}
71 fn cell_ended(&self, _lane: &str) {}
73 fn take_pending(&self, _lane: &str) -> Option<String> {
75 None
76 }
77}
78
79pub struct AllowAll;
80impl Guard for AllowAll {}
81
82pub fn not_available(kind: &str) -> String {
83 format!("Ultron RLM host request is not available in this host yet: {kind}")
84}
85
86pub struct Host {
87 cwd: PathBuf,
88 jobs_dir: PathBuf,
89 spill_dir: PathBuf,
90 guard: Box<dyn Guard>,
91 launcher: Option<Arc<sandbox::Launcher>>,
93 shell_path: Option<String>,
95 env: Vec<(String, String)>,
98 base_env: Option<Arc<Vec<(String, String)>>>,
101 owns_jobs: bool,
103 shared: Arc<Shared>,
104}
105
106#[derive(Clone, Debug, PartialEq)]
109pub struct JobEnd {
110 pub record: JobRecord,
111 pub awaited: bool,
112 pub root_aborted: bool,
113}
114
115impl JobEnd {
116 pub fn event(&self) -> Option<Value> {
118 journal::job_event(&self.record, self.awaited, self.root_aborted)
119 }
120}
121
122pub type JobEnds = Arc<dyn Fn(JobEnd) + Send + Sync>;
124pub type RootOf = Arc<dyn Fn(&str) -> Option<String> + Send + Sync>;
127
128#[derive(Clone, Copy, PartialEq, Eq, Debug)]
129enum StopReason {
130 Cancelled,
132 RootAborted,
134 Closed,
136}
137
138struct Shared {
141 jobs: Mutex<Vec<Arc<Job>>>,
142 journal: Mutex<Option<Arc<dyn DocumentStore>>>,
143 released: AtomicBool,
144 ends: Mutex<Option<JobEnds>>,
145 roots: Mutex<Option<RootOf>>,
146 writes: cell_writes::CellWrites,
148}
149
150impl Shared {
151 fn persist(&self) {
154 self.persist_locked(&sync::lock(&self.jobs));
155 }
156
157 fn persist_locked(&self, jobs: &[Arc<Job>]) {
160 let Some(journal) = sync::lock(&self.journal).clone() else {
161 return;
162 };
163 if self.released.load(Ordering::SeqCst) {
164 return;
165 }
166 let records: Vec<JobRecord> = jobs.iter().map(|job| job_record(job, false)).collect();
167 let _ = journal.write(&journal::persisted(&records));
168 }
169}
170
171struct Record {
172 status: Status,
173 exit_code: Option<i32>,
174 started_at: u64,
175 ended_at: Option<u64>,
176 output_bytes: usize,
177 retained_bytes: usize,
178 tail: String,
179 error: Option<String>,
180}
181
182struct Job {
183 id: String,
184 lane: String,
186 root_id: Option<String>,
187 command: String,
188 cwd: String,
190 timeout: Option<f64>,
191 output_path: PathBuf,
193 raw_path: PathBuf,
195 exit_path: PathBuf,
196 record: Mutex<Record>,
197 process: Mutex<Option<(i32, Option<u64>)>>,
199 exited: Mutex<Option<Option<i32>>>,
202 cancel: (Mutex<bool>, Condvar),
204 reason: Mutex<Option<StopReason>>,
206 tracking: (Mutex<Tracking>, Condvar),
208 waiters: AtomicUsize,
209}
210
211#[derive(Clone, Copy, Debug, PartialEq, Eq)]
213enum Tracking {
214 Following,
216 Released,
218 Ended,
220}
221
222impl Job {
223 fn new(
224 id: String,
225 (lane, root_id): (String, Option<String>),
226 (command, cwd): (String, String),
227 timeout: Option<f64>,
228 jobs_dir: &Path,
229 record: Record,
230 ) -> Job {
231 Job {
232 output_path: jobs_dir.join(format!("{id}.log")),
233 raw_path: jobs_dir.join(format!("{id}.out")),
234 exit_path: jobs_dir.join(format!("{id}.exit")),
235 id,
236 lane,
237 root_id,
238 command,
239 cwd,
240 timeout,
241 record: Mutex::new(record),
242 process: Mutex::new(None),
243 exited: Mutex::new(None),
244 cancel: (Mutex::new(false), Condvar::new()),
245 reason: Mutex::new(None),
246 tracking: (Mutex::new(Tracking::Following), Condvar::new()),
247 waiters: AtomicUsize::new(0),
248 }
249 }
250
251 fn finished(&self) {
252 self.stop_following(Tracking::Ended);
253 }
254
255 fn stop_following(&self, to: Tracking) {
256 *sync::lock(&self.tracking.0) = to;
257 self.tracking.1.notify_all();
258 }
259
260 fn following(&self) -> bool {
261 *sync::lock(&self.tracking.0) == Tracking::Following
262 }
263
264 fn wait_settled(&self) {
266 let mut tracking = sync::lock(&self.tracking.0);
267 while *tracking == Tracking::Following {
268 tracking = self.tracking.1.wait(tracking).unwrap_or_else(PoisonError::into_inner);
269 }
270 }
271}
272
273fn job_record(job: &Job, tail: bool) -> JobRecord {
275 let record = sync::lock(&job.record);
276 let process = *sync::lock(&job.process);
277 JobRecord {
278 id: job.id.clone(),
279 lane: job.lane.clone(),
280 root_id: job.root_id.clone(),
281 command: job.command.clone(),
282 cwd: job.cwd.clone(),
283 timeout: job.timeout,
284 status: record.status,
285 exit_code: record.exit_code.map(i64::from),
286 started_at: record.started_at,
287 ended_at: record.ended_at,
288 output_path: job.output_path.to_string_lossy().into_owned(),
289 output_bytes: record.output_bytes as u64,
290 retained_bytes: record.retained_bytes as u64,
291 tail: if tail { record.tail.clone() } else { String::new() },
292 error: record.error.clone(),
293 pid: process.map(|(pid, _)| i64::from(pid)),
294 pid_start: process.and_then(|(_, start)| start),
295 }
296}
297
298fn process_start(pid: i32) -> Option<u64> {
300 let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?;
301 let fields: Vec<&str> = stat.rsplit_once(')')?.1.split_whitespace().collect();
302 if fields.first() == Some(&"Z") {
303 return None;
304 }
305 fields.get(19)?.parse().ok()
306}
307
308fn keeper_alive(process: Option<(i32, Option<u64>)>) -> bool {
310 match process {
311 Some((pid, start)) => process_start(pid).is_some_and(|now| start.is_none_or(|start| start == now)),
312 None => false,
313 }
314}
315
316fn exit_status(path: &Path) -> Option<i32> {
318 let text = std::fs::read_to_string(path).ok()?;
319 text.split_whitespace().last()?.parse().ok()
320}
321
322fn command_of(payload: &Map<String, Value>) -> Result<String, String> {
323 match payload.get("command") {
324 Some(Value::String(command)) if !command.trim().is_empty() => Ok(command.clone()),
325 _ => Err("bash command must be a non-empty string".into()),
326 }
327}
328
329fn timeout_of(value: Option<&Value>) -> Result<Option<f64>, String> {
331 match value {
332 None | Some(Value::Null) => Ok(None),
333 Some(value) => match value.as_f64() {
334 Some(seconds) if seconds > 0.0 => Ok(Some(seconds.min(MAX_TIMEOUT_SECONDS))),
335 _ => Err("bash timeout must be a positive number of seconds".into()),
336 },
337 }
338}
339
340fn wait_seconds(value: Option<&Value>, name: &str) -> Result<Option<f64>, String> {
341 match value {
342 None | Some(Value::Null) => Ok(None),
343 Some(value) => match value.as_f64() {
344 Some(seconds) if seconds >= 0.0 => Ok(Some(seconds.min(JOB_MAX_WAIT_SECONDS))),
345 _ => Err(format!("{name} must be a non-negative number of seconds")),
346 },
347 }
348}
349
350fn safe_integer(value: Option<&Value>, default: u64) -> Option<u64> {
352 match value {
353 None => Some(default),
354 Some(value) => jsrt::number::safe_integer(value).and_then(|n| u64::try_from(n).ok()),
355 }
356}
357
358pub const PARENT_TOKEN_ENV: &str = "ULTRON_PARENT_TOKEN";
361
362pub fn sandboxed_argv(argv: Vec<String>, env: &[(String, String)]) -> Vec<String> {
366 if env.is_empty() {
367 return argv;
368 }
369 std::iter::once("env".to_string())
370 .chain(env.iter().map(|(key, value)| format!("{key}={value}")))
371 .chain(argv)
372 .collect()
373}
374
375#[derive(Clone, Copy)]
377struct CommandEnv<'a> {
378 cwd: &'a Path,
379 base: Option<&'a [(String, String)]>,
380 env: &'a [(String, String)],
381}
382
383fn path_of(env: &[(String, String)]) -> Option<String> {
384 env.iter()
385 .find(|(key, _)| key == "PATH")
386 .map(|(_, value)| value.clone())
387}
388
389fn command_env(process: &mut Command, base: Option<&[(String, String)]>, env: &[(String, String)], isolated: bool) {
394 if let (Some(base), false) = (base, isolated) {
395 process.env_clear().envs(base.iter().map(|(key, value)| (key, value)));
396 }
397 process.envs(env.iter().map(|(key, value)| (key, value)));
398}
399
400fn run_shell(
404 shell: &shell::ShellConfig,
405 command: &str,
406 CommandEnv { cwd, base, env }: CommandEnv<'_>,
407 launcher: Option<&sandbox::Launcher>,
408 mut on_text: impl FnMut(&str, usize),
409 stop: impl Fn() -> bool,
410) -> Result<Option<i32>, String> {
411 let argv = shell.argv(command);
412 let mut process = match launcher {
413 Some(launcher) => launcher.command(&sandboxed_argv(argv, env), cwd)?,
414 None => {
415 let mut process = Command::new(&argv[0]);
416 process.args(&argv[1..]).current_dir(cwd);
417 process
418 }
419 };
420 let isolated = launcher.is_some_and(|launcher| launcher.mechanism().is_some());
421 command_env(&mut process, base, env, isolated);
422 process.env_remove(PARENT_TOKEN_ENV);
424 let (reader, writer) = std::io::pipe().map_err(|error| error.to_string())?;
425 let stderr = writer.try_clone().map_err(|error| error.to_string())?;
426 let mut child = process
427 .stdin(Stdio::null())
428 .stdout(writer)
429 .stderr(stderr)
430 .process_group(0)
431 .spawn()
432 .map_err(|error| error.to_string())?;
433 drop(process);
435 let group = i32::try_from(child.id()).unwrap_or(0);
437 let (sender, chunks) = mpsc::channel::<Vec<u8>>();
438 let pump = std::thread::spawn(move || {
439 let mut reader = reader;
440 let mut buffer = vec![0u8; 64 * 1024];
441 while let Ok(read) = reader.read(&mut buffer) {
442 if read == 0 || sender.send(buffer[..read].to_vec()).is_err() {
443 break;
444 }
445 }
446 });
447 let mut decoder = Decoder::default();
448 let mut killed = false;
449 loop {
450 match chunks.recv_timeout(POLL) {
451 Ok(data) => on_text(&clean(&decoder.decode(&data, true)), data.len()),
452 Err(mpsc::RecvTimeoutError::Disconnected) => break,
453 Err(mpsc::RecvTimeoutError::Timeout) => {}
454 }
455 if !killed && stop() {
456 killed = true;
457 kernel::kill_command_group(group);
458 }
459 }
460 let _ = pump.join();
461 on_text(&clean(&decoder.decode(&[], false)), 0);
462 let status = child.wait().map_err(|error| error.to_string())?;
463 Ok(status.code())
464}
465
466impl Host {
467 pub fn guard_stats(&self) -> Vec<Value> {
469 self.guard.guard_stats()
470 }
471
472 pub fn begin_turn(&self) {
474 self.guard.begin_turn();
475 }
476
477 pub fn cell_started(&self, lane: &str) {
479 self.shared.writes.begin(lane);
480 self.guard.cell_started(lane);
481 }
482
483 pub fn recording_cell_writes(self, on: bool) -> Self {
485 self.shared.writes.set_enabled(on);
486 self
487 }
488
489 pub fn cwd(&self) -> &Path {
491 &self.cwd
492 }
493
494 pub fn take_cell_writes(&self, lane: &str) -> Vec<cell_writes::CellWrite> {
496 self.shared.writes.take(lane)
497 }
498
499 pub fn cell_ended(&self, lane: &str) {
501 self.guard.cell_ended(lane);
502 }
503
504 pub fn take_pending(&self, lane: &str) -> Option<String> {
506 self.guard.take_pending(lane)
507 }
508
509 pub fn new(
512 cwd: impl Into<PathBuf>,
513 jobs_dir: impl Into<PathBuf>,
514 spill_dir: impl Into<PathBuf>,
515 guard: Box<dyn Guard>,
516 ) -> Self {
517 let jobs_dir = jobs_dir.into();
518 let _ = std::fs::DirBuilder::new().recursive(true).mode(0o700).create(&jobs_dir);
519 let cwd = cwd.into();
520 Host {
521 shared: Arc::new(Shared {
522 jobs: Mutex::new(Vec::new()),
523 journal: Mutex::new(None),
524 released: AtomicBool::new(false),
525 ends: Mutex::new(None),
526 roots: Mutex::new(None),
527 writes: cell_writes::CellWrites::default(),
528 }),
529 cwd,
530 jobs_dir,
531 spill_dir: spill_dir.into(),
532 guard,
533 launcher: None,
534 shell_path: None,
535 env: Vec::new(),
536 base_env: None,
537 owns_jobs: true,
538 }
539 }
540
541 pub fn with_base_env(mut self, env: Vec<(String, String)>) -> Self {
545 self.base_env = Some(Arc::new(env));
546 self
547 }
548
549 pub fn with_env(mut self, env: Vec<(String, String)>) -> Self {
552 self.env = env;
553 self
554 }
555
556 pub fn sharing_jobs(mut self, session: &Host) -> Self {
560 self.shared = session.shared.clone();
561 self.owns_jobs = false;
562 self
563 }
564
565 pub fn with_shell_path(mut self, shell_path: Option<String>) -> Self {
568 self.shell_path = shell_path;
569 self
570 }
571
572 fn shell(&self) -> Result<shell::ShellConfig, String> {
573 self.shell_named(self.shell_path.as_deref())
574 }
575
576 fn shell_named(&self, custom: Option<&str>) -> Result<shell::ShellConfig, String> {
578 match &self.base_env {
579 Some(env) => shell::shell_config_on_path(custom, path_of(env).as_deref()),
580 None => shell::system_shell_config(custom),
581 }
582 }
583
584 pub fn with_journal(self, journal: Arc<dyn DocumentStore>) -> Result<Host, String> {
589 let now = jsrt::date::now_ms() as u64;
590 let restored = journal::restore_durable(journal.read()?.as_ref(), now)?;
591 let mut adopted = Vec::new();
592 {
593 let mut jobs = sync::lock(&self.shared.jobs);
594 for saved in restored.records {
595 let job = Arc::new(Job::new(
596 saved.id.clone(),
597 (saved.lane.clone(), saved.root_id.clone()),
598 (saved.command.clone(), saved.cwd.clone()),
599 saved.timeout,
600 Path::new(&saved.output_path).parent().unwrap_or(&self.jobs_dir),
602 Record {
603 status: saved.status,
604 exit_code: saved.exit_code.and_then(|code| i32::try_from(code).ok()),
605 started_at: saved.started_at,
606 ended_at: saved.ended_at,
607 output_bytes: saved.output_bytes as usize,
608 retained_bytes: saved.retained_bytes as usize,
609 tail: saved.tail.clone(),
610 error: saved.error.clone(),
611 },
612 ));
613 if saved.status == Status::Running {
614 let pid = saved.pid.and_then(|pid| i32::try_from(pid).ok());
615 *sync::lock(&job.process) = pid.map(|pid| (pid, saved.pid_start));
616 adopted.push(job.clone());
617 } else {
618 job.finished();
619 }
620 jobs.push(job);
621 }
622 }
623 *sync::lock(&self.shared.journal) = Some(journal);
624 for job in adopted {
625 if keeper_alive(*sync::lock(&job.process)) {
626 let (runner, shared) = (job.clone(), self.shared.clone());
627 std::thread::spawn(move || follow(&runner, None, &shared));
628 } else {
629 follow(&job, None, &self.shared);
631 }
632 }
633 self.shared.persist();
634 Ok(self)
635 }
636
637 pub fn release_jobs(&self) {
645 loop {
646 let jobs = sync::lock(&self.shared.jobs);
647 let stopping: Vec<Arc<Job>> = jobs
648 .iter()
649 .filter(|job| *sync::lock(&job.cancel.0) && job.following())
650 .cloned()
651 .collect();
652 if stopping.is_empty() {
653 self.shared.persist_locked(&jobs);
654 self.shared.released.store(true, Ordering::SeqCst);
655 break;
656 }
657 drop(jobs);
658 for job in stopping {
659 job.wait_settled();
660 }
661 }
662 for job in sync::lock(&self.shared.jobs).iter() {
663 job.cancel.1.notify_all();
664 }
665 }
666
667 pub fn stop_jobs(&self) {
670 let jobs = sync::lock(&self.shared.jobs).clone();
671 for job in jobs {
672 cancel(&job, StopReason::Closed, &self.shared);
673 }
674 }
675
676 pub fn on_job_end(&self, ends: JobEnds) {
678 *sync::lock(&self.shared.ends) = Some(ends);
679 }
680
681 pub fn set_roots(&self, roots: RootOf) {
683 *sync::lock(&self.shared.roots) = Some(roots);
684 }
685
686 pub fn cancel_root(&self, root: &str) {
688 let jobs: Vec<Arc<Job>> = sync::lock(&self.shared.jobs)
689 .iter()
690 .filter(|job| job.root_id.as_deref() == Some(root))
691 .cloned()
692 .collect();
693 for job in jobs {
694 cancel(&job, StopReason::RootAborted, &self.shared);
695 }
696 }
697
698 pub fn job_waiters(&self, id: &str) -> usize {
701 sync::lock(&self.shared.jobs)
702 .iter()
703 .find(|job| job.id == id)
704 .map_or(0, |job| job.waiters.load(Ordering::SeqCst))
705 }
706
707 pub fn job_records(&self, lane: &str) -> Vec<JobRecord> {
710 sync::lock(&self.shared.jobs)
711 .iter()
712 .rev()
713 .filter(|job| visible(job, lane))
714 .map(|job| job_record(job, true))
715 .collect()
716 }
717
718 pub fn running_jobs(&self) -> usize {
719 sync::lock(&self.shared.jobs)
720 .iter()
721 .filter(|job| sync::lock(&job.record).status == Status::Running)
722 .count()
723 }
724
725 pub fn running(&self, lane: &str, excluded: &dyn Fn(Option<&str>) -> bool) -> usize {
727 sync::lock(&self.shared.jobs)
728 .iter()
729 .filter(|job| job.lane == lane && !excluded(job.root_id.as_deref()))
730 .filter(|job| sync::lock(&job.record).status == Status::Running)
731 .count()
732 }
733
734 pub fn handle_lane(
737 &self,
738 lane: &str,
739 kind: &str,
740 payload: Map<String, Value>,
741 signal: &Signal,
742 ) -> Result<Value, String> {
743 if kind == "files.before_write" {
744 let writes = match payload.get("writes") {
746 Some(Value::Array(writes)) if (1..=64).contains(&writes.len()) => writes,
747 _ => return Err("files.before_write needs 1 to 64 writes".into()),
748 };
749 let mut proposed = Vec::new();
750 for write in writes {
751 match (write.get("path"), write.get("content")) {
752 (Some(Value::String(path)), Some(Value::String(content))) if !path.is_empty() => {
753 proposed.push((path.as_str(), content.as_str()));
754 }
755 _ => return Err("files.before_write writes need a path and text content".into()),
756 }
757 }
758 let results: Vec<Value> = proposed
759 .into_iter()
760 .map(|(path, content)| {
761 let result = self.guard.before_write(lane, path, content);
762 if result["blocked"] != true {
763 let path = file_hooks::resolve_lexical(&self.cwd, path);
764 self.shared.writes.record(lane, &path, content);
765 }
766 result
767 })
768 .collect();
769 return Ok(json!({ "results": results }));
770 }
771 if matches!(kind, "bash" | "shell.run")
772 && let Some(Value::String(command)) = payload.get("command")
773 && let Some(reason) = self.guard.before_shell(lane, command)
774 {
775 return Err(format!("bash command was not run: {reason}"));
776 }
777 let requested = payload.get("command").and_then(Value::as_str).map(str::to_string);
778 let mut result = self.dispatch_shell(lane, kind, payload, signal)?;
779 if matches!(
781 kind,
782 "bash" | "shell.run" | "shell.result" | "shell.get" | "shell.cancel"
783 ) {
784 let command = requested.unwrap_or_else(|| result["command"].as_str().unwrap_or_default().to_string());
785 let policy = self
786 .launcher
787 .as_deref()
788 .filter(|launcher| launcher.mechanism().is_some())
789 .map(|launcher| &launcher.policy);
790 bash_notes::annotate(&mut result, &command, policy);
791 }
792 Ok(result)
793 }
794
795 fn dispatch_shell(
796 &self,
797 lane: &str,
798 kind: &str,
799 payload: Map<String, Value>,
800 signal: &Signal,
801 ) -> Result<Value, String> {
802 match kind {
803 "bash" if payload.contains_key("yield_after") => self.shell_bash(lane, &payload, signal),
804 "bash" => self.blocking_bash(&payload, signal, &mut |_| {}),
805 "shell.run" => {
806 kernel::ops::fields(kind, &payload)?;
807 let command = command_of(&payload)?;
808 let timeout = timeout_of(payload.get("timeout"))?;
809 let yield_after = wait_seconds(payload.get("yield_after"), "yield_after")?.unwrap_or(0.0);
810 let job = self.start(lane, command, timeout)?;
811 wait(&job, Some(yield_after), signal);
812 Ok(snapshot(&job))
813 }
814 "shell.result" => {
815 kernel::ops::fields(kind, &payload)?;
816 let job = self.find(lane, payload.get("id"))?;
817 if sync::lock(&job.record).status == Status::Running {
819 wait(&job, wait_seconds(payload.get("wait"), "wait")?, signal);
820 }
821 Ok(snapshot(&job))
822 }
823 "shell.get" => {
824 kernel::ops::fields(kind, &payload)?;
825 let job = self.find(lane, payload.get("id"))?;
826 Ok(snapshot(&job))
827 }
828 "shell.cancel" => {
829 kernel::ops::fields(kind, &payload)?;
830 let job = self.find(lane, payload.get("id"))?;
831 cancel(&job, StopReason::Cancelled, &self.shared);
832 Ok(snapshot(&job))
833 }
834 "shell.read" => {
835 kernel::ops::fields(kind, &payload)?;
836 let job = self.find(lane, payload.get("id"))?;
837 read_output(&job, payload.get("cursor"), payload.get("max_bytes"))
838 }
839 "shell.list" => {
840 kernel::ops::fields(kind, &payload)?;
841 let jobs = sync::lock(&self.shared.jobs).clone();
842 Ok(Value::Array(
843 jobs.iter()
844 .rev()
845 .filter(|job| visible(job, lane))
846 .map(|job| {
847 let mut snapshot = snapshot(job);
848 snapshot["text"] = Value::Null;
849 snapshot
850 })
851 .collect(),
852 ))
853 }
854 other if other.starts_with("shell.") => Err(kernel::ops::not_wired(other)),
855 other => Err(not_available(other)),
856 }
857 }
858
859 pub fn user_bash(&self, command: &str, signal: &Signal, on_chunk: &mut dyn FnMut(&str)) -> Result<Value, String> {
863 if let Some(reason) = self.guard.before_shell(kernel::ROOT_LANE, command) {
864 return Err(format!("bash command was not run: {reason}"));
865 }
866 let mut payload = Map::new();
867 payload.insert("command".into(), command.into());
868 self.blocking_bash(&payload, signal, on_chunk)
869 }
870
871 fn blocking_bash(
873 &self,
874 payload: &Map<String, Value>,
875 signal: &Signal,
876 on_chunk: &mut dyn FnMut(&str),
877 ) -> Result<Value, String> {
878 let command = command_of(payload)?;
879 let timeout = timeout_of(payload.get("timeout"))?;
880 let deadline = timeout.map(|seconds| Instant::now() + Duration::from_secs_f64(seconds));
881 let timed_out = std::cell::Cell::new(false);
882 let mut chunks: std::collections::VecDeque<String> = Default::default();
883 let mut kept_units = 0usize;
884 let mut total_bytes = 0usize;
885 let mut spill: Option<(PathBuf, File)> = None;
886 let open_spill =
887 |spill: &mut Option<(PathBuf, File)>, chunks: &std::collections::VecDeque<String>| -> Result<(), String> {
888 if spill.is_none() {
889 let path = self
890 .spill_dir
891 .join(format!("pi-bash-{}.log", jsrt::random::random_hex(8)));
892 let mut file = File::create(&path).map_err(|error| error.to_string())?;
893 for chunk in chunks {
894 let _ = file.write_all(chunk.as_bytes());
895 }
896 *spill = Some((path, file));
897 }
898 Ok(())
899 };
900 let shell = self.shell()?;
901 let mut failure = None;
902 let exit = run_shell(
903 &shell,
904 &command,
905 CommandEnv {
906 cwd: &self.cwd,
907 base: self.base_env.as_deref().map(Vec::as_slice),
908 env: &self.env,
909 },
910 self.launcher.as_deref(),
911 |text, raw| {
912 if raw > 0 || !text.is_empty() {
914 on_chunk(text);
915 }
916 total_bytes += raw;
917 if total_bytes > DEFAULT_MAX_BYTES
918 && let Err(error) = open_spill(&mut spill, &chunks)
919 {
920 failure = Some(error);
921 }
922 if let Some((_, file)) = spill.as_mut() {
923 let _ = file.write_all(text.as_bytes());
924 }
925 chunks.push_back(text.to_owned());
926 kept_units += text.encode_utf16().count();
927 while kept_units > DEFAULT_MAX_BYTES * 2 && chunks.len() > 1 {
928 if let Some(removed) = chunks.pop_front() {
929 kept_units -= removed.encode_utf16().count();
930 }
931 }
932 },
933 || {
934 if deadline.is_some_and(|deadline| Instant::now() >= deadline) {
935 timed_out.set(true);
936 }
937 timed_out.get() || signal.is_aborted()
938 },
939 )?;
940 if let Some(error) = failure {
941 return Err(error);
942 }
943 let full: String = chunks.iter().map(String::as_str).collect();
944 let truncated = truncate_tail(&full);
945 if truncated.is_some() {
946 open_spill(&mut spill, &chunks)?;
947 }
948 let stopped = timed_out.get() || signal.is_aborted();
949 Ok(json!({
950 "output": truncated.clone().unwrap_or(full),
951 "exit_code": if stopped { None } else { exit },
952 "cancelled": stopped && !timed_out.get(),
953 "timed_out": timed_out.get(),
954 "truncated": truncated.is_some(),
955 "full_output_path": spill.map(|(path, _)| path.to_string_lossy().into_owned()),
956 }))
957 }
958
959 fn shell_bash(&self, lane: &str, payload: &Map<String, Value>, signal: &Signal) -> Result<Value, String> {
961 kernel::ops::fields("bash", payload)?;
962 let command = command_of(payload)?;
963 let timeout = timeout_of(payload.get("timeout"))?;
964 let yield_after = wait_seconds(payload.get("yield_after"), "yield_after")?;
965 let job = self.start(lane, command, timeout)?;
966 wait(&job, yield_after, signal);
967 if sync::lock(&job.record).status == Status::Running && signal.is_aborted() {
968 job.waiters.fetch_add(1, Ordering::SeqCst);
969 cancel(&job, StopReason::Cancelled, &self.shared);
970 job.waiters.fetch_sub(1, Ordering::SeqCst);
971 }
972 bash_result(&job)
973 }
974
975 fn find(&self, lane: &str, id: Option<&Value>) -> Result<Arc<Job>, String> {
976 let Some(Value::String(id)) = id.filter(|id| id.as_str().is_some_and(|id| !id.is_empty())) else {
977 return Err("job id must be a nonempty string".into());
978 };
979 sync::lock(&self.shared.jobs)
980 .iter()
981 .find(|job| &job.id == id && visible(job, lane))
982 .cloned()
983 .ok_or_else(|| format!("Unknown shell job {id}"))
984 }
985
986 fn start(&self, lane: &str, command: String, timeout: Option<f64>) -> Result<Arc<Job>, String> {
987 let root = sync::lock(&self.shared.roots).clone().and_then(|roots| roots(lane));
988 let mut jobs = sync::lock(&self.shared.jobs);
989 let running = |job: &Arc<Job>| sync::lock(&job.record).status == Status::Running;
990 if jobs.iter().filter(|job| running(job)).count() >= JOB_MAX_ACTIVE {
991 return Err(format!("At most {JOB_MAX_ACTIVE} shell jobs can run at once"));
992 }
993 let id = format!("job-{}", jsrt::random::random_hex(4));
994 let job = Arc::new(Job::new(
995 id,
996 (lane.to_string(), root),
997 (command, self.cwd.to_string_lossy().into_owned()),
998 timeout,
999 &self.jobs_dir,
1000 Record {
1001 status: Status::Running,
1002 exit_code: None,
1003 started_at: jsrt::date::now_ms() as u64,
1004 ended_at: None,
1005 output_bytes: 0,
1006 retained_bytes: 0,
1007 tail: String::new(),
1008 error: None,
1009 },
1010 ));
1011 jobs.push(job.clone());
1012 let excess = jobs.len().saturating_sub(JOB_MAX_RETAINED);
1014 let pruned: Vec<String> = jobs
1015 .iter()
1016 .filter(|job| !running(job))
1017 .take(excess)
1018 .map(|job| job.id.clone())
1019 .collect();
1020 jobs.retain(|job| {
1021 let keep = !pruned.contains(&job.id);
1022 if !keep {
1023 for path in [&job.output_path, &job.raw_path, &job.exit_path] {
1024 let _ = std::fs::remove_file(path);
1025 }
1026 }
1027 keep
1028 });
1029 drop(jobs);
1030 let (started, spawned) = mpsc::channel();
1033 let (runner, shared, cwd, launcher, shell) = (
1034 job.clone(),
1035 self.shared.clone(),
1036 (self.cwd.clone(), self.base_env.clone(), self.env.clone()),
1037 self.launcher.clone(),
1038 self.shell()
1040 .and_then(|shell| Ok((shell, self.shell_named(None)?.shell))),
1041 );
1042 std::thread::spawn(move || {
1043 let env = CommandEnv {
1044 cwd: &cwd.0,
1045 base: cwd.1.as_deref().map(Vec::as_slice),
1046 env: &cwd.2,
1047 };
1048 match shell.and_then(|(shell, keeper)| spawn_keeper(&runner, (&shell, &keeper), env, launcher.as_deref())) {
1049 Ok(child) => {
1050 let pid = child.id() as i32;
1051 *sync::lock(&runner.process) = Some((pid, process_start(pid)));
1052 let _ = started.send(());
1053 follow(&runner, Some(child), &shared);
1054 }
1055 Err(error) => {
1056 let mut record = sync::lock(&runner.record);
1057 record.status = Status::Failed;
1058 record.ended_at = Some(jsrt::date::now_ms() as u64);
1059 record.error = Some(error);
1060 drop(record);
1061 let _ = started.send(());
1062 runner.finished();
1063 }
1064 }
1065 });
1066 let _ = spawned.recv();
1067 self.shared.persist();
1068 Ok(job)
1069 }
1070}
1071
1072const KEEPER: &str = r#""$1" "$2" "$0" 2>&1 </dev/null; printf '%s\n' "$?" >&2"#;
1077
1078fn spawn_keeper(
1079 job: &Job,
1080 (shell, keeper): (&shell::ShellConfig, &str),
1081 CommandEnv { cwd, base, env }: CommandEnv<'_>,
1082 launcher: Option<&sandbox::Launcher>,
1083) -> Result<std::process::Child, String> {
1084 let create = |path: &Path| {
1085 std::fs::OpenOptions::new()
1086 .write(true)
1087 .create(true)
1088 .truncate(true)
1089 .mode(0o600)
1090 .open(path)
1091 .map_err(|error| format!("{}: {error}", path.display()))
1092 };
1093 let raw = create(&job.raw_path)?;
1094 let exit = create(&job.exit_path)?;
1095 let mut argv = vec![keeper.to_string(), "-c".into(), KEEPER.into(), job.command.clone()];
1096 argv.push(shell.shell.clone());
1097 argv.extend(shell.args.iter().cloned());
1098 let mut process = match launcher {
1099 Some(launcher) => launcher.command_persistent(&sandboxed_argv(argv, env), cwd)?,
1101 None => {
1102 let mut process = Command::new(&argv[0]);
1103 process.args(&argv[1..]).current_dir(cwd);
1104 process
1105 }
1106 };
1107 let isolated = launcher.is_some_and(|launcher| launcher.mechanism().is_some());
1108 command_env(&mut process, base, env, isolated);
1109 process
1110 .env_remove(PARENT_TOKEN_ENV)
1111 .stdin(Stdio::null())
1112 .stdout(raw)
1113 .stderr(exit)
1114 .process_group(0)
1115 .spawn()
1116 .map_err(|error| error.to_string())
1117}
1118
1119struct Digest {
1122 decoder: Decoder,
1123 log: Option<File>,
1124}
1125
1126impl Digest {
1127 fn new(job: &Job) -> Digest {
1129 let log = std::fs::OpenOptions::new()
1130 .write(true)
1131 .create(true)
1132 .truncate(true)
1133 .mode(0o600)
1134 .open(&job.output_path)
1135 .ok();
1136 let mut record = sync::lock(&job.record);
1137 record.output_bytes = 0;
1138 record.retained_bytes = 0;
1139 record.tail.clear();
1140 Digest {
1141 decoder: Decoder::default(),
1142 log,
1143 }
1144 }
1145
1146 fn text(&mut self, job: &Job, text: &str) {
1147 if text.is_empty() {
1148 return;
1149 }
1150 let mut record = sync::lock(&job.record);
1151 record.output_bytes += text.len();
1152 let room = JOB_RETAINED_BYTES.saturating_sub(record.retained_bytes);
1153 if room > 0 {
1154 let kept = &text.as_bytes()[..room.min(text.len())];
1155 if let Some(log) = self.log.as_mut() {
1156 let _ = log.write_all(kept);
1157 }
1158 record.retained_bytes += kept.len();
1159 }
1160 let tail = format!("{}{text}", record.tail);
1161 record.tail = tail_bytes(&tail, JOB_TEXT_BYTES / 2).to_owned();
1162 }
1163
1164 fn pump(&mut self, job: &Job, raw: &mut Option<File>) {
1166 let Some(file) = raw.as_mut() else {
1167 return;
1168 };
1169 let mut buffer = vec![0u8; 64 * 1024];
1170 while let Ok(read) = file.read(&mut buffer) {
1171 if read == 0 {
1172 break;
1173 }
1174 let text = clean(&self.decoder.decode(&buffer[..read], true));
1175 self.text(job, &text);
1176 }
1177 }
1178
1179 fn finish(&mut self, job: &Job) {
1180 let text = clean(&self.decoder.decode(&[], false));
1181 self.text(job, &text);
1182 }
1183}
1184
1185fn follow(job: &Arc<Job>, child: Option<std::process::Child>, shared: &Shared) {
1189 let own = child.is_some();
1190 if let Some(mut child) = child {
1191 let waiter = job.clone();
1193 std::thread::spawn(move || {
1194 let code = child.wait().ok().and_then(|status| status.code());
1195 let guard = sync::lock(&waiter.cancel.0);
1196 *sync::lock(&waiter.exited) = Some(code);
1197 drop(guard);
1198 waiter.cancel.1.notify_all();
1199 });
1200 }
1201 let mut digest = Digest::new(job);
1202 let mut raw = File::open(&job.raw_path).ok();
1203 let started_at = sync::lock(&job.record).started_at;
1204 let deadline = job
1205 .timeout
1206 .map(|seconds| started_at.saturating_add((seconds * 1000.0) as u64));
1207 let process = *sync::lock(&job.process);
1208 let mut stopped: Option<Status> = None;
1209 let ended = loop {
1210 let ended = match own {
1211 true => *sync::lock(&job.exited),
1212 false => {
1213 let code = exit_status(&job.exit_path);
1214 (code.is_some() || !keeper_alive(process)).then_some(code)
1215 }
1216 };
1217 digest.pump(job, &mut raw);
1218 if let Some(ended) = ended {
1219 break ended;
1220 }
1221 if shared.released.load(Ordering::SeqCst) {
1222 job.stop_following(Tracking::Released);
1223 return;
1224 }
1225 if stopped.is_none() {
1226 if deadline.is_some_and(|deadline| jsrt::date::now_ms() as u64 >= deadline) {
1227 stopped = Some(Status::TimedOut);
1228 } else if *sync::lock(&job.cancel.0) {
1229 stopped = Some(Status::Cancelled);
1230 }
1231 if stopped.is_some()
1233 && let Some((pid, _)) = process
1234 {
1235 kernel::kill_command_group(pid);
1236 }
1237 }
1238 let cancel = sync::lock(&job.cancel.0);
1239 if sync::lock(&job.exited).is_none() {
1240 drop(
1241 job.cancel
1242 .1
1243 .wait_timeout(cancel, POLL)
1244 .unwrap_or_else(PoisonError::into_inner),
1245 );
1246 }
1247 };
1248 digest.finish(job);
1249 let code = exit_status(&job.exit_path);
1251 let jobs = sync::lock(&shared.jobs);
1252 if shared.released.load(Ordering::SeqCst) {
1253 drop(jobs);
1255 job.stop_following(Tracking::Released);
1256 return;
1257 }
1258 {
1259 let mut record = sync::lock(&job.record);
1260 record.ended_at = Some(match own {
1261 true => jsrt::date::now_ms() as u64,
1262 false => std::fs::metadata(&job.exit_path)
1264 .and_then(|meta| meta.modified())
1265 .ok()
1266 .filter(|_| code.is_some())
1267 .and_then(|time| time.duration_since(UNIX_EPOCH).ok())
1268 .map_or_else(|| jsrt::date::now_ms() as u64, |d| d.as_millis() as u64),
1269 });
1270 match stopped {
1271 Some(status) => {
1272 record.status = status;
1273 record.exit_code = None;
1274 if status == Status::TimedOut {
1275 record.error = Some(format!(
1276 "timed out after {}s; process tree killed",
1277 jsrt::number::to_string(job.timeout.unwrap_or(0.0))
1278 ));
1279 }
1280 }
1281 None => match (code, own) {
1282 (Some(code), _) => {
1283 record.status = Status::Completed;
1284 record.exit_code = Some(code);
1285 }
1286 (None, true) => {
1288 record.status = Status::Completed;
1289 record.exit_code = ended;
1290 }
1291 (None, false) => {
1292 record.status = Status::Lost;
1293 record.exit_code = None;
1294 record.error =
1295 Some("its process ended while no Ultron worker was running; the exit status is unknown".into());
1296 }
1297 },
1298 }
1299 }
1300 shared.persist_locked(&jobs);
1301 drop(jobs);
1302 let reason = *sync::lock(&job.reason);
1304 let end = JobEnd {
1305 record: job_record(job, true),
1306 awaited: job.waiters.load(Ordering::SeqCst) > 0,
1307 root_aborted: reason == Some(StopReason::RootAborted),
1308 };
1309 let ends = sync::lock(&shared.ends).clone();
1310 job.finished();
1311 if reason != Some(StopReason::Closed)
1312 && let Some(ends) = ends
1313 {
1314 ends(end);
1315 }
1316}
1317
1318fn wait(job: &Arc<Job>, seconds: Option<f64>, signal: &Signal) {
1320 job.waiters.fetch_add(1, Ordering::SeqCst);
1321 let deadline = seconds.map(|seconds| Instant::now() + Duration::from_secs_f64(seconds.max(0.02)));
1322 let woken = job.clone();
1323 let link = signal.link(move |_| {
1324 let _tracking = sync::lock(&woken.tracking.0);
1325 woken.tracking.1.notify_all();
1326 });
1327 let mut done = sync::lock(&job.tracking.0);
1328 while *done == Tracking::Following && !signal.is_aborted() {
1329 done = match deadline {
1330 None => job.tracking.1.wait(done).unwrap_or_else(PoisonError::into_inner),
1331 Some(deadline) => {
1332 let now = Instant::now();
1333 if now >= deadline {
1334 break;
1335 }
1336 job.tracking
1337 .1
1338 .wait_timeout(done, deadline - now)
1339 .unwrap_or_else(PoisonError::into_inner)
1340 .0
1341 }
1342 };
1343 }
1344 drop(done);
1345 drop(link);
1346 job.waiters.fetch_sub(1, Ordering::SeqCst);
1347}
1348
1349fn cancel(job: &Job, reason: StopReason, shared: &Shared) {
1352 {
1353 let _jobs = sync::lock(&shared.jobs);
1355 if shared.released.load(Ordering::SeqCst) || !job.following() {
1356 return;
1357 }
1358 sync::lock(&job.reason).get_or_insert(reason);
1359 *sync::lock(&job.cancel.0) = true;
1360 }
1361 job.cancel.1.notify_all();
1362 job.wait_settled();
1363}
1364
1365fn read_file(path: &Path, cursor: usize, max: usize) -> String {
1366 let mut buffer = Vec::new();
1367 if let Ok(mut file) = File::open(path)
1368 && file.seek(SeekFrom::Start(cursor as u64)).is_ok()
1369 {
1370 let _ = file.take(max as u64).read_to_end(&mut buffer);
1371 }
1372 String::from_utf8_lossy(&buffer).into_owned()
1373}
1374
1375fn job_text(job: &Job, record: &Record) -> (String, bool) {
1377 let half = JOB_TEXT_BYTES / 2;
1378 if record.output_bytes <= JOB_TEXT_BYTES && record.retained_bytes == record.output_bytes {
1379 return (read_file(&job.output_path, 0, JOB_TEXT_BYTES), false);
1380 }
1381 let head = read_file(&job.output_path, 0, half);
1382 let omitted = record.output_bytes.saturating_sub(half + record.tail.len());
1383 let marker = format!(
1384 "\n[... {omitted} bytes omitted; {} bytes kept in {}: await job.read(cursor={half}, max_bytes=65536) ...]\n",
1385 record.retained_bytes,
1386 job.output_path.display()
1387 );
1388 (format!("{head}{marker}{}", record.tail), true)
1389}
1390
1391fn snapshot(job: &Job) -> Value {
1392 let record = sync::lock(&job.record);
1393 let (text, truncated) = job_text(job, &record);
1394 let elapsed = record
1395 .ended_at
1396 .unwrap_or_else(|| jsrt::date::now_ms() as u64)
1397 .saturating_sub(record.started_at);
1398 json!({
1399 "id": job.id,
1400 "lane": job.lane,
1401 "command": job.command,
1402 "status": record.status.as_str(),
1403 "running": record.status == Status::Running,
1404 "exit_code": record.exit_code,
1405 "ok": record.status == Status::Completed && record.exit_code == Some(0),
1406 "timed_out": record.status == Status::TimedOut,
1407 "cancelled": record.status == Status::Cancelled,
1408 "text": text,
1409 "truncated": truncated,
1410 "output_path": job.output_path.to_string_lossy(),
1411 "output_bytes": record.output_bytes,
1412 "elapsed_seconds": jsrt::json::number((elapsed as f64 / 100.0).round() / 10.0),
1413 "error": record.error,
1414 })
1415}
1416
1417fn bash_result(job: &Job) -> Result<Value, String> {
1419 let (status, exit_code, error, partial) = {
1420 let record = sync::lock(&job.record);
1421 (
1422 record.status,
1423 record.exit_code,
1424 record.error.clone(),
1425 record.retained_bytes < record.output_bytes,
1426 )
1427 };
1428 if status == Status::Failed {
1429 return Err(error.unwrap_or_else(|| "bash command failed".into()));
1430 }
1431 let snapshot = snapshot(job);
1432 let truncated = snapshot["truncated"].as_bool() == Some(true);
1433 let running = status == Status::Running;
1434 Ok(json!({
1435 "output": snapshot["text"],
1436 "exit_code": exit_code,
1437 "cancelled": status == Status::Cancelled,
1438 "timed_out": status == Status::TimedOut,
1439 "truncated": truncated,
1440 "full_output_path": if truncated { snapshot["output_path"].clone() } else { Value::Null },
1441 "partial_file": partial,
1442 "running": running,
1443 "job": if running { snapshot } else { Value::Null },
1444 }))
1445}
1446
1447fn read_output(job: &Job, cursor: Option<&Value>, max_bytes: Option<&Value>) -> Result<Value, String> {
1448 let cursor = safe_integer(cursor, 0).ok_or("cursor must be a non-negative integer")? as usize;
1449 let max = safe_integer(max_bytes, 65_536)
1450 .filter(|max| (1..=1_048_576).contains(max))
1451 .ok_or("max_bytes must be an integer between 1 and 1048576")? as usize;
1452 let record = sync::lock(&job.record);
1453 let start = cursor.min(record.retained_bytes);
1454 let text = read_file(&job.output_path, start, max.min(record.retained_bytes - start));
1455 let next = start + text.len();
1456 Ok(json!({
1457 "text": text,
1458 "next_cursor": next,
1459 "done": record.status != Status::Running && next >= record.retained_bytes,
1460 "truncated": record.retained_bytes < record.output_bytes,
1461 }))
1462}
1463
1464impl HostHandler for Host {
1465 fn handle(&self, kind: &str, payload: Map<String, Value>, signal: &Signal) -> Result<Value, String> {
1466 self.handle_lane(kernel::ROOT_LANE, kind, payload, signal)
1467 }
1468}
1469
1470fn visible(job: &Job, lane: &str) -> bool {
1471 kernel::sees(lane, &job.lane)
1472}
1473
1474impl Drop for Host {
1475 fn drop(&mut self) {
1477 if self.owns_jobs && !self.shared.released.load(Ordering::SeqCst) {
1478 self.stop_jobs();
1479 }
1480 }
1481}