use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, HashMap};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use thiserror::Error;
use tokio::sync::{Mutex, RwLock};
use tokio::task::JoinHandle;
use crate::manifest::AgentManifest;
#[derive(Debug, Error)]
pub enum SupervisorError {
#[error("invalid agent id (must be non-empty, alphanumeric + `-_.`): {0:?}")]
InvalidId(String),
#[error("invalid agent command: {reason} ({command:?})")]
InvalidCommand {
command: String,
reason: &'static str,
},
#[error("agent {0} not found")]
NotFound(String),
#[error("timed out after {timeout:?} waiting for agent {id} to reach a target status (last status: {last:?})")]
WaitTimeout {
id: String,
last: AgentStatus,
timeout: std::time::Duration,
},
#[error("could not resolve home directory")]
NoHomeDir,
#[error("supervisor I/O error: {0}")]
Io(#[from] std::io::Error),
#[error("supervisor JSON error: {0}")]
Json(#[from] serde_json::Error),
#[error("{0}")]
Other(String),
#[error("another supervisor already owns this manifest (lock file: {0}). Refusing to spawn duplicates.")]
AlreadyRunning(PathBuf),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum RestartPolicy {
Never,
#[default]
OnFailure,
Always,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum AgentStatus {
#[default]
Stopped,
Starting,
Running,
Backoff,
Errored,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AgentSpec {
pub id: String,
pub name: String,
pub command: String,
#[serde(default)]
pub args: Vec<String>,
#[serde(default)]
pub cwd: Option<PathBuf>,
#[serde(default)]
pub env: BTreeMap<String, String>,
#[serde(default)]
pub restart: RestartPolicy,
#[serde(default = "default_max_restarts")]
pub max_restarts: u32,
#[serde(default = "default_backoff")]
pub backoff_secs: u64,
#[serde(default)]
pub auto_start: bool,
#[serde(default)]
pub token: String,
#[serde(default)]
pub capabilities: Vec<String>,
}
fn default_max_restarts() -> u32 {
10
}
fn default_backoff() -> u64 {
5
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ManagedAgent {
#[serde(flatten)]
pub spec: AgentSpec,
pub status: AgentStatus,
pub pid: Option<u32>,
pub last_exit_code: Option<i32>,
pub restart_count: u32,
pub started_at: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub blocked_by_pid: Option<i32>,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[derive(Default)]
pub enum StopSignal {
#[default]
Term,
Kill,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum LogStream {
#[default]
Combined,
Stdout,
Stderr,
}
impl LogStream {
pub fn from_wire(s: Option<&str>) -> LogStream {
match s {
Some("stdout") => LogStream::Stdout,
Some("stderr") => LogStream::Stderr,
_ => LogStream::Combined,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LogTail {
pub lines: Vec<String>,
pub stdout: Vec<String>,
pub stderr: Vec<String>,
pub stdout_total: usize,
pub stderr_total: usize,
pub stdout_path: String,
pub stderr_path: String,
pub more: bool,
}
#[derive(Default)]
struct StreamWindow {
lines: Vec<String>,
total: usize,
more: bool,
}
#[derive(Debug, Default, Serialize, Deserialize)]
struct Manifest {
#[serde(default)]
agents: Vec<AgentSpec>,
}
struct AgentSlot {
spec: AgentSpec,
runtime: AgentRuntime,
stop_tx: Option<tokio::sync::watch::Sender<bool>>,
task: Option<JoinHandle<()>>,
job: Option<Arc<JobObject>>,
}
#[derive(Debug, Clone, Default)]
struct AgentRuntime {
status: AgentStatus,
pid: Option<u32>,
last_exit_code: Option<i32>,
restart_count: u32,
started_at: Option<i64>,
blocked_by_pid: Option<i32>,
}
#[derive(Clone)]
pub struct Supervisor {
manifest_path: PathBuf,
log_dir: PathBuf,
state: Arc<RwLock<HashMap<String, AgentSlot>>>,
manifest_lock: Arc<Mutex<()>>,
_process_lock: Arc<std::fs::File>,
pub grace_secs: u64,
default_child_env: Arc<RwLock<BTreeMap<String, String>>>,
}
impl Supervisor {
pub fn user_default() -> Result<Self, SupervisorError> {
let manifest_path = Self::user_default_manifest_path()?;
let log_dir = manifest_path
.parent()
.map(|p| p.join("logs"))
.unwrap_or_else(|| PathBuf::from("logs"));
Self::with_paths(manifest_path, log_dir)
}
pub fn user_default_manifest_path() -> Result<PathBuf, SupervisorError> {
let root = car_home::root().ok_or(SupervisorError::NoHomeDir)?;
Ok(root.join("agents.json"))
}
pub fn list_from_manifest(manifest_path: &Path) -> Result<Vec<ManagedAgent>, SupervisorError> {
let m = load_manifest(manifest_path)?;
let mut out: Vec<ManagedAgent> = m
.agents
.into_iter()
.map(|spec| ManagedAgent {
spec,
status: AgentStatus::default(),
pid: None,
last_exit_code: None,
restart_count: 0,
started_at: None,
blocked_by_pid: None,
})
.collect();
out.sort_by(|a, b| a.spec.id.cmp(&b.spec.id));
Ok(out)
}
pub fn health_from_manifest(manifest_path: &Path) -> Result<Vec<AgentHealth>, SupervisorError> {
let m = load_manifest(manifest_path)?;
let mut out: Vec<AgentHealth> = m
.agents
.into_iter()
.map(|spec| {
let command = spec.command.clone();
match validate_command(&command) {
Ok(()) => AgentHealth {
id: spec.id,
command,
ok: true,
reason: None,
},
Err(e) => AgentHealth {
id: spec.id,
command,
ok: false,
reason: Some(e.to_string()),
},
}
})
.collect();
out.sort_by(|a, b| a.id.cmp(&b.id));
Ok(out)
}
pub fn with_paths(manifest_path: PathBuf, log_dir: PathBuf) -> Result<Self, SupervisorError> {
if let Some(parent) = manifest_path.parent() {
std::fs::create_dir_all(parent)?;
}
std::fs::create_dir_all(&log_dir)?;
let lock_path = {
let mut s = manifest_path.as_os_str().to_owned();
s.push(".lock");
PathBuf::from(s)
};
let lock_file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&lock_path)?;
match lock_file.try_lock() {
Ok(()) => {}
Err(std::fs::TryLockError::WouldBlock) => {
return Err(SupervisorError::AlreadyRunning(lock_path));
}
Err(std::fs::TryLockError::Error(e)) => return Err(SupervisorError::Io(e)),
}
let agents_dir = manifest_path
.parent()
.map(|p| p.join("agents"))
.unwrap_or_else(|| PathBuf::from("agents"));
std::fs::create_dir_all(&agents_dir)?;
let legacy = load_manifest(&manifest_path)?;
let mut by_id: HashMap<String, AgentSpec> = HashMap::new();
if !legacy.agents.is_empty() {
tracing::warn!(
count = legacy.agents.len(),
path = %manifest_path.display(),
"loading agents from legacy agents.json. This file is \
deprecated; entries are mirrored to agents/<id>/manifest.toml \
and the legacy file will stop being read in a future release."
);
}
for spec in legacy.agents {
by_id.insert(spec.id.clone(), spec);
}
let manifests = crate::manifest::load_manifest_dir(&agents_dir)?;
for m in &manifests {
if m.is_pure_data() || m.is_remote_service() {
continue;
}
match crate::manifest::to_agent_spec(m) {
Ok(spec) => {
by_id.insert(spec.id.clone(), spec);
}
Err(e) => {
tracing::warn!(
manifest_id = %m.agent.id,
error = %e,
"manifest.toml could not project to an AgentSpec; \
agent will not be supervised this boot"
);
}
}
}
let existing_manifest_ids: std::collections::HashSet<&str> =
manifests.iter().map(|m| m.agent.id.as_str()).collect();
for spec in by_id.values() {
if existing_manifest_ids.contains(spec.id.as_str()) {
continue;
}
let m = crate::manifest::from_legacy_spec(spec);
if let Err(e) = crate::manifest::write_manifest(&agents_dir, &m) {
tracing::warn!(
id = %spec.id,
error = %e,
"failed to mirror legacy AgentSpec to manifest.toml; \
entry remains in agents.json only"
);
}
}
let mut state: HashMap<String, AgentSlot> = HashMap::new();
for (id, spec) in by_id {
state.insert(
id,
AgentSlot {
spec,
runtime: AgentRuntime::default(),
stop_tx: None,
task: None,
job: None,
},
);
}
Ok(Self {
manifest_path,
log_dir,
state: Arc::new(RwLock::new(state)),
manifest_lock: Arc::new(Mutex::new(())),
_process_lock: Arc::new(lock_file),
grace_secs: 10,
default_child_env: Arc::new(RwLock::new(BTreeMap::new())),
})
}
pub async fn set_default_child_env<I, K, V>(&self, entries: I)
where
I: IntoIterator<Item = (K, V)>,
K: Into<String>,
V: Into<String>,
{
let mut g = self.default_child_env.write().await;
g.clear();
for (k, v) in entries {
g.insert(k.into(), v.into());
}
}
pub async fn default_child_env(&self) -> BTreeMap<String, String> {
self.default_child_env.read().await.clone()
}
pub fn manifest_path(&self) -> &Path {
&self.manifest_path
}
pub fn log_dir(&self) -> &Path {
&self.log_dir
}
pub async fn list(&self) -> Vec<ManagedAgent> {
let state = self.state.read().await;
let mut out: Vec<ManagedAgent> = state
.values()
.map(|slot| self.snapshot_locked(slot))
.collect();
out.sort_by(|a, b| a.spec.id.cmp(&b.spec.id));
out
}
pub async fn get(&self, id: &str) -> Option<ManagedAgent> {
self.state
.read()
.await
.get(id)
.map(|slot| self.snapshot_locked(slot))
}
pub async fn wait_for(
&self,
id: &str,
targets: &[AgentStatus],
timeout: std::time::Duration,
poll_interval: std::time::Duration,
) -> Result<ManagedAgent, SupervisorError> {
let deadline = tokio::time::Instant::now() + timeout;
let poll = poll_interval.max(std::time::Duration::from_millis(10));
loop {
let snap = self
.get(id)
.await
.ok_or_else(|| SupervisorError::NotFound(id.to_string()))?;
if targets.contains(&snap.status) {
return Ok(snap);
}
let now = tokio::time::Instant::now();
if now >= deadline {
return Err(SupervisorError::WaitTimeout {
id: id.to_string(),
last: snap.status,
timeout,
});
}
tokio::time::sleep(poll.min(deadline - now)).await;
}
}
pub async fn health(&self) -> Vec<AgentHealth> {
let state = self.state.read().await;
let mut out: Vec<AgentHealth> = state
.values()
.map(|slot| {
let command = slot.spec.command.clone();
match validate_command(&command) {
Ok(()) => AgentHealth {
id: slot.spec.id.clone(),
command,
ok: true,
reason: None,
},
Err(e) => AgentHealth {
id: slot.spec.id.clone(),
command,
ok: false,
reason: Some(e.to_string()),
},
}
})
.collect();
out.sort_by(|a, b| a.id.cmp(&b.id));
out
}
pub async fn upsert(&self, mut spec: AgentSpec) -> Result<ManagedAgent, SupervisorError> {
validate_id(&spec.id)?;
validate_command(&spec.command)?;
{
let mut state = self.state.write().await;
if spec.token.is_empty() {
if let Some(existing) = state.get(&spec.id) {
if !existing.spec.token.is_empty() {
spec.token = existing.spec.token.clone();
}
}
if spec.token.is_empty() {
spec.token = mint_agent_token();
}
}
if let Some(existing) = state.get_mut(&spec.id) {
existing.spec = spec.clone();
} else {
state.insert(
spec.id.clone(),
AgentSlot {
spec: spec.clone(),
runtime: AgentRuntime::default(),
stop_tx: None,
task: None,
job: None,
},
);
}
}
self.persist().await?;
Ok(ManagedAgent {
spec,
status: AgentStatus::Stopped,
pid: None,
last_exit_code: None,
restart_count: 0,
started_at: None,
blocked_by_pid: None,
})
}
pub async fn install_manifest(
&self,
manifest: AgentManifest,
host: &crate::install::HostCapabilities,
) -> Result<(crate::install::InstallCheckReport, Option<ManagedAgent>), SupervisorError> {
let report = crate::install::install_check(&manifest, host)?;
if manifest.is_pure_data() || manifest.is_remote_service() {
let agents_dir = self
.manifest_path
.parent()
.map(|p| p.join("agents"))
.unwrap_or_else(|| PathBuf::from("agents"));
std::fs::create_dir_all(&agents_dir)?;
crate::manifest::write_manifest(&agents_dir, &manifest)?;
return Ok((report, None));
}
let mut spec = crate::manifest::to_agent_spec(&manifest)?;
if spec.token.is_empty() {
spec.token = mint_agent_token();
}
let managed = self.upsert(spec).await?;
if managed.spec.auto_start {
let started = self.start(&managed.spec.id).await?;
Ok((report, Some(started)))
} else {
Ok((report, Some(managed)))
}
}
pub async fn agent_token(&self, id: &str) -> Option<String> {
let state = self.state.read().await;
let slot = state.get(id)?;
if slot.spec.token.is_empty() {
None
} else {
Some(slot.spec.token.clone())
}
}
pub async fn validate_agent_token(&self, id: &str, token: &str) -> bool {
let Some(stored) = self.agent_token(id).await else {
return false;
};
constant_time_eq(stored.as_bytes(), token.as_bytes())
}
pub async fn remove(&self, id: &str) -> Result<bool, SupervisorError> {
validate_id(id)?;
let _ = self.stop(id, StopSignal::Term).await;
let removed = {
let mut state = self.state.write().await;
state.remove(id).is_some()
};
if removed {
self.persist().await?;
}
Ok(removed)
}
pub async fn start(&self, id: &str) -> Result<ManagedAgent, SupervisorError> {
validate_id(id)?;
let spec = {
let state = self.state.read().await;
let slot = state
.get(id)
.ok_or_else(|| SupervisorError::NotFound(id.to_string()))?;
if matches!(
slot.runtime.status,
AgentStatus::Running | AgentStatus::Starting
) {
return Ok(self.snapshot_locked(slot));
}
slot.spec.clone()
};
self.spawn_supervision(spec).await;
tokio::task::yield_now().await;
Ok(self.get(id).await.unwrap_or_else(|| ManagedAgent {
spec: AgentSpec {
id: id.to_string(),
name: id.to_string(),
command: String::new(),
args: vec![],
cwd: None,
env: BTreeMap::new(),
restart: RestartPolicy::default(),
max_restarts: default_max_restarts(),
backoff_secs: default_backoff(),
auto_start: false,
token: String::new(),
capabilities: Vec::new(),
},
status: AgentStatus::Starting,
pid: None,
last_exit_code: None,
restart_count: 0,
started_at: None,
blocked_by_pid: None,
}))
}
pub async fn stop(
&self,
id: &str,
signal: StopSignal,
) -> Result<ManagedAgent, SupervisorError> {
{
let state = self.state.read().await;
if !state.contains_key(id) {
return Err(SupervisorError::NotFound(id.to_string()));
}
}
self.teardown_running(id, signal).await;
{
let mut state = self.state.write().await;
if let Some(slot) = state.get_mut(id) {
slot.runtime.status = AgentStatus::Stopped;
slot.runtime.pid = None;
slot.runtime.started_at = None;
}
}
self.get(id)
.await
.ok_or_else(|| SupervisorError::NotFound(id.to_string()))
}
pub async fn restart(&self, id: &str) -> Result<ManagedAgent, SupervisorError> {
let _ = self.stop(id, StopSignal::Term).await;
self.start(id).await
}
pub async fn sweep_orphan_logs(&self) -> usize {
let registered: std::collections::HashSet<String> =
{ self.state.read().await.keys().cloned().collect() };
let Ok(entries) = std::fs::read_dir(&self.log_dir) else {
return 0;
};
let mut removed = 0usize;
for entry in entries.flatten() {
let path = entry.path();
let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
continue;
};
let Some(id) = name
.strip_suffix(".stdout.log")
.or_else(|| name.strip_suffix(".stderr.log"))
else {
continue;
};
if registered.contains(id) {
continue;
}
match entry.metadata() {
Ok(meta) if meta.is_file() && meta.len() == 0 => {}
_ => continue,
}
if external_agent_pid(id).is_some() {
continue;
}
if std::fs::remove_file(&path).is_ok() {
removed += 1;
}
}
if removed > 0 {
tracing::info!(
removed,
dir = %self.log_dir.display(),
"swept empty log files belonging to no registered agent and no live process"
);
}
removed
}
pub async fn start_all(&self) -> Vec<String> {
self.sweep_orphan_logs().await;
let candidates: Vec<AgentSpec> = {
let state = self.state.read().await;
state
.values()
.filter(|slot| {
slot.spec.auto_start
&& !matches!(
slot.runtime.status,
AgentStatus::Running | AgentStatus::Starting
)
})
.map(|slot| slot.spec.clone())
.collect()
};
let mut started = Vec::with_capacity(candidates.len());
for spec in candidates {
if let Some(ext) = external_agent_pid(&spec.id) {
tracing::warn!(
agent = %spec.id,
pid = ext.pid,
pid_file = %ext.path.display(),
"agent already running externally. Skipping auto_start — call agents.start once the external instance exits to take over supervision.",
);
continue;
}
let id = spec.id.clone();
self.spawn_supervision(spec).await;
started.push(id);
}
started
}
pub async fn tail_log(&self, id: &str, n: usize) -> Result<Vec<String>, SupervisorError> {
let tail = self.read_log(id, LogStream::Combined, n, 0).await?;
Ok(tail.lines)
}
pub async fn read_log(
&self,
id: &str,
stream: LogStream,
n: usize,
offset: usize,
) -> Result<LogTail, SupervisorError> {
validate_id(id)?;
let stdout_path = self.log_dir.join(format!("{id}.stdout.log"));
let stderr_path = self.log_dir.join(format!("{id}.stderr.log"));
let want_stdout = matches!(stream, LogStream::Stdout | LogStream::Combined);
let want_stderr = matches!(stream, LogStream::Stderr | LogStream::Combined);
let stdout = if want_stdout {
read_stream_window(&stdout_path, n, offset).await?
} else {
StreamWindow::default()
};
let stderr = if want_stderr {
read_stream_window(&stderr_path, n, offset).await?
} else {
StreamWindow::default()
};
let mut lines = Vec::with_capacity(stdout.lines.len() + stderr.lines.len());
lines.extend(stdout.lines.iter().cloned());
lines.extend(stderr.lines.iter().cloned());
let more = stdout.more || stderr.more;
Ok(LogTail {
lines,
stdout: stdout.lines,
stderr: stderr.lines,
stdout_total: stdout.total,
stderr_total: stderr.total,
stdout_path: stdout_path.to_string_lossy().into_owned(),
stderr_path: stderr_path.to_string_lossy().into_owned(),
more,
})
}
fn snapshot_locked(&self, slot: &AgentSlot) -> ManagedAgent {
ManagedAgent {
spec: slot.spec.clone(),
status: slot.runtime.status,
pid: slot.runtime.pid,
last_exit_code: slot.runtime.last_exit_code,
restart_count: slot.runtime.restart_count,
started_at: slot.runtime.started_at,
blocked_by_pid: slot.runtime.blocked_by_pid,
}
}
async fn persist(&self) -> Result<(), SupervisorError> {
let _g = self.manifest_lock.lock().await;
let (manifest, current_ids): (Manifest, std::collections::HashSet<String>) = {
let state = self.state.read().await;
let mut agents: Vec<AgentSpec> = state.values().map(|slot| slot.spec.clone()).collect();
agents.sort_by(|a, b| a.id.cmp(&b.id));
let ids: std::collections::HashSet<String> =
agents.iter().map(|s| s.id.clone()).collect();
(
Manifest {
agents: agents.clone(),
},
ids,
)
};
write_json_atomic(&self.manifest_path, &manifest)?;
let agents_dir = self
.manifest_path
.parent()
.map(|p| p.join("agents"))
.unwrap_or_else(|| PathBuf::from("agents"));
if let Err(e) = std::fs::create_dir_all(&agents_dir) {
tracing::warn!(
dir = %agents_dir.display(),
error = %e,
"could not create agents/ dir for manifest mirror"
);
return Ok(());
}
for spec in &manifest.agents {
let m = crate::manifest::from_legacy_spec(spec);
if let Err(e) = crate::manifest::write_manifest(&agents_dir, &m) {
tracing::warn!(
id = %spec.id,
error = %e,
"mirroring AgentSpec to manifest.toml failed; legacy \
agents.json was still updated"
);
}
}
if let Ok(entries) = std::fs::read_dir(&agents_dir) {
for entry in entries.flatten() {
let p = entry.path();
if !p.is_dir() {
continue;
}
let Some(name) = p.file_name().and_then(|s| s.to_str()) else {
continue;
};
if current_ids.contains(name) {
continue;
}
if p.join("manifest.toml").is_file() {
if let Err(e) = std::fs::remove_dir_all(&p) {
tracing::warn!(
dir = %p.display(),
error = %e,
"reaping stale manifest dir failed"
);
}
}
}
}
Ok(())
}
async fn spawn_supervision(&self, spec: AgentSpec) {
self.teardown_running(&spec.id, StopSignal::Term).await;
let (tx, rx) = tokio::sync::watch::channel(false);
let task = tokio::spawn(supervisor_loop(self.clone(), spec.clone(), rx));
let mut state = self.state.write().await;
if let Some(slot) = state.get_mut(&spec.id) {
slot.runtime.status = AgentStatus::Starting;
slot.runtime.restart_count = 0;
slot.stop_tx = Some(tx);
slot.task = Some(task);
}
}
async fn teardown_running(&self, id: &str, signal: StopSignal) {
let (stop_tx, task, pid, job, started_at) = {
let mut state = self.state.write().await;
match state.get_mut(id) {
Some(slot) => (
slot.stop_tx.take(),
slot.task.take(),
slot.runtime.pid,
slot.job.take(),
slot.runtime.started_at,
),
None => return,
}
};
if let Some(tx) = stop_tx {
let _ = tx.send(true);
}
if let Some(pid) = pid {
kill_process(pid, signal, self.grace_secs, job, started_at).await;
}
if let Some(handle) = task {
handle.abort();
}
}
async fn set_blocked_by(&self, id: &str, pid: Option<i32>) {
let mut state = self.state.write().await;
if let Some(slot) = state.get_mut(id) {
slot.runtime.blocked_by_pid = pid;
}
}
async fn set_status(
&self,
id: &str,
status: AgentStatus,
pid: Option<u32>,
started_at: Option<i64>,
) {
let mut state = self.state.write().await;
if let Some(slot) = state.get_mut(id) {
if matches!(status, AgentStatus::Running) {
slot.runtime.last_exit_code = None;
}
slot.runtime.status = status;
slot.runtime.pid = pid;
slot.runtime.started_at = started_at;
slot.runtime.blocked_by_pid = None;
}
}
async fn store_job(&self, id: &str, job: Option<Arc<JobObject>>) {
let mut state = self.state.write().await;
if let Some(slot) = state.get_mut(id) {
slot.job = job;
}
}
async fn record_exit(&self, id: &str, exit_code: i32) {
let mut state = self.state.write().await;
if let Some(slot) = state.get_mut(id) {
slot.runtime.last_exit_code = Some(exit_code);
slot.runtime.pid = None;
slot.runtime.started_at = None;
}
}
async fn bump_restart(&self, id: &str) -> u32 {
let mut state = self.state.write().await;
if let Some(slot) = state.get_mut(id) {
slot.runtime.restart_count = slot.runtime.restart_count.saturating_add(1);
slot.runtime.status = AgentStatus::Backoff;
slot.runtime.restart_count
} else {
0
}
}
async fn mark_errored(&self, id: &str) {
let mut state = self.state.write().await;
if let Some(slot) = state.get_mut(id) {
slot.runtime.status = AgentStatus::Errored;
}
}
}
async fn supervisor_loop(
supervisor: Supervisor,
spec: AgentSpec,
mut stop_rx: tokio::sync::watch::Receiver<bool>,
) {
let id = spec.id.clone();
loop {
if *stop_rx.borrow() {
return;
}
if let Some(ext) = external_agent_pid(&spec.id) {
tracing::warn!(
agent = %id,
pid = ext.pid,
pid_file = %ext.path.display(),
"external agent instance still alive (pid file). Supervisor refusing to double-spawn; sleeping {}s then re-checking. Leave it and supervision resumes automatically once it exits. If you want it gone, check that pid {} really is this agent first (`ps -p <pid> -o command=`) — a pid file left by a killed agent can be recycled by an unrelated process, in which case remove the pid file rather than killing anything — then `car restart {}`.",
spec.backoff_secs.max(5),
ext.pid,
id
);
supervisor
.set_status(&id, AgentStatus::Backoff, None, None)
.await;
supervisor.set_blocked_by(&id, Some(ext.pid)).await;
let backoff = std::time::Duration::from_secs(spec.backoff_secs.max(5));
tokio::select! {
_ = stop_rx.changed() => return,
_ = tokio::time::sleep(backoff) => continue,
}
}
let default_env = supervisor.default_child_env().await;
match spawn_child(&supervisor.log_dir, &spec, &default_env).await {
Ok(SpawnedChild {
mut child,
pid,
job,
}) => {
let started_at = chrono::Utc::now().timestamp();
supervisor.store_job(&id, job).await;
write_supervisor_pid_file(&id, pid);
supervisor
.set_status(&id, AgentStatus::Running, Some(pid), Some(started_at))
.await;
tokio::select! {
biased;
_ = stop_rx.changed() => {
let _ = child.wait().await;
clear_supervisor_pid_file(&id, pid);
reap_empty_logs(supervisor.log_dir(), &id);
return;
}
res = child.wait() => {
let code = match res {
Ok(status) => status.code().unwrap_or(-1),
Err(_) => -1,
};
clear_supervisor_pid_file(&id, pid);
reap_empty_logs(supervisor.log_dir(), &id);
supervisor.record_exit(&id, code).await;
let should_restart = match spec.restart {
RestartPolicy::Never => false,
RestartPolicy::OnFailure => code != 0,
RestartPolicy::Always => true,
};
if !should_restart {
supervisor.set_status(&id, AgentStatus::Stopped, None, None).await;
return;
}
let count = supervisor.bump_restart(&id).await;
if count > spec.max_restarts {
tracing::warn!(agent = %id, count, max = spec.max_restarts,
"agent exceeded max_restarts; marking errored");
supervisor.mark_errored(&id).await;
return;
}
let backoff = restart_backoff(spec.backoff_secs, count);
tokio::select! {
_ = stop_rx.changed() => return,
_ = tokio::time::sleep(backoff) => {}
}
}
}
}
Err(e) => {
tracing::error!(agent = %id, error = %e, "spawn failed");
supervisor.record_exit(&id, -1).await;
let count = supervisor.bump_restart(&id).await;
if count > spec.max_restarts {
supervisor.mark_errored(&id).await;
return;
}
let backoff = restart_backoff(spec.backoff_secs, count);
tokio::select! {
_ = stop_rx.changed() => return,
_ = tokio::time::sleep(backoff) => {}
}
}
}
}
}
const BACKOFF_CAP_SECS: u64 = 60;
fn restart_backoff(base_secs: u64, attempt: u32) -> std::time::Duration {
let base = base_secs.max(1);
let shift = attempt.saturating_sub(1).min(16);
let grown = base.saturating_mul(1u64 << shift).min(BACKOFF_CAP_SECS);
let jitter_ceiling_ms = grown.saturating_mul(1000) / 8;
let jitter_ms = if jitter_ceiling_ms == 0 {
0
} else {
(jitter_nanos() % u128::from(jitter_ceiling_ms)) as u64
};
std::time::Duration::from_secs(grown) + std::time::Duration::from_millis(jitter_ms)
}
fn jitter_nanos() -> u128 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| u128::from(d.subsec_nanos()))
.unwrap_or(0)
}
struct SpawnedChild {
child: tokio::process::Child,
pid: u32,
job: Option<Arc<JobObject>>,
}
fn program_command(program: &str) -> tokio::process::Command {
#[cfg(windows)]
{
use std::path::Path;
let has_batch_ext = |p: &Path| {
p.extension()
.and_then(|e| e.to_str())
.map(|e| {
let e = e.to_ascii_lowercase();
e == "cmd" || e == "bat"
})
.unwrap_or(false)
};
let shim: Option<std::path::PathBuf> = {
let p = Path::new(program);
if p.extension().is_some() || p.components().count() > 1 {
has_batch_ext(p).then(|| p.to_path_buf())
} else if let Some(path) = std::env::var_os("PATH") {
let pathext =
std::env::var_os("PATHEXT").unwrap_or_else(|| ".COM;.EXE;.BAT;.CMD".into());
let exts: Vec<String> = pathext
.to_string_lossy()
.split(';')
.filter(|e| !e.is_empty())
.map(|e| e.to_string())
.collect();
let mut found = None;
'outer: for dir in std::env::split_paths(&path) {
for ext in &exts {
let cand = dir.join(format!("{program}{ext}"));
if cand.is_file() {
found = has_batch_ext(&cand).then_some(cand);
break 'outer;
}
}
}
found
} else {
None
}
};
if let Some(shim) = shim {
let mut c = tokio::process::Command::new("cmd");
c.arg("/C").arg(shim);
if let Some(path) = car_winenv::cmd_path_override() {
c.env("PATH", path);
}
return c;
}
if is_cmd_exe(program) {
let mut c = tokio::process::Command::new(program);
if let Some(path) = car_winenv::cmd_path_override() {
c.env("PATH", path);
}
return c;
}
}
tokio::process::Command::new(program)
}
#[cfg(windows)]
fn is_cmd_exe(program: &str) -> bool {
std::path::Path::new(program)
.file_name()
.and_then(|f| f.to_str())
.map(|f| f.eq_ignore_ascii_case("cmd.exe") || f.eq_ignore_ascii_case("cmd"))
.unwrap_or(false)
}
async fn spawn_child(
log_dir: &Path,
spec: &AgentSpec,
default_env: &BTreeMap<String, String>,
) -> std::io::Result<SpawnedChild> {
use std::process::Stdio;
let stdout_path = log_dir.join(format!("{}.stdout.log", spec.id));
let stderr_path = log_dir.join(format!("{}.stderr.log", spec.id));
let stdout = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&stdout_path)?;
let stderr = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&stderr_path)?;
let mut cmd = program_command(&spec.command);
cmd.args(&spec.args);
if let Some(cwd) = &spec.cwd {
cmd.current_dir(cwd);
}
for (k, v) in default_env {
cmd.env(k, v);
}
cmd.env("CAR_AGENT_ID", &spec.id);
if !spec.token.is_empty() {
cmd.env("CAR_AGENT_TOKEN", &spec.token);
}
for (k, v) in &spec.env {
cmd.env(k, v);
}
cmd.stdin(Stdio::null());
cmd.stdout(Stdio::from(stdout));
cmd.stderr(Stdio::from(stderr));
cmd.kill_on_drop(true);
#[cfg(unix)]
unsafe {
cmd.pre_exec(|| {
if libc_setsid() == -1 {
return Err(std::io::Error::last_os_error());
}
Ok(())
});
}
let child = cmd.spawn()?;
let pid = child
.id()
.ok_or_else(|| std::io::Error::other("child spawned without pid"))?;
#[cfg(target_os = "windows")]
let job = match JobObject::new() {
Ok(j) => match j.assign(pid) {
Ok(()) => Some(Arc::new(j)),
Err(e) => {
tracing::warn!(
agent = %spec.id,
pid,
error = ?e,
"Job Object created but process assignment failed; \
cascade-kill on stop will be disabled for this child"
);
None
}
},
Err(e) => {
tracing::warn!(
agent = %spec.id,
pid,
error = ?e,
"CreateJobObjectW failed; cascade-kill on stop will be \
disabled for this child"
);
None
}
};
#[cfg(not(target_os = "windows"))]
let job: Option<Arc<JobObject>> = None;
Ok(SpawnedChild { child, pid, job })
}
#[cfg(unix)]
fn libc_setsid() -> i32 {
extern "C" {
fn setsid() -> i32;
}
unsafe { setsid() }
}
async fn kill_process(
pid: u32,
signal: StopSignal,
grace_secs: u64,
#[cfg_attr(unix, allow(unused_variables))] job: Option<Arc<JobObject>>,
#[cfg_attr(unix, allow(unused_variables))] started_at_unix: Option<i64>,
) {
#[cfg(unix)]
{
let _ = job;
let pid_i = pid as i32;
match signal {
StopSignal::Term => {
signal_process_tree(pid_i, libc_sigterm());
let deadline = std::time::Duration::from_secs(grace_secs.max(1));
let mut waited = std::time::Duration::ZERO;
let step = std::time::Duration::from_millis(200);
while waited < deadline {
if !pid_alive(pid_i) {
return;
}
tokio::time::sleep(step).await;
waited += step;
}
signal_process_tree(pid_i, libc_sigkill());
}
StopSignal::Kill => {
signal_process_tree(pid_i, libc_sigkill());
}
}
}
#[cfg(target_os = "windows")]
{
let _ = (signal, grace_secs);
match job {
Some(j) => {
if let Err(e) = j.terminate(1) {
tracing::warn!(
pid,
error = ?e,
"TerminateJobObject failed; supervised child tree may be incomplete-killed"
);
}
}
None => {
tracing::warn!(
pid,
"Windows supervised process has no Job Object — \
using TerminateProcess fallback (no cascade kill; \
grandchildren may leak). See Parslee-ai/car#231 §5.1."
);
terminate_process_by_pid_verified(pid, started_at_unix);
}
}
}
}
#[cfg(target_os = "windows")]
fn terminate_process_by_pid_verified(pid: u32, expected_started_at_unix: Option<i64>) {
use windows::Win32::Foundation::{CloseHandle, FALSE, FILETIME};
use windows::Win32::System::Threading::{
GetProcessTimes, OpenProcess, TerminateProcess, PROCESS_QUERY_LIMITED_INFORMATION,
PROCESS_TERMINATE,
};
let expected = match expected_started_at_unix {
Some(t) => t,
None => {
tracing::warn!(
pid,
"fallback terminate skipped: no spawn timestamp in slot — \
cannot verify pid identity against possible reuse"
);
return;
}
};
unsafe {
let access = PROCESS_TERMINATE | PROCESS_QUERY_LIMITED_INFORMATION;
let handle = match OpenProcess(access, FALSE, pid) {
Ok(h) => h,
Err(e) => {
tracing::warn!(pid, ?e, "OpenProcess for fallback terminate failed");
return;
}
};
let mut creation = FILETIME::default();
let mut exit_ft = FILETIME::default();
let mut kernel = FILETIME::default();
let mut user = FILETIME::default();
let times_result =
GetProcessTimes(handle, &mut creation, &mut exit_ft, &mut kernel, &mut user);
if let Err(e) = times_result {
tracing::warn!(
pid,
?e,
"GetProcessTimes failed during pid-reuse verification — \
skipping terminate to avoid potentially killing the wrong process"
);
let _ = CloseHandle(handle);
return;
}
const TICKS_BETWEEN_EPOCHS: u64 = 116_444_736_000_000_000;
const TICKS_PER_SECOND: u64 = 10_000_000;
let creation_ticks =
((creation.dwHighDateTime as u64) << 32) | (creation.dwLowDateTime as u64);
let actual_unix = if creation_ticks >= TICKS_BETWEEN_EPOCHS {
((creation_ticks - TICKS_BETWEEN_EPOCHS) / TICKS_PER_SECOND) as i64
} else {
tracing::warn!(
pid,
creation_ticks,
"process creation time is pre-1970 — refusing to terminate"
);
let _ = CloseHandle(handle);
return;
};
let drift = (actual_unix - expected).abs();
if drift > 2 {
tracing::warn!(
pid,
expected,
actual_unix,
drift,
"pid reuse detected: process creation time differs from supervised spawn timestamp — \
refusing to terminate (the process now holding this pid is not the one we supervised)"
);
let _ = CloseHandle(handle);
return;
}
if let Err(e) = TerminateProcess(handle, 1) {
tracing::warn!(
pid,
?e,
"TerminateProcess failed (process may be protected by anti-malware \
or already exited)"
);
}
let _ = CloseHandle(handle);
}
}
#[cfg(target_os = "windows")]
pub struct JobObject {
handle: windows::Win32::Foundation::HANDLE,
}
#[cfg(target_os = "windows")]
unsafe impl Send for JobObject {}
#[cfg(target_os = "windows")]
unsafe impl Sync for JobObject {}
#[cfg(target_os = "windows")]
impl JobObject {
pub fn new() -> windows::core::Result<Self> {
use windows::Win32::System::JobObjects::{
CreateJobObjectW, JobObjectExtendedLimitInformation, SetInformationJobObject,
JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
};
unsafe {
let handle = CreateJobObjectW(None, windows::core::PCWSTR::null())?;
let job = Self { handle };
let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
let info_ptr = &info as *const _ as *const std::ffi::c_void;
let info_size = std::mem::size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>() as u32;
SetInformationJobObject(
job.handle,
JobObjectExtendedLimitInformation,
info_ptr,
info_size,
)?; Ok(job)
}
}
pub fn assign(&self, pid: u32) -> windows::core::Result<()> {
use windows::Win32::Foundation::{CloseHandle, FALSE};
use windows::Win32::System::JobObjects::AssignProcessToJobObject;
use windows::Win32::System::Threading::{
OpenProcess, PROCESS_SET_QUOTA, PROCESS_TERMINATE,
};
unsafe {
let process_handle = OpenProcess(PROCESS_SET_QUOTA | PROCESS_TERMINATE, FALSE, pid)?;
let result = AssignProcessToJobObject(self.handle, process_handle);
let _ = CloseHandle(process_handle);
result?;
Ok(())
}
}
pub fn terminate(&self, exit_code: u32) -> windows::core::Result<()> {
use windows::Win32::System::JobObjects::TerminateJobObject;
unsafe { TerminateJobObject(self.handle, exit_code) }
}
}
#[cfg(target_os = "windows")]
impl Drop for JobObject {
fn drop(&mut self) {
use windows::Win32::Foundation::CloseHandle;
if !self.handle.is_invalid() {
unsafe {
let _ = CloseHandle(self.handle);
}
}
}
}
#[cfg(not(target_os = "windows"))]
pub(crate) struct JobObject {
_private: (),
}
#[cfg(unix)]
fn libc_sigterm() -> i32 {
15
}
#[cfg(unix)]
fn libc_sigkill() -> i32 {
9
}
#[cfg(unix)]
fn send_signal(pid: i32, sig: i32) {
extern "C" {
fn kill(pid: i32, sig: i32) -> i32;
}
unsafe {
let _ = kill(pid, sig);
}
}
#[cfg(unix)]
fn process_group_of(pid: i32) -> i32 {
extern "C" {
fn getpgid(pid: i32) -> i32;
}
unsafe { getpgid(pid) }
}
#[cfg(unix)]
fn signal_process_tree(pid: i32, sig: i32) {
if pid > 1 && process_group_of(pid) == pid {
send_signal(-pid, sig);
} else {
send_signal(pid, sig);
}
}
#[cfg(unix)]
fn pid_alive(pid: i32) -> bool {
extern "C" {
fn kill(pid: i32, sig: i32) -> i32;
}
unsafe { kill(pid, 0) == 0 }
}
#[cfg(not(unix))]
fn pid_alive(_pid: i32) -> bool {
false
}
fn supervisor_pid_file(agent_id: &str) -> Option<std::path::PathBuf> {
Some(
car_home::root()?
.join("run")
.join(format!("{agent_id}.supervisor.pid")),
)
}
fn agent_owned_pid_file(agent_id: &str) -> Option<std::path::PathBuf> {
Some(
car_home::root()?
.join("run")
.join(format!("{agent_id}.pid")),
)
}
fn write_supervisor_pid_file(agent_id: &str, pid: u32) {
let Some(path) = supervisor_pid_file(agent_id) else {
return;
};
if let Some(dir) = path.parent() {
if let Err(e) = std::fs::create_dir_all(dir) {
tracing::warn!(
agent = %agent_id, path = %dir.display(), error = %e,
"could not create pid-file directory; double-spawn guard degraded for this agent"
);
return;
}
}
if let Err(e) = std::fs::write(&path, pid.to_string()) {
tracing::warn!(
agent = %agent_id, path = %path.display(), error = %e,
"could not write pid file; double-spawn guard degraded for this agent"
);
}
}
fn reap_empty_logs(log_dir: &Path, agent_id: &str) {
for stream in ["stdout", "stderr"] {
let path = log_dir.join(format!("{agent_id}.{stream}.log"));
match std::fs::metadata(&path) {
Ok(meta) if meta.len() == 0 => {
let _ = std::fs::remove_file(&path);
}
_ => {}
}
}
}
fn clear_supervisor_pid_file(agent_id: &str, pid: u32) {
let Some(path) = supervisor_pid_file(agent_id) else {
return;
};
match std::fs::read_to_string(&path) {
Ok(content) if content.trim() == pid.to_string() => {
let _ = std::fs::remove_file(&path);
}
_ => {}
}
}
#[derive(Debug, Clone)]
struct ExternalInstance {
pid: i32,
path: std::path::PathBuf,
}
fn read_live_pid(agent_id: &str, path: &Path, reap: bool) -> Option<i32> {
let content = match std::fs::read_to_string(path) {
Ok(s) => s,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return None,
Err(e) => {
tracing::warn!(
agent = %agent_id,
path = %path.display(),
error = %e,
"reading agent pid file failed; assuming no external instance"
);
return None;
}
};
let pid: i32 = match content.trim().parse() {
Ok(n) => n,
Err(_) => {
tracing::warn!(
agent = %agent_id,
path = %path.display(),
content = %content.trim(),
reaped = reap,
"agent pid file content unparseable"
);
if reap {
let _ = std::fs::remove_file(path);
}
return None;
}
};
if pid_alive(pid) {
Some(pid)
} else {
tracing::info!(
agent = %agent_id,
pid,
path = %path.display(),
reaped = reap,
"stale agent pid file (process gone)"
);
if reap {
let _ = std::fs::remove_file(path);
}
None
}
}
fn external_agent_pid(agent_id: &str) -> Option<ExternalInstance> {
if let Some(path) = supervisor_pid_file(agent_id) {
if let Some(pid) = read_live_pid(agent_id, &path, true) {
return Some(ExternalInstance { pid, path });
}
}
let path = agent_owned_pid_file(agent_id)?;
let pid = read_live_pid(agent_id, &path, false)?;
Some(ExternalInstance { pid, path })
}
fn validate_id(id: &str) -> Result<(), SupervisorError> {
if id.is_empty() {
return Err(SupervisorError::InvalidId(id.to_string()));
}
if !id
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.')
{
return Err(SupervisorError::InvalidId(id.to_string()));
}
if id == "." || id == ".." {
return Err(SupervisorError::InvalidId(id.to_string()));
}
Ok(())
}
const LOG_TAIL_BYTE_CEILING: u64 = 8 * 1024 * 1024;
async fn read_stream_window(
path: &Path,
n: usize,
offset: usize,
) -> Result<StreamWindow, SupervisorError> {
use tokio::io::{AsyncReadExt, AsyncSeekExt};
let mut file = match tokio::fs::File::open(path).await {
Ok(f) => f,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
return Ok(StreamWindow::default());
}
Err(e) => return Err(e.into()),
};
let file_len = file.metadata().await?.len();
if file_len == 0 {
return Ok(StreamWindow::default());
}
const CHUNK: u64 = 64 * 1024;
let read_cap = file_len.min(LOG_TAIL_BYTE_CEILING);
let mut buf: Vec<u8> = Vec::with_capacity(read_cap as usize);
let mut pos = file_len; let mut bytes_read: u64 = 0;
let mut hit_ceiling = false;
loop {
if pos == 0 {
break; }
if bytes_read >= read_cap {
hit_ceiling = true;
break;
}
let this_chunk = CHUNK.min(pos).min(read_cap - bytes_read);
let chunk_start = pos - this_chunk;
file.seek(std::io::SeekFrom::Start(chunk_start)).await?;
let mut chunk = vec![0u8; this_chunk as usize];
file.read_exact(&mut chunk).await?;
chunk.extend_from_slice(&buf);
buf = chunk;
pos = chunk_start;
bytes_read += this_chunk;
}
let reached_start = pos == 0;
let truncated = !reached_start;
if truncated && hit_ceiling {
tracing::warn!(
path = %path.display(),
file_len,
bytes_read,
dropped = file_len.saturating_sub(bytes_read),
"log tail hit the {LOG_TAIL_BYTE_CEILING}-byte ceiling; older lines \
were not scanned — use the on-disk file for full history"
);
}
let text = String::from_utf8_lossy(&buf);
let mut all: Vec<&str> = text.lines().collect();
if truncated && !all.is_empty() {
all.remove(0);
}
let scanned = all.len();
let total = scanned;
let end = scanned.saturating_sub(offset);
let start = if n == 0 { 0 } else { end.saturating_sub(n) };
let window: Vec<String> = all[start..end].iter().map(|s| s.to_string()).collect();
let more = start > 0 || truncated;
Ok(StreamWindow {
lines: window,
total,
more,
})
}
#[cfg(not(windows))]
const SCRATCH_REASON: &str = "command lives under a world-writable scratch directory \
(/tmp, /private/tmp, /var/tmp, /dev/shm)";
#[cfg(windows)]
const SCRATCH_REASON: &str = "command lives under a world-writable scratch directory \
(%TEMP%, %SystemRoot%\\Temp, %SystemDrive%\\Users\\Public)";
#[cfg(windows)]
fn windows_scratch_prefixes() -> Vec<String> {
fn norm(s: &str) -> String {
format!("{}\\", s.replace('/', "\\").trim_end_matches('\\')).to_ascii_lowercase()
}
let mut v = vec![norm(&std::env::temp_dir().to_string_lossy())];
for var in ["TEMP", "TMP"] {
if let Some(t) = std::env::var_os(var) {
v.push(norm(&Path::new(&t).to_string_lossy()));
}
}
let sysroot = std::env::var_os("SystemRoot")
.map(|s| Path::new(&s).to_string_lossy().into_owned())
.unwrap_or_else(|| r"C:\Windows".to_string());
v.push(norm(&format!("{}\\Temp", sysroot.trim_end_matches('\\'))));
let drive = std::env::var_os("SystemDrive")
.map(|s| Path::new(&s).to_string_lossy().into_owned())
.unwrap_or_else(|| "C:".to_string());
v.push(norm(&format!(
"{}\\Users\\Public",
drive.trim_end_matches('\\')
)));
v
}
#[cfg(not(windows))]
fn command_under_world_writable_scratch(command: &str) -> bool {
const SCRATCH_PREFIXES: &[&str] = &["/tmp/", "/private/tmp/", "/var/tmp/", "/dev/shm/"];
SCRATCH_PREFIXES.iter().any(|p| command.starts_with(p))
}
#[cfg(windows)]
fn command_under_world_writable_scratch(command: &str) -> bool {
let cand = command.replace('/', "\\").to_ascii_lowercase();
windows_scratch_prefixes()
.iter()
.any(|p| cand.starts_with(p.as_str()))
}
pub fn validate_command(command: &str) -> Result<(), SupervisorError> {
if command.is_empty() {
return Err(SupervisorError::InvalidCommand {
command: command.to_string(),
reason: "command is empty",
});
}
let path = Path::new(command);
if !path.is_absolute() {
return Err(SupervisorError::InvalidCommand {
command: command.to_string(),
reason: "command must be an absolute path; PATH lookup is not allowed",
});
}
if path
.components()
.any(|c| matches!(c, std::path::Component::ParentDir))
{
return Err(SupervisorError::InvalidCommand {
command: command.to_string(),
reason: "command path must not contain `..` segments",
});
}
if command_under_world_writable_scratch(command) {
return Err(SupervisorError::InvalidCommand {
command: command.to_string(),
reason: SCRATCH_REASON,
});
}
let meta = std::fs::metadata(path).map_err(|_| SupervisorError::InvalidCommand {
command: command.to_string(),
reason: "command file does not exist or is not readable",
})?;
if !meta.is_file() {
return Err(SupervisorError::InvalidCommand {
command: command.to_string(),
reason: "command path is not a regular file",
});
}
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
if meta.permissions().mode() & 0o111 == 0 {
return Err(SupervisorError::InvalidCommand {
command: command.to_string(),
reason: "command file has no execute bit set",
});
}
}
Ok(())
}
fn mint_agent_token() -> String {
use base64::Engine as _;
let a = uuid::Uuid::new_v4();
let b = uuid::Uuid::new_v4();
let mut bytes = [0u8; 32];
bytes[..16].copy_from_slice(a.as_bytes());
bytes[16..].copy_from_slice(b.as_bytes());
base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(bytes)
}
fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
if a.len() != b.len() {
return false;
}
let mut diff: u8 = 0;
for (x, y) in a.iter().zip(b.iter()) {
diff |= x ^ y;
}
diff == 0
}
pub fn resolve_interpreter(name: &str) -> Result<PathBuf, SupervisorError> {
if name.is_empty() {
return Err(SupervisorError::InvalidCommand {
command: name.to_string(),
reason: "interpreter name is empty",
});
}
if name.contains('/') || name.contains('\\') {
return Err(SupervisorError::InvalidCommand {
command: name.to_string(),
reason: "interpreter name must be a bare program name, not a path; \
pass paths via `command`",
});
}
if name == "." || name == ".." || name.contains("..") {
return Err(SupervisorError::InvalidCommand {
command: name.to_string(),
reason: "interpreter name must not contain `..` segments",
});
}
let path_var = std::env::var_os("PATH").ok_or(SupervisorError::InvalidCommand {
command: name.to_string(),
reason: "no $PATH set; cannot resolve interpreter",
})?;
for dir in std::env::split_paths(&path_var) {
if dir.as_os_str().is_empty() {
continue;
}
#[cfg_attr(not(windows), allow(unused_mut))]
let mut candidates = vec![dir.join(name)];
#[cfg(windows)]
if std::path::Path::new(name).extension().is_none() {
let pathext =
std::env::var("PATHEXT").unwrap_or_else(|_| ".COM;.EXE;.BAT;.CMD".to_string());
for ext in pathext.split(';').filter(|e| !e.is_empty()) {
candidates.push(dir.join(format!("{name}{ext}"))); }
}
for candidate in candidates {
if std::fs::metadata(&candidate).is_ok() {
let abs = candidate.to_string_lossy().into_owned();
validate_command(&abs)?;
return Ok(candidate);
}
}
}
Err(SupervisorError::InvalidCommand {
command: name.to_string(),
reason: "interpreter not found on $PATH",
})
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AgentHealth {
pub id: String,
pub command: String,
pub ok: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub reason: Option<String>,
}
fn load_manifest(path: &Path) -> Result<Manifest, SupervisorError> {
if !path.exists() {
return Ok(Manifest::default());
}
let bytes = std::fs::read(path)?;
let manifest: Manifest = serde_json::from_slice(&bytes)?;
Ok(manifest)
}
fn write_json_atomic<T: Serialize>(path: &Path, value: &T) -> Result<(), SupervisorError> {
let parent = path.parent().ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"manifest path has no parent",
)
})?;
std::fs::create_dir_all(parent)?;
let tmp = parent.join(format!(
".{}.tmp",
path.file_name()
.and_then(|s| s.to_str())
.unwrap_or("supervisor-write")
));
let json = serde_json::to_vec_pretty(value)?;
std::fs::write(&tmp, json)?;
std::fs::rename(&tmp, path)?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn temp_supervisor() -> (tempfile::TempDir, Supervisor) {
let target = std::env::var_os("CARGO_TARGET_DIR")
.map(std::path::PathBuf::from)
.unwrap_or_else(|| {
std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("..")
.join("..")
.join("target")
});
std::fs::create_dir_all(&target).ok();
let target = std::fs::canonicalize(&target).unwrap_or(target);
let tmp = tempfile::TempDir::new_in(&target).unwrap();
let s = Supervisor::with_paths(tmp.path().join("agents.json"), tmp.path().join("logs"))
.unwrap();
(tmp, s)
}
fn echo_spec(id: &str, message: &str) -> AgentSpec {
#[cfg(windows)]
let (command, args) = (
std::env::var("COMSPEC").unwrap_or_else(|_| r"C:\Windows\System32\cmd.exe".to_string()),
vec![
"/C".into(),
format!("echo {message}& ping -n 31 127.0.0.1 >nul"),
],
);
#[cfg(unix)]
let (command, args) = (
"/bin/sh".to_string(),
vec!["-c".into(), format!("echo {message}; sleep 30")],
);
AgentSpec {
id: id.into(),
name: id.into(),
command,
args,
cwd: None,
env: Default::default(),
restart: RestartPolicy::Never,
max_restarts: 1,
backoff_secs: 1,
auto_start: false,
token: String::new(),
capabilities: Vec::new(),
}
}
#[tokio::test]
async fn upsert_persists_and_lists() {
let (_tmp, s) = temp_supervisor();
s.upsert(echo_spec("alpha", "hi")).await.unwrap();
let list = s.list().await;
assert_eq!(list.len(), 1);
assert_eq!(list[0].spec.id, "alpha");
assert!(s.manifest_path().exists());
}
#[tokio::test]
async fn manifest_round_trips_across_supervisors() {
let (tmp, s) = temp_supervisor();
s.upsert(echo_spec("a", "x")).await.unwrap();
s.upsert(echo_spec("b", "y")).await.unwrap();
let list = Supervisor::list_from_manifest(&tmp.path().join("agents.json")).unwrap();
assert_eq!(list.len(), 2);
assert_eq!(list[0].spec.id, "a");
assert_eq!(list[1].spec.id, "b");
}
#[tokio::test]
async fn start_then_stop_runs_child_and_reaps_it() {
let (_tmp, s) = temp_supervisor();
s.upsert(echo_spec("runme", "hello")).await.unwrap();
s.start("runme").await.unwrap();
for _ in 0..50 {
let snap = s.list().await;
if matches!(snap[0].status, AgentStatus::Running) && snap[0].pid.is_some() {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
let snap = s.list().await;
assert!(matches!(snap[0].status, AgentStatus::Running), "{snap:?}");
assert!(snap[0].pid.is_some());
let pid = snap[0].pid.unwrap() as i32;
s.stop("runme", StopSignal::Term).await.unwrap();
for _ in 0..50 {
if !pid_alive(pid) {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
assert!(!pid_alive(pid), "child must be reaped after stop");
let after = s.list().await;
assert!(matches!(after[0].status, AgentStatus::Stopped));
assert!(after[0].pid.is_none());
}
#[tokio::test]
async fn wait_for_blocks_until_running_and_times_out_otherwise() {
let (_tmp, s) = temp_supervisor();
s.upsert(echo_spec("w", "hi")).await.unwrap();
let err = s
.wait_for(
"w",
&[AgentStatus::Running],
std::time::Duration::from_millis(150),
std::time::Duration::from_millis(20),
)
.await
.unwrap_err();
assert!(
matches!(err, SupervisorError::WaitTimeout { .. }),
"{err:?}"
);
assert!(matches!(
s.wait_for(
"ghost",
&[AgentStatus::Running],
std::time::Duration::from_millis(50),
std::time::Duration::from_millis(20),
)
.await,
Err(SupervisorError::NotFound(_))
));
s.start("w").await.unwrap();
let agent = s
.wait_for(
"w",
&[AgentStatus::Running],
std::time::Duration::from_secs(5),
std::time::Duration::from_millis(25),
)
.await
.unwrap();
assert_eq!(agent.status, AgentStatus::Running);
assert!(agent.pid.is_some());
s.stop("w", StopSignal::Term).await.unwrap();
}
#[cfg(not(target_os = "windows"))]
#[tokio::test]
async fn crash_loop_restarts_then_errors_with_exit_code() {
let (_tmp, s) = temp_supervisor();
let spec = AgentSpec {
id: "crasher".into(),
name: "crasher".into(),
command: "/bin/sh".into(),
args: vec!["-c".into(), "exit 1".into()],
cwd: None,
env: Default::default(),
restart: RestartPolicy::Always,
max_restarts: 2,
backoff_secs: 0,
auto_start: false,
token: String::new(),
capabilities: Vec::new(),
};
s.upsert(spec).await.unwrap();
s.start("crasher").await.unwrap();
let mut peak_restart_count = 0u32;
let mut terminal = None;
for _ in 0..600 {
let snap = s.list().await;
peak_restart_count = peak_restart_count.max(snap[0].restart_count);
if matches!(snap[0].status, AgentStatus::Errored) {
terminal = Some(snap[0].clone());
break;
}
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
}
let terminal = terminal.expect("agent never reached terminal Errored state");
assert!(
matches!(terminal.status, AgentStatus::Errored),
"expected terminal Errored, got {terminal:?}"
);
assert!(
peak_restart_count >= 2,
"restart_count should climb to at least max_restarts (2), peaked at {peak_restart_count}"
);
assert_eq!(
terminal.last_exit_code,
Some(1),
"last_exit_code should reflect the child's `exit 1`"
);
}
#[cfg(target_os = "windows")]
#[tokio::test]
async fn stop_cascades_to_grandchildren_on_windows() {
use std::process::Command as StdCommand;
let (_tmp, s) = temp_supervisor();
let spec = AgentSpec {
id: "tree".into(),
name: "tree".into(),
command: std::env::var("COMSPEC")
.unwrap_or_else(|_| r"C:\Windows\System32\cmd.exe".to_string()),
args: vec!["/C".into(), "ping".into(), "-t".into(), "127.0.0.1".into()],
cwd: None,
env: Default::default(),
restart: RestartPolicy::Never,
max_restarts: 1,
backoff_secs: 1,
auto_start: false,
token: String::new(),
capabilities: Vec::new(),
};
s.upsert(spec).await.unwrap();
s.start("tree").await.unwrap();
let mut parent_pid: Option<u32> = None;
for _ in 0..100 {
let snap = s.list().await;
if matches!(snap[0].status, AgentStatus::Running) {
if let Some(pid) = snap[0].pid {
parent_pid = Some(pid);
break;
}
}
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
}
let parent_pid = parent_pid.expect("supervisor never reported Running pid for cmd.exe");
let mut grandchild_pid: Option<u32> = None;
for _ in 0..50 {
let out = StdCommand::new("powershell")
.args([
"-NoProfile",
"-Command",
&format!(
"Get-CimInstance Win32_Process \
-Filter 'ParentProcessId={parent_pid} AND Name=\"ping.exe\"' | \
Select-Object -ExpandProperty ProcessId"
),
])
.output();
if let Ok(out) = out {
let text = String::from_utf8_lossy(&out.stdout);
for line in text.lines() {
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
if let Ok(pid) = trimmed.parse::<u32>() {
grandchild_pid = Some(pid);
break;
}
}
if grandchild_pid.is_some() {
break;
}
}
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
}
let grandchild_pid = grandchild_pid
.expect("never observed ping.exe grandchild under cmd.exe parent — supervisor or PowerShell/CIM bug");
s.stop("tree", StopSignal::Term).await.unwrap();
let pid_alive_win = |pid: u32| -> bool {
StdCommand::new("tasklist")
.args(["/FI", &format!("PID eq {pid}")])
.output()
.map(|o| {
let text = String::from_utf8_lossy(&o.stdout);
!text.contains("No tasks are running")
})
.unwrap_or(false)
};
let mut parent_gone = false;
let mut grandchild_gone = false;
for _ in 0..50 {
parent_gone = parent_gone || !pid_alive_win(parent_pid);
grandchild_gone = grandchild_gone || !pid_alive_win(grandchild_pid);
if parent_gone && grandchild_gone {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
}
assert!(
parent_gone,
"cmd.exe (parent pid {parent_pid}) still alive after stop"
);
assert!(
grandchild_gone,
"ping.exe (grandchild pid {grandchild_pid}) still alive after stop — \
§5.1 zombie-leak regression"
);
}
#[cfg(unix)]
fn test_pid_alive(pid: i32) -> bool {
pid_alive(pid)
}
#[cfg(windows)]
fn test_pid_alive(pid: i32) -> bool {
std::process::Command::new("tasklist")
.args(["/FI", &format!("PID eq {pid}"), "/NH", "/FO", "CSV"])
.output()
.map(|o| String::from_utf8_lossy(&o.stdout).contains(&format!("\"{pid}\"")))
.unwrap_or(false)
}
#[tokio::test]
async fn respawn_tears_down_prior_child_no_orphan() {
let (_tmp, s) = temp_supervisor();
s.upsert(echo_spec("solo", "hi")).await.unwrap();
s.start("solo").await.unwrap();
let mut pid1 = None;
for _ in 0..100 {
let snap = s.list().await;
if matches!(snap[0].status, AgentStatus::Running) {
if let Some(p) = snap[0].pid {
pid1 = Some(p as i32);
break;
}
}
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
let pid1 = pid1.expect("first child should reach Running");
let spec = s.list().await.into_iter().next().unwrap().spec;
s.spawn_supervision(spec).await;
let mut pid2 = None;
for _ in 0..100 {
let snap = s.list().await;
if matches!(snap[0].status, AgentStatus::Running) {
if let Some(p) = snap[0].pid {
if p as i32 != pid1 {
pid2 = Some(p as i32);
break;
}
}
}
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
let pid2 = pid2.expect("second child should reach Running with a fresh pid");
assert_ne!(pid1, pid2, "respawn should produce a distinct child");
for _ in 0..100 {
if !test_pid_alive(pid1) {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
assert!(
!test_pid_alive(pid1),
"prior child must be reaped, not orphaned"
);
assert!(test_pid_alive(pid2), "new child should still be alive");
s.stop("solo", StopSignal::Kill).await.unwrap();
}
#[test]
fn restart_backoff_is_exponential_capped_and_floored() {
assert!(restart_backoff(5, 1).as_secs() >= 5);
assert!(restart_backoff(5, 3).as_secs() >= 20);
let deep = restart_backoff(5, 50).as_secs();
assert!(deep >= BACKOFF_CAP_SECS, "deep backoff {deep}s below cap");
assert!(
deep <= BACKOFF_CAP_SECS + BACKOFF_CAP_SECS / 8 + 1,
"deep backoff {deep}s ignored cap"
);
let _ = restart_backoff(5, u32::MAX);
assert!(restart_backoff(0, 1).as_secs() >= 1);
}
#[tokio::test]
async fn tail_log_returns_recent_lines() {
let (_tmp, s) = temp_supervisor();
let mut spec = echo_spec("logs", "line-from-child");
#[cfg(unix)]
{
spec.args = vec!["-c".into(), "echo line-from-child".into()];
}
#[cfg(windows)]
{
spec.args = vec!["/C".into(), "echo line-from-child".into()];
}
s.upsert(spec).await.unwrap();
s.start("logs").await.unwrap();
for _ in 0..50 {
let lines = s.tail_log("logs", 10).await.unwrap();
if !lines.is_empty() {
assert!(lines.iter().any(|l| l.contains("line-from-child")));
return;
}
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
panic!("tail_log never observed the child's output");
}
#[tokio::test]
async fn sweep_removes_only_empty_unregistered_logs() {
let (_tmp, s) = temp_supervisor();
std::fs::create_dir_all(s.log_dir()).unwrap();
let touch = |id: &str, stream: &str, body: &str| {
std::fs::write(s.log_dir().join(format!("{id}.{stream}.log")), body).unwrap();
};
touch("vllm-mlx-4242", "stdout", "");
touch("vllm-mlx-4242", "stderr", "");
touch("vllm-mlx-9999", "stdout", "panicked at ...");
s.upsert(echo_spec("real-agent", "hi")).await.unwrap();
touch("real-agent", "stdout", "");
std::fs::write(s.log_dir().join("notes.txt"), "").unwrap();
let removed = s.sweep_orphan_logs().await;
assert_eq!(removed, 2, "both streams of the one inert orphan");
let gone = |p: &str| !s.log_dir().join(p).exists();
assert!(gone("vllm-mlx-4242.stdout.log"), "inert orphan must go");
assert!(gone("vllm-mlx-4242.stderr.log"), "inert orphan must go");
assert!(
s.log_dir().join("vllm-mlx-9999.stdout.log").exists(),
"a log with content is never removed, however dead its process"
);
assert!(
s.log_dir().join("real-agent.stdout.log").exists(),
"a registered agent owns its log even while empty"
);
assert!(
s.log_dir().join("notes.txt").exists(),
"the sweep must only ever consider *.stdout.log / *.stderr.log"
);
}
#[test]
fn reap_removes_empty_logs_and_keeps_written_ones() {
let (_tmp, s) = temp_supervisor();
std::fs::create_dir_all(s.log_dir()).unwrap();
std::fs::write(s.log_dir().join("gone.stdout.log"), "").unwrap();
std::fs::write(s.log_dir().join("gone.stderr.log"), "").unwrap();
reap_empty_logs(s.log_dir(), "gone");
assert!(!s.log_dir().join("gone.stdout.log").exists());
assert!(!s.log_dir().join("gone.stderr.log").exists());
std::fs::write(s.log_dir().join("mixed.stdout.log"), "started\n").unwrap();
std::fs::write(s.log_dir().join("mixed.stderr.log"), "").unwrap();
reap_empty_logs(s.log_dir(), "mixed");
assert!(
s.log_dir().join("mixed.stdout.log").exists(),
"content is never discarded"
);
assert!(!s.log_dir().join("mixed.stderr.log").exists());
reap_empty_logs(s.log_dir(), "never-existed");
}
fn write_logs(s: &Supervisor, id: &str, stdout: &str, stderr: &str) {
std::fs::write(s.log_dir().join(format!("{id}.stdout.log")), stdout).unwrap();
std::fs::write(s.log_dir().join(format!("{id}.stderr.log")), stderr).unwrap();
}
#[tokio::test]
async fn long_stderr_no_longer_buries_live_stdout() {
let (_tmp, s) = temp_supervisor();
let stdout: String = (0..10).map(|i| format!("live-stdout-{i}\n")).collect();
let stderr: String = (0..126).map(|i| format!("old-stderr-{i}\n")).collect();
write_logs(&s, "a", &stdout, &stderr);
let lines = s.tail_log("a", 100).await.unwrap();
for i in 0..10 {
assert!(
lines.iter().any(|l| l == &format!("live-stdout-{i}")),
"live stdout line {i} was buried"
);
}
}
#[tokio::test]
async fn read_log_stream_selection_and_paging() {
let (_tmp, s) = temp_supervisor();
let stdout: String = (0..50).map(|i| format!("out-{i}\n")).collect();
let stderr: String = (0..30).map(|i| format!("err-{i}\n")).collect();
write_logs(&s, "a", &stdout, &stderr);
let t = s.read_log("a", LogStream::Stdout, 10, 0).await.unwrap();
assert_eq!(t.stdout.len(), 10);
assert_eq!(t.stdout.first().unwrap(), "out-40");
assert_eq!(t.stdout.last().unwrap(), "out-49");
assert!(t.stderr.is_empty(), "stderr excluded when stream=stdout");
assert_eq!(t.stdout_total, 50);
assert!(t.more, "40 older stdout lines remain");
assert!(t.stdout_path.ends_with("a.stdout.log"));
let prev = s.read_log("a", LogStream::Stdout, 10, 10).await.unwrap();
assert_eq!(prev.stdout.first().unwrap(), "out-30");
assert_eq!(prev.stdout.last().unwrap(), "out-39");
let e = s.read_log("a", LogStream::Stderr, 5, 0).await.unwrap();
assert_eq!(e.stderr.last().unwrap(), "err-29");
assert!(e.stdout.is_empty());
assert_eq!(e.stderr_total, 30);
let full = s.read_log("a", LogStream::Stdout, 0, 0).await.unwrap();
assert_eq!(full.stdout.len(), 50);
assert!(!full.more);
}
#[tokio::test]
async fn read_log_missing_files_are_empty_not_error() {
let (_tmp, s) = temp_supervisor();
let t = s
.read_log("never-ran", LogStream::Combined, 100, 0)
.await
.unwrap();
assert!(t.lines.is_empty());
assert_eq!(t.stdout_total, 0);
assert_eq!(t.stderr_total, 0);
assert!(!t.more);
}
#[tokio::test]
async fn read_stream_window_bounded_tail_matches_whole_file_when_under_ceiling() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("x.log");
let content: String = (0..1000).map(|i| format!("line-{i}\n")).collect();
std::fs::write(&path, &content).unwrap();
let w = read_stream_window(&path, 10, 0).await.unwrap();
assert_eq!(w.total, 1000);
assert_eq!(w.lines.len(), 10);
assert_eq!(w.lines.first().unwrap(), "line-990");
assert_eq!(w.lines.last().unwrap(), "line-999");
assert!(w.more, "older lines remain");
let prev = read_stream_window(&path, 10, 10).await.unwrap();
assert_eq!(prev.lines.first().unwrap(), "line-980");
assert_eq!(prev.lines.last().unwrap(), "line-989");
assert!(prev.more);
let full = read_stream_window(&path, 0, 0).await.unwrap();
assert_eq!(full.lines.len(), 1000);
assert_eq!(full.total, 1000);
assert!(!full.more);
let top = read_stream_window(&path, 1000, 0).await.unwrap();
assert_eq!(top.lines.len(), 1000);
assert!(!top.more);
}
#[tokio::test]
async fn read_stream_window_only_reads_a_bounded_window_for_large_files() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("big.log");
let mut content = String::with_capacity(2_400_000);
for i in 0..100_000 {
content.push_str(&format!("entry-{i:06}\n"));
}
std::fs::write(&path, &content).unwrap();
let w = read_stream_window(&path, 5, 0).await.unwrap();
assert_eq!(w.lines.len(), 5);
assert_eq!(w.lines.last().unwrap(), "entry-099999");
assert_eq!(w.lines.first().unwrap(), "entry-099995");
assert_eq!(w.total, 100_000, "under ceiling ⇒ exact total");
assert!(w.more);
}
#[tokio::test]
async fn read_stream_window_truncates_honestly_past_the_byte_ceiling() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("huge.log");
let per_line = 30usize;
let line_count = (LOG_TAIL_BYTE_CEILING as usize / per_line) + 5_000;
let mut content = String::with_capacity(line_count * per_line);
for i in 0..line_count {
content.push_str(&format!("ln{i:027}\n"));
}
assert!(content.len() as u64 > LOG_TAIL_BYTE_CEILING);
std::fs::write(&path, &content).unwrap();
let w = read_stream_window(&path, 3, 0).await.unwrap();
assert_eq!(w.lines.len(), 3);
assert_eq!(
w.lines.last().unwrap(),
&format!("ln{:027}", line_count - 1)
);
assert!(
w.total < line_count,
"truncated read should report fewer than all {line_count} lines, got {}",
w.total
);
assert!(w.more, "truncated tail must force more=true");
}
#[tokio::test]
async fn remove_stops_running_agent() {
let (_tmp, s) = temp_supervisor();
s.upsert(echo_spec("ephemeral", "x")).await.unwrap();
s.start("ephemeral").await.unwrap();
for _ in 0..50 {
let snap = s.list().await;
if matches!(snap[0].status, AgentStatus::Running) {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
let removed = s.remove("ephemeral").await.unwrap();
assert!(removed);
assert!(s.list().await.is_empty());
}
#[tokio::test]
async fn invalid_ids_rejected() {
let (_tmp, s) = temp_supervisor();
let mut spec = echo_spec("ok", "x");
spec.id = "..".into();
assert!(s.upsert(spec).await.is_err());
}
#[tokio::test]
async fn start_all_skips_auto_start_false() {
let (_tmp, s) = temp_supervisor();
let mut a = echo_spec("auto", "x");
a.auto_start = true;
let mut b = echo_spec("manual", "y");
b.auto_start = false;
s.upsert(a).await.unwrap();
s.upsert(b).await.unwrap();
let started = s.start_all().await;
assert_eq!(started, vec!["auto".to_string()]);
}
#[test]
fn auto_start_defaults_to_false_when_omitted_from_json() {
let spec: AgentSpec =
serde_json::from_str(r#"{"id":"x","name":"X","command":"/bin/sh"}"#).unwrap();
assert!(!spec.auto_start, "default flipped 2026-05 — must be false");
}
#[tokio::test]
async fn validate_command_rejects_relative_path() {
let (_tmp, s) = temp_supervisor();
let mut spec = echo_spec("rel", "x");
spec.command = "sh".into();
let err = s.upsert(spec).await.unwrap_err();
assert!(
matches!(err, SupervisorError::InvalidCommand { .. }),
"expected InvalidCommand, got {err:?}"
);
}
#[tokio::test]
async fn validate_command_rejects_tmp_prefix() {
let bin = std::env::temp_dir().join("car-registry-validate-test.sh");
if !bin.starts_with("/tmp") && !bin.starts_with("/private/tmp") {
return;
}
std::fs::write(&bin, "#!/bin/sh\necho hi\n").unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
std::fs::set_permissions(&bin, std::fs::Permissions::from_mode(0o755)).unwrap();
}
let (_tmp, s) = temp_supervisor();
let mut spec = echo_spec("scratch", "x");
spec.command = bin.to_string_lossy().into_owned();
let err = s.upsert(spec).await.unwrap_err();
assert!(
matches!(err, SupervisorError::InvalidCommand { reason, .. }
if reason.contains("scratch")),
"expected scratch-dir rejection, got {err:?}"
);
let _ = std::fs::remove_file(&bin);
}
#[cfg(windows)]
#[tokio::test]
async fn validate_command_rejects_windows_temp_prefix() {
let bin = std::env::temp_dir().join("car-registry-validate-test.exe");
std::fs::write(&bin, b"MZ").unwrap();
let (_tmp, s) = temp_supervisor();
let mut spec = echo_spec("scratch-win", "x");
spec.command = bin.to_string_lossy().into_owned();
let err = s.upsert(spec).await.unwrap_err();
assert!(
matches!(err, SupervisorError::InvalidCommand { reason, .. }
if reason.contains("scratch")),
"expected scratch-dir rejection on Windows %TEMP%, got {err:?}"
);
let _ = std::fs::remove_file(&bin);
}
#[tokio::test]
async fn validate_command_rejects_missing_file() {
let (_tmp, s) = temp_supervisor();
let mut spec = echo_spec("ghost", "x");
spec.command = "/usr/local/bin/no-such-binary-please".into();
let err = s.upsert(spec).await.unwrap_err();
assert!(matches!(err, SupervisorError::InvalidCommand { .. }));
}
#[tokio::test]
async fn validate_command_rejects_directory() {
let (_tmp, s) = temp_supervisor();
let mut spec = echo_spec("dir", "x");
spec.command = if cfg!(windows) {
r"C:\Windows".into()
} else {
"/usr".into()
};
let err = s.upsert(spec).await.unwrap_err();
assert!(matches!(
err,
SupervisorError::InvalidCommand { reason, .. } if reason.contains("regular file")
));
}
#[tokio::test]
async fn validate_command_rejects_parent_dir_segment() {
let (_tmp, s) = temp_supervisor();
let mut spec = echo_spec("dotdot", "x");
spec.command = if cfg!(windows) {
r"C:\Windows\..\Windows\System32\cmd.exe".into()
} else {
"/usr/bin/../bin/sh".into()
};
let err = s.upsert(spec).await.unwrap_err();
assert!(matches!(
err,
SupervisorError::InvalidCommand { reason, .. } if reason.contains("..")
));
}
#[tokio::test]
async fn upsert_accepts_legitimate_command() {
let (_tmp, s) = temp_supervisor();
s.upsert(echo_spec("sane", "x")).await.unwrap();
}
#[cfg(unix)]
#[test]
fn resolve_interpreter_finds_sh_on_path() {
let resolved = resolve_interpreter("sh").unwrap();
assert!(resolved.is_absolute(), "got {:?}", resolved);
assert_eq!(resolved.file_name().unwrap(), "sh");
}
#[test]
fn resolve_interpreter_rejects_path_shaped_name() {
let err = resolve_interpreter("/bin/sh").unwrap_err();
assert!(matches!(
err,
SupervisorError::InvalidCommand { reason, .. }
if reason.contains("bare program name")
));
}
#[test]
fn resolve_interpreter_rejects_parent_dir_in_name() {
let err = resolve_interpreter("..").unwrap_err();
assert!(matches!(
err,
SupervisorError::InvalidCommand { reason, .. }
if reason.contains("..")
));
}
#[test]
fn resolve_interpreter_rejects_missing_name() {
let err = resolve_interpreter("no-such-interpreter-please-2026").unwrap_err();
assert!(matches!(
err,
SupervisorError::InvalidCommand { reason, .. }
if reason.contains("not found on $PATH")
));
}
#[tokio::test]
async fn upsert_mints_token_when_empty_and_retains_on_reupsert() {
let (_tmp, s) = temp_supervisor();
let agent = s.upsert(echo_spec("with-token", "x")).await.unwrap();
assert_eq!(agent.spec.token.len(), 43, "got {:?}", agent.spec.token);
assert!(agent
.spec
.token
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'));
let spec = echo_spec("with-token", "y");
assert!(spec.token.is_empty());
let again = s.upsert(spec).await.unwrap();
assert_eq!(again.spec.token, agent.spec.token);
let mut spec = echo_spec("with-token", "z");
spec.token = "rotated-explicitly-by-operator".into();
let rotated = s.upsert(spec).await.unwrap();
assert_eq!(rotated.spec.token, "rotated-explicitly-by-operator");
assert_eq!(
s.agent_token("with-token").await.as_deref(),
Some("rotated-explicitly-by-operator")
);
assert!(s.agent_token("nope").await.is_none());
}
#[tokio::test]
async fn validate_agent_token_uses_constant_time_compare() {
let (_tmp, s) = temp_supervisor();
let agent = s.upsert(echo_spec("auth-test", "x")).await.unwrap();
assert!(s.validate_agent_token("auth-test", &agent.spec.token).await);
assert!(!s.validate_agent_token("auth-test", "wrong").await);
assert!(!s.validate_agent_token("nope", &agent.spec.token).await);
}
#[tokio::test]
async fn default_child_env_is_set_and_round_trips() {
let (_tmp, s) = temp_supervisor();
assert!(s.default_child_env().await.is_empty());
s.set_default_child_env([
("CAR_DAEMON_URL", "ws://127.0.0.1:9100"),
("CAR_AUTH_TOKEN", "abc123"),
])
.await;
let got = s.default_child_env().await;
assert_eq!(got.len(), 2);
assert_eq!(
got.get("CAR_DAEMON_URL").map(String::as_str),
Some("ws://127.0.0.1:9100")
);
assert_eq!(
got.get("CAR_AUTH_TOKEN").map(String::as_str),
Some("abc123")
);
s.set_default_child_env([("CAR_DAEMON_URL", "ws://127.0.0.1:9200")])
.await;
let got = s.default_child_env().await;
assert_eq!(got.len(), 1);
assert_eq!(
got.get("CAR_DAEMON_URL").map(String::as_str),
Some("ws://127.0.0.1:9200")
);
}
#[tokio::test]
async fn health_flags_a_broken_command_after_upsert() {
let (tmp, s) = temp_supervisor();
let real = tmp.path().join("disposable.sh");
std::fs::write(&real, "#!/bin/sh\necho hi\n").unwrap();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut perm = std::fs::metadata(&real).unwrap().permissions();
perm.set_mode(0o755);
std::fs::set_permissions(&real, perm).unwrap();
}
let mut spec = echo_spec("vanish", "x");
spec.command = real.to_string_lossy().into_owned();
s.upsert(spec).await.unwrap();
let report = s.health().await;
let me = report.iter().find(|h| h.id == "vanish").unwrap();
assert!(me.ok, "expected fresh-upsert spec to be healthy");
std::fs::remove_file(&real).unwrap();
let report = s.health().await;
let me = report.iter().find(|h| h.id == "vanish").unwrap();
assert!(!me.ok, "expected health to flag missing command");
assert!(
me.reason
.as_deref()
.unwrap_or("")
.contains("does not exist"),
"got reason {:?}",
me.reason
);
}
fn write_legacy_agents_json(path: &Path, specs: &[AgentSpec]) {
let manifest = Manifest {
agents: specs.to_vec(),
};
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(path, serde_json::to_vec_pretty(&manifest).unwrap()).unwrap();
}
fn temp_tmpdir() -> tempfile::TempDir {
let target = std::env::var_os("CARGO_TARGET_DIR")
.map(std::path::PathBuf::from)
.unwrap_or_else(|| {
std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("..")
.join("..")
.join("target")
});
std::fs::create_dir_all(&target).ok();
let target = std::fs::canonicalize(&target).unwrap_or(target);
tempfile::TempDir::new_in(&target).unwrap()
}
fn reboot_with_paths(manifest: PathBuf, logs: PathBuf) -> Supervisor {
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
loop {
match Supervisor::with_paths(manifest.clone(), logs.clone()) {
Ok(s) => return s,
Err(SupervisorError::AlreadyRunning(_)) if std::time::Instant::now() < deadline => {
std::thread::sleep(std::time::Duration::from_millis(25));
}
Err(e) => panic!("reboot after drop failed: {e}"),
}
}
}
#[test]
fn boot_with_legacy_only_mirrors_to_new_layout() {
let tmp = temp_tmpdir();
let legacy = tmp.path().join("agents.json");
write_legacy_agents_json(
&legacy,
&[AgentSpec {
id: "legacy-ui".into(),
name: "Legacy UI".into(),
command: "/bin/sh".into(),
args: vec!["-c".into(), "true".into()],
cwd: None,
env: Default::default(),
restart: RestartPolicy::OnFailure,
max_restarts: 5,
backoff_secs: 2,
auto_start: false,
token: "tok-leg".into(),
capabilities: Vec::new(),
}],
);
let s = Supervisor::with_paths(legacy.clone(), tmp.path().join("logs")).unwrap();
let agents = futures::executor::block_on(s.list());
assert_eq!(agents.len(), 1);
assert_eq!(agents[0].spec.id, "legacy-ui");
let mirrored = tmp.path().join("agents/legacy-ui/manifest.toml");
assert!(
mirrored.exists(),
"expected mirrored manifest at {}",
mirrored.display()
);
let text = std::fs::read_to_string(&mirrored).unwrap();
let m: crate::manifest::AgentManifest = toml::from_str(&text).unwrap();
assert_eq!(m.agent.id, "legacy-ui");
if let crate::manifest::TransportSpec::ExternalProcess(t) = &m.transport {
assert_eq!(t.token, "tok-leg");
} else {
panic!("expected external_process transport, got {:?}", m.transport);
}
}
#[test]
fn boot_with_new_layout_only_loads_manifest_dir() {
let tmp = temp_tmpdir();
let agents_dir = tmp.path().join("agents");
std::fs::create_dir_all(&agents_dir).unwrap();
let m = crate::manifest::from_legacy_spec(&AgentSpec {
id: "new-only".into(),
name: "New Only".into(),
command: "/bin/sh".into(),
args: vec![],
cwd: None,
env: Default::default(),
restart: RestartPolicy::Never,
max_restarts: 1,
backoff_secs: 1,
auto_start: false,
token: "tok-new".into(),
capabilities: Vec::new(),
});
crate::manifest::write_manifest(&agents_dir, &m).unwrap();
let legacy = tmp.path().join("agents.json");
let s = Supervisor::with_paths(legacy, tmp.path().join("logs")).unwrap();
let agents = futures::executor::block_on(s.list());
assert_eq!(agents.len(), 1);
assert_eq!(agents[0].spec.id, "new-only");
assert_eq!(agents[0].spec.token, "tok-new");
}
#[test]
fn boot_with_mixed_sources_new_layout_wins_on_id_conflict() {
let tmp = temp_tmpdir();
let legacy = tmp.path().join("agents.json");
write_legacy_agents_json(
&legacy,
&[
AgentSpec {
id: "overlap".into(),
name: "Overlap (legacy)".into(),
command: "/bin/sh".into(),
args: vec!["-c".into(), "echo legacy".into()],
cwd: None,
env: Default::default(),
restart: RestartPolicy::Never,
max_restarts: 1,
backoff_secs: 1,
auto_start: false,
token: "legacy-token".into(),
capabilities: Vec::new(),
},
AgentSpec {
id: "legacy-only".into(),
name: "Legacy Only".into(),
command: "/bin/sh".into(),
args: vec![],
cwd: None,
env: Default::default(),
restart: RestartPolicy::Never,
max_restarts: 1,
backoff_secs: 1,
auto_start: false,
token: "leg-only-tok".into(),
capabilities: Vec::new(),
},
],
);
let agents_dir = tmp.path().join("agents");
std::fs::create_dir_all(&agents_dir).unwrap();
let new_overlap = crate::manifest::from_legacy_spec(&AgentSpec {
id: "overlap".into(),
name: "Overlap (new)".into(),
command: "/bin/sh".into(),
args: vec!["-c".into(), "echo new".into()],
cwd: None,
env: Default::default(),
restart: RestartPolicy::OnFailure,
max_restarts: 3,
backoff_secs: 2,
auto_start: false,
token: "new-token".into(),
capabilities: Vec::new(),
});
crate::manifest::write_manifest(&agents_dir, &new_overlap).unwrap();
let s = Supervisor::with_paths(legacy, tmp.path().join("logs")).unwrap();
let mut agents = futures::executor::block_on(s.list());
agents.sort_by(|a, b| a.spec.id.cmp(&b.spec.id));
assert_eq!(agents.len(), 2);
let overlap = agents.iter().find(|a| a.spec.id == "overlap").unwrap();
assert_eq!(overlap.spec.name, "Overlap (new)");
assert_eq!(overlap.spec.token, "new-token");
assert_eq!(overlap.spec.args, vec!["-c", "echo new"]);
let legacy_only = agents.iter().find(|a| a.spec.id == "legacy-only").unwrap();
assert_eq!(legacy_only.spec.token, "leg-only-tok");
}
#[test]
fn migration_is_idempotent_across_reboots() {
let tmp = temp_tmpdir();
let legacy = tmp.path().join("agents.json");
write_legacy_agents_json(
&legacy,
&[AgentSpec {
id: "iddy".into(),
name: "Iddy".into(),
command: "/bin/sh".into(),
args: vec![],
cwd: None,
env: Default::default(),
restart: RestartPolicy::Never,
max_restarts: 1,
backoff_secs: 1,
auto_start: false,
token: "tok-iddy".into(),
capabilities: Vec::new(),
}],
);
let s1 = Supervisor::with_paths(legacy.clone(), tmp.path().join("logs")).unwrap();
let mirrored = tmp.path().join("agents/iddy/manifest.toml");
let first_meta = std::fs::metadata(&mirrored).unwrap();
drop(s1);
let _s2 = reboot_with_paths(legacy, tmp.path().join("logs"));
let second_meta = std::fs::metadata(&mirrored).unwrap();
assert_eq!(
first_meta.modified().unwrap(),
second_meta.modified().unwrap()
);
}
#[test]
fn same_path_supervisor_rejects_second() {
let tmp = temp_tmpdir();
let manifest = tmp.path().join("agents.json");
let logs = tmp.path().join("logs");
let s1 = Supervisor::with_paths(manifest.clone(), logs.clone()).unwrap();
let lock_path = {
let mut s = manifest.as_os_str().to_owned();
s.push(".lock");
PathBuf::from(s)
};
match Supervisor::with_paths(manifest.clone(), logs.clone()) {
Err(SupervisorError::AlreadyRunning(p)) => assert_eq!(p, lock_path),
Err(e) => panic!("expected AlreadyRunning, got error: {e}"),
Ok(_) => panic!("expected AlreadyRunning, got Ok"),
}
drop(s1);
let _s3 = reboot_with_paths(manifest, logs);
}
#[tokio::test]
async fn list_from_manifest_works_while_lock_is_held() {
let (tmp, s) = temp_supervisor();
s.upsert(echo_spec("alpha", "a")).await.unwrap();
s.upsert(echo_spec("beta", "b")).await.unwrap();
let manifest = tmp.path().join("agents.json");
let agents = Supervisor::list_from_manifest(&manifest).unwrap();
assert_eq!(agents.len(), 2);
assert_eq!(agents[0].spec.id, "alpha");
assert_eq!(agents[1].spec.id, "beta");
assert_eq!(agents[0].pid, None);
assert_eq!(agents[0].status, AgentStatus::Stopped);
let health = Supervisor::health_from_manifest(&manifest).unwrap();
assert_eq!(health.len(), 2);
assert!(health.iter().all(|h| h.ok), "{health:?}");
}
#[tokio::test]
async fn upsert_writes_both_legacy_and_new_layout() {
let (tmp, s) = temp_supervisor();
s.upsert(echo_spec("dual", "x")).await.unwrap();
assert!(tmp.path().join("agents.json").exists());
let m_path = tmp.path().join("agents/dual/manifest.toml");
assert!(m_path.exists(), "expected mirror at {}", m_path.display());
}
#[tokio::test]
async fn install_manifest_rejects_when_host_lacks_required_capability() {
let (_tmp, s) = temp_supervisor();
let m = crate::manifest::from_legacy_spec(&AgentSpec {
id: "needs-magic".into(),
name: "Magic Agent".into(),
command: "/bin/sh".into(),
args: vec![],
cwd: None,
env: Default::default(),
restart: RestartPolicy::Never,
max_restarts: 1,
backoff_secs: 1,
auto_start: false,
token: String::new(),
capabilities: Vec::new(),
});
let mut m = m;
m.capabilities = Some(crate::manifest::CapabilityDeclarations {
required: std::collections::BTreeMap::from([(
"inference".into(),
vec!["text-generation".into()],
)]),
..Default::default()
});
let host = crate::install::HostCapabilities {
car_version: "0.8.0".into(),
..Default::default()
};
let err = s
.install_manifest(m, &host)
.await
.expect_err("missing cap must fail");
assert!(err.to_string().contains("inference.text-generation"));
assert!(s.list().await.is_empty());
}
#[tokio::test]
async fn install_manifest_adopts_external_process_when_validation_passes() {
let (_tmp, s) = temp_supervisor();
let m = crate::manifest::from_legacy_spec(&AgentSpec {
id: "installed-agent".into(),
name: "Installed".into(),
#[cfg(unix)]
command: "/bin/sh".into(),
#[cfg(windows)]
command: std::env::var("COMSPEC")
.unwrap_or_else(|_| r"C:\Windows\System32\cmd.exe".to_string()),
#[cfg(unix)]
args: vec!["-c".into(), "true".into()],
#[cfg(windows)]
args: vec!["/C".into(), "exit 0".into()],
cwd: None,
env: Default::default(),
restart: RestartPolicy::Never,
max_restarts: 1,
backoff_secs: 1,
auto_start: false,
token: String::new(),
capabilities: Vec::new(),
});
let host = crate::install::HostCapabilities {
car_version: "0.8.0".into(),
..Default::default()
};
let (report, managed) = s.install_manifest(m, &host).await.unwrap();
assert!(report.missing_optional.is_empty());
let managed = managed.expect("external_process manifest must adopt");
assert_eq!(managed.spec.id, "installed-agent");
assert!(!managed.spec.token.is_empty(), "token must be minted");
assert_eq!(managed.status, AgentStatus::Stopped);
assert_eq!(managed.pid, None);
assert_eq!(s.list().await.len(), 1);
}
#[tokio::test]
async fn install_manifest_auto_start_true_starts_external_process_immediately() {
let (_tmp, s) = temp_supervisor();
let m = crate::manifest::from_legacy_spec(&AgentSpec {
id: "auto-installed-agent".into(),
name: "Auto Installed".into(),
#[cfg(unix)]
command: "/bin/sh".into(),
#[cfg(windows)]
command: std::env::var("COMSPEC")
.unwrap_or_else(|_| r"C:\Windows\System32\cmd.exe".to_string()),
#[cfg(unix)]
args: vec!["-c".into(), "echo auto-installed; sleep 30".into()],
#[cfg(windows)]
args: vec![
"/C".into(),
"echo auto-installed& ping -n 31 127.0.0.1 >nul".into(),
],
cwd: None,
env: Default::default(),
restart: RestartPolicy::Never,
max_restarts: 1,
backoff_secs: 1,
auto_start: true,
token: String::new(),
capabilities: Vec::new(),
});
let host = crate::install::HostCapabilities {
car_version: "0.8.0".into(),
..Default::default()
};
let (report, managed) = s.install_manifest(m, &host).await.unwrap();
assert!(report.missing_optional.is_empty());
let managed = managed.expect("external_process manifest must adopt");
assert_eq!(managed.spec.id, "auto-installed-agent");
assert!(managed.spec.auto_start);
assert!(
matches!(managed.status, AgentStatus::Starting | AgentStatus::Running),
"install should return the post-start snapshot, got {:?}",
managed.status
);
for _ in 0..50 {
let list = s.list().await;
if matches!(list[0].status, AgentStatus::Running) {
let _ = s.stop("auto-installed-agent", StopSignal::Term).await;
return;
}
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
let latest = s.list().await;
let _ = s.stop("auto-installed-agent", StopSignal::Term).await;
panic!(
"auto-started install never reached running; latest status was {:?}",
latest[0].status
);
}
#[tokio::test]
async fn install_manifest_writes_pure_data_to_disk_without_adoption() {
let (tmp, s) = temp_supervisor();
let m = AgentManifest {
agent: crate::manifest::AgentIdentity {
id: "pure-bundle".into(),
name: "Pure Data".into(),
namespace: Some("parslee".into()),
version: Some("0.1.0".into()),
description: None,
license: None,
homepage: None,
},
publisher: None,
runtime: None,
lifecycle: None,
transport: crate::manifest::TransportSpec::PureData,
capabilities: None,
};
let host = crate::install::HostCapabilities {
car_version: "0.8.0".into(),
..Default::default()
};
let (_report, managed) = s.install_manifest(m, &host).await.unwrap();
assert!(
managed.is_none(),
"pure_data must NOT adopt into supervisor"
);
let m_path = tmp.path().join("agents/pure-bundle/manifest.toml");
assert!(m_path.exists());
assert!(s.list().await.is_empty());
}
#[tokio::test]
async fn remove_reaps_manifest_dir() {
let (tmp, s) = temp_supervisor();
s.upsert(echo_spec("reap", "x")).await.unwrap();
let agent_dir = tmp.path().join("agents/reap");
assert!(agent_dir.exists());
let removed = s.remove("reap").await.unwrap();
assert!(removed);
assert!(!agent_dir.exists(), "expected manifest dir to be reaped");
}
}
#[cfg(test)]
mod orphan_guard_tests {
use super::*;
static HOME_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
fn with_temp_home<T>(f: impl FnOnce() -> T) -> T {
let _guard = HOME_LOCK.lock().unwrap_or_else(|e| e.into_inner());
let tmp = tempfile::tempdir().expect("tempdir");
let prior = std::env::var_os("HOME");
let prior_car_home = std::env::var_os(car_home::ENV_VAR);
std::env::remove_var(car_home::ENV_VAR);
std::env::set_var("HOME", tmp.path());
let out = f();
match prior {
Some(p) => std::env::set_var("HOME", p),
None => std::env::remove_var("HOME"),
}
if let Some(p) = prior_car_home {
std::env::set_var(car_home::ENV_VAR, p);
}
out
}
#[test]
fn supervisor_written_pid_file_round_trips() {
with_temp_home(|| {
let me = std::process::id();
write_supervisor_pid_file("agent-a", me);
let path = supervisor_pid_file("agent-a").expect("path");
assert!(path.exists(), "supervisor must write the pid file");
#[cfg(unix)]
assert_eq!(
external_agent_pid("agent-a").map(|e| e.pid),
Some(me as i32),
"a live pid in the file must be reported as a blocker"
);
#[cfg(not(unix))]
assert_eq!(
external_agent_pid("agent-a").map(|e| e.pid),
None,
"off Unix the guard is a deliberate no-op (pid_alive is always false)"
);
});
}
#[test]
fn clearing_removes_only_our_own_pid() {
with_temp_home(|| {
let me = std::process::id();
write_supervisor_pid_file("agent-b", me);
let path = supervisor_pid_file("agent-b").expect("path");
std::fs::write(&path, (me + 1).to_string()).unwrap();
clear_supervisor_pid_file("agent-b", me);
assert!(path.exists(), "must not delete another writer's pid file");
std::fs::write(&path, me.to_string()).unwrap();
clear_supervisor_pid_file("agent-b", me);
assert!(!path.exists(), "must remove our own pid file on exit");
});
}
#[test]
fn a_dead_pid_is_not_a_blocker_and_the_stale_file_is_reaped() {
with_temp_home(|| {
let path = supervisor_pid_file("agent-c").expect("path");
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, "4294967000").unwrap();
assert!(external_agent_pid("agent-c").is_none());
});
}
#[test]
fn unparseable_pid_file_is_removed_rather_than_blocking_forever() {
with_temp_home(|| {
let path = supervisor_pid_file("agent-d").expect("path");
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(&path, "not-a-pid").unwrap();
assert!(external_agent_pid("agent-d").is_none());
assert!(
!path.exists(),
"garbage must not wedge the agent off forever"
);
});
}
#[test]
fn supervisor_never_writes_the_agent_owned_pid_path() {
with_temp_home(|| {
let me = std::process::id();
write_supervisor_pid_file("trader", me);
let ours = supervisor_pid_file("trader").expect("path");
let theirs = agent_owned_pid_file("trader").expect("path");
assert_ne!(ours, theirs, "the two records must not share a path");
assert!(
ours.exists(),
"the supervisor must still record the child it spawned"
);
assert!(
!theirs.exists(),
"car#931: writing the agent's own lock path makes a singleton \
agent race itself and refuse to start"
);
std::fs::create_dir_all(theirs.parent().unwrap()).unwrap();
std::fs::write(&theirs, me.to_string()).unwrap();
clear_supervisor_pid_file("trader", me);
assert!(!ours.exists(), "our own record is removed on exit");
assert!(
theirs.exists(),
"car#931: the agent's lock file is not ours to unlink"
);
});
}
#[cfg(unix)]
#[test]
fn an_agent_written_pid_file_still_blocks_a_double_spawn() {
with_temp_home(|| {
let me = std::process::id();
let theirs = agent_owned_pid_file("agent-e").expect("path");
std::fs::create_dir_all(theirs.parent().unwrap()).unwrap();
std::fs::write(&theirs, me.to_string()).unwrap();
let found = external_agent_pid("agent-e").expect("live external instance");
assert_eq!(found.pid, me as i32);
assert_eq!(
found.path, theirs,
"the diagnostic must name the agent's file, not CAR's"
);
});
}
#[test]
fn a_stale_agent_written_pid_file_is_ignored_but_left_in_place() {
with_temp_home(|| {
let theirs = agent_owned_pid_file("agent-f").expect("path");
std::fs::create_dir_all(theirs.parent().unwrap()).unwrap();
std::fs::write(&theirs, "4294967000").unwrap();
assert!(external_agent_pid("agent-f").is_none());
assert!(theirs.exists(), "the agent's file is not ours to reap");
std::fs::write(&theirs, "not-a-pid").unwrap();
assert!(external_agent_pid("agent-f").is_none());
assert!(
theirs.exists(),
"even garbage in the agent's file is not ours to delete"
);
});
}
#[cfg(unix)]
#[test]
fn our_own_record_is_consulted_before_the_agents() {
with_temp_home(|| {
let me = std::process::id();
write_supervisor_pid_file("agent-g", me);
let theirs = agent_owned_pid_file("agent-g").expect("path");
std::fs::write(&theirs, me.to_string()).unwrap();
let found = external_agent_pid("agent-g").expect("live instance");
assert_eq!(
found.path,
supervisor_pid_file("agent-g").expect("path"),
"CAR's own record is the authoritative one"
);
let ours = supervisor_pid_file("agent-g").expect("path");
std::fs::write(&ours, "4294967000").unwrap();
let found = external_agent_pid("agent-g").expect("live instance");
assert_eq!(found.path, theirs);
assert!(!ours.exists(), "our stale record is reaped");
});
}
#[cfg(unix)]
#[test]
fn tree_kill_only_targets_a_process_that_leads_its_own_group() {
let me = std::process::id() as i32;
let leads_own_group = process_group_of(me) == me;
if leads_own_group {
return;
}
signal_process_tree(me, 0);
assert!(
pid_alive(me),
"the interlock must never widen the blast radius to our own group"
);
}
#[cfg(unix)]
#[test]
fn process_group_lookup_reports_a_plausible_group() {
let me = std::process::id() as i32;
assert!(
process_group_of(me) > 0,
"getpgid on self must succeed; the interlock depends on it"
);
}
}