use std::collections::HashMap;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::process::Stdio;
use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
use std::sync::{Arc, Mutex as StdMutex};
use std::time::Duration;
use portable_pty::{CommandBuilder, PtySize, native_pty_system};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use tokio::io::{AsyncRead, AsyncWriteExt};
use tokio::process::{ChildStdin, Command};
use tokio::sync::Mutex;
use tokio_util::sync::CancellationToken;
mod lifecycle;
use crate::log_file::{finalize_redaction, read_tail, redaction_pending, require_redaction};
use crate::model::{
IoMode, ManagedRun, ProcessController, ProcessIdentity, ProcessInfo, RestartPolicy, RunSpec,
now_ms,
};
use crate::process::{identity_for_pid, parse_signal, process_info_for_pid};
use crate::store::Store;
use lifecycle::{LifecycleOutcome, LifecycleTracker};
static PTY_SPAWN_LOCK: StdMutex<()> = StdMutex::new(());
const CANCEL_NONE: u8 = 0;
const CANCEL_EXPLICIT: u8 = 1;
const CANCEL_TIMEOUT: u8 = 2;
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub struct RunLogs {
pub id: String,
pub stdout: String,
pub stderr: String,
pub stdout_truncated: bool,
pub stderr_truncated: bool,
pub tail_bytes: usize,
}
#[derive(Debug, Clone)]
pub(crate) struct LogSliceRequest {
pub pattern: Option<regex::Regex>,
pub context: usize,
pub max_matches: usize,
pub start: usize,
pub end: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub(crate) struct LogLine {
pub number: usize,
pub stream: String,
pub text: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub(crate) struct LogSlice {
pub id: String,
pub lines: Vec<LogLine>,
pub stdout_lines: usize,
pub stderr_lines: usize,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub total_matches: Option<usize>,
pub truncated: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub(crate) struct InputReceipt {
pub id: String,
pub recipients: usize,
pub bytes: usize,
}
#[derive(Clone)]
enum InputSink {
Pipes(Arc<Mutex<ChildStdin>>),
Pty(Arc<StdMutex<Box<dyn Write + Send>>>),
}
#[derive(Clone)]
struct ReplicaControl {
identity: ProcessIdentity,
input: Option<InputSink>,
}
#[derive(Clone)]
struct RunControl {
cancel: CancellationToken,
cancel_cause: Arc<AtomicU8>,
replicas: Vec<ReplicaControl>,
lifecycles: LifecycleTracker,
}
type ExitFuture = Pin<Box<dyn Future<Output = i32> + Send>>;
struct Spawned {
identity: ProcessIdentity,
input: Option<InputSink>,
exit: ExitFuture,
pumps: Vec<tokio::task::JoinHandle<()>>,
}
struct Monitor {
id: String,
spec: RunSpec,
stdout: PathBuf,
stderr: PathBuf,
replica: u32,
cancel: CancellationToken,
cancel_cause: Arc<AtomicU8>,
}
struct RunManagerInner {
store: Arc<Store>,
handles: Mutex<HashMap<String, RunControl>>,
logs: PathBuf,
}
#[derive(Clone)]
pub(crate) struct RunManager {
inner: Arc<RunManagerInner>,
}
impl RunManager {
pub(crate) fn new(store: Arc<Store>) -> Result<Self, String> {
let logs = store
.path()
.parent()
.unwrap_or_else(|| Path::new("."))
.join("runs");
std::fs::create_dir_all(&logs).map_err(|error| error.to_string())?;
Ok(Self {
inner: Arc::new(RunManagerInner {
store,
handles: Mutex::new(HashMap::new()),
logs,
}),
})
}
pub(crate) fn store(&self) -> Arc<Store> {
Arc::clone(&self.inner.store)
}
pub(crate) async fn recover(&self) -> Result<(), String> {
for run in self.inner.store.recoverable_runs()? {
if terminal(&run.status) {
continue;
}
let log_paths = run_log_paths(&run);
let alive = run
.processes
.iter()
.filter(|identity| process_matches(identity))
.cloned()
.collect::<Vec<_>>();
if alive.is_empty() {
self.close_vanished(run)?;
continue;
}
let snapshots = alive
.iter()
.map(|identity| run_process_info(identity, &run.spec))
.collect::<Vec<_>>();
self.inner.store.record_processes(
&snapshots,
"execution.recover",
"result",
None,
&run_controller(&run.spec),
"managed",
Some(&run.id),
&serde_json::json!({ "recovered": snapshots.len() }),
)?;
let cancel = CancellationToken::new();
let cancel_cause = Arc::new(AtomicU8::new(CANCEL_NONE));
arm_timeout(
&cancel,
&cancel_cause,
run.spec.timeout_ms,
now_ms().saturating_sub(run.created_at),
);
let replicas = alive
.iter()
.cloned()
.map(|identity| {
let token = cancel.child_token();
(
ReplicaControl {
identity: identity.clone(),
input: None,
},
identity,
token,
)
})
.collect::<Vec<_>>();
self.inner.handles.lock().await.insert(
run.id.clone(),
RunControl {
cancel,
cancel_cause: Arc::clone(&cancel_cause),
replicas: replicas
.iter()
.map(|(control, _, _)| control.clone())
.collect(),
lifecycles: LifecycleTracker::new(replicas.len() as u32),
},
);
for (replica, (_, identity, token)) in replicas.into_iter().enumerate() {
let manager = self.clone();
let id = run.id.clone();
let spec = run.spec.clone();
let log_paths = log_paths.clone();
let cancel_cause = Arc::clone(&cancel_cause);
tokio::spawn(async move {
loop {
if token.is_cancelled() {
let _ = signal_group(identity.pid, libc::SIGTERM);
}
if !process_matches(&identity) {
manager.remove_replica(&id, &identity).await;
let status = if cancel_cause.load(Ordering::Acquire) == CANCEL_TIMEOUT {
"timed_out"
} else if token.is_cancelled() {
"stopped"
} else {
"interrupted"
};
if let Some(outcome) = manager
.finish_lifecycle(&id, replica as u32, status, None)
.await
{
if let Some((stdout, stderr)) = log_paths {
manager
.publish_terminal(&id, &spec, &stdout, &stderr, outcome)
.await;
} else {
let _ = manager
.refresh_run(&id, &outcome.status, outcome.exit_code)
.await;
}
}
break;
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
});
}
}
Ok(())
}
pub(crate) async fn orphaned_runs(
&self,
updated_before: u64,
) -> Result<Vec<ManagedRun>, String> {
let candidates = self.inner.store.recoverable_runs()?;
let handles = self.inner.handles.lock().await;
Ok(candidates
.into_iter()
.filter(|run| {
!terminal(&run.status)
&& run.updated_at < updated_before
&& !handles.contains_key(&run.id)
&& !run.processes.iter().any(process_matches)
})
.collect())
}
pub(crate) fn close_vanished(&self, mut run: ManagedRun) -> Result<ManagedRun, String> {
run.status = run_log_paths(&run)
.as_ref()
.and_then(|(stdout, stderr)| {
finalize_redaction(&[stdout, stderr], &[])
.err()
.map(|error| format!("log_redaction_failed: {error}"))
})
.unwrap_or_else(|| "interrupted".to_string());
run.processes.clear();
run.updated_at = now_ms();
self.inner.store.put_run(&run)?;
Ok(run)
}
pub(crate) async fn is_active(&self, id: &str) -> bool {
self.inner.handles.lock().await.contains_key(id)
}
#[cfg(test)]
pub(crate) async fn start(&self, spec: RunSpec) -> Result<ManagedRun, String> {
self.start_with_id(uuid::Uuid::now_v7().to_string(), spec)
.await
}
pub(crate) async fn start_with_id(
&self,
id: String,
mut spec: RunSpec,
) -> Result<ManagedRun, String> {
validate_spec(&spec)?;
if spec.replicas == 0 {
spec.replicas = 1;
}
let directory = self.inner.logs.join(&id);
std::fs::create_dir_all(&directory).map_err(|error| error.to_string())?;
if !spec.resolved_env.is_empty() {
require_redaction(&directory)?;
}
let stdout = directory.join("stdout.log");
let stderr = directory.join("stderr.log");
let created_at = now_ms();
let run = ManagedRun {
id: id.clone(),
spec: spec.clone(),
status: "starting".to_string(),
processes: Vec::new(),
process_history: Vec::new(),
created_at,
updated_at: created_at,
exit_code: None,
stdout_path: Some(stdout.to_string_lossy().into_owned()),
stderr_path: Some(stderr.to_string_lossy().into_owned()),
};
self.inner.store.put_run(&run)?;
let cancel = CancellationToken::new();
let cancel_cause = Arc::new(AtomicU8::new(CANCEL_NONE));
arm_timeout(&cancel, &cancel_cause, spec.timeout_ms, 0);
let mut controls: Vec<ReplicaControl> = Vec::new();
let mut snapshots = Vec::new();
let mut monitors: Vec<(Monitor, Spawned)> = Vec::new();
for replica in 0..spec.replicas {
let spawned = match spawn_once(&spec, &stdout, &stderr, replica).await {
Ok(spawned) => spawned,
Err(error) => {
for control in &controls {
let _ = signal_group(control.identity.pid, libc::SIGKILL);
}
for (_, mut spawned) in monitors.drain(..) {
let _ = spawned.exit.await;
for pump in spawned.pumps.drain(..) {
let _ = pump.await;
}
spawned.input = None;
}
let values = spec.resolved_env.values().collect::<Vec<_>>();
let status = match finalize_redaction(&[&stdout, &stderr], &values) {
Ok(()) => "failed".to_string(),
Err(error) => format!("log_redaction_failed: {error}"),
};
let mut failed = self.get(&id)?;
failed.status = status;
failed.process_history = controls
.iter()
.map(|control| control.identity.clone())
.collect();
failed.updated_at = now_ms();
self.inner.store.put_run(&failed)?;
return Err(error);
}
};
snapshots.push(run_process_info(&spawned.identity, &spec));
let token = cancel.child_token();
controls.push(ReplicaControl {
identity: spawned.identity.clone(),
input: spawned.input.clone(),
});
monitors.push((
Monitor {
id: id.clone(),
spec: spec.clone(),
stdout: stdout.clone(),
stderr: stderr.clone(),
replica,
cancel: token,
cancel_cause: Arc::clone(&cancel_cause),
},
spawned,
));
}
let controller = run_controller(&spec);
let details = launch_details(&spec)?;
if let Err(error) = self.inner.store.record_processes(
&snapshots,
"execution.start",
"result",
None,
&controller,
"managed",
Some(&id),
&details,
) {
for control in &controls {
let _ = signal_group(control.identity.pid, libc::SIGKILL);
}
return Err(format!(
"Started run processes but could not persist their context: {error}"
));
}
self.inner.handles.lock().await.insert(
id.clone(),
RunControl {
cancel,
cancel_cause,
replicas: controls,
lifecycles: LifecycleTracker::new(spec.replicas),
},
);
self.refresh_run(&id, "running", None).await?;
let run = self.get(&id)?;
for (monitor, spawned) in monitors {
self.monitor(monitor, spawned);
}
Ok(run)
}
pub(crate) fn get(&self, id: &str) -> Result<ManagedRun, String> {
self.inner
.store
.get_run(id)?
.ok_or_else(|| format!("Run \"{id}\" not found"))
}
pub(crate) async fn signal(
&self,
id: &str,
signal: &str,
controller: &ProcessController,
) -> Result<ManagedRun, String> {
let signal_name = signal.to_string();
let signal = parse_signal(signal)?;
let value = signal_to_raw(signal)?;
let run = self.get(id)?;
let handles = self.inner.handles.lock().await;
let replicas = handles
.get(id)
.ok_or_else(|| format!("Run \"{id}\" is not active in this daemon"))?
.replicas
.as_slice();
let snapshots = replicas
.iter()
.map(|replica| run_process_info(&replica.identity, &run.spec))
.collect::<Vec<_>>();
self.inner.store.record_processes(
&snapshots,
"execution.signal",
"intent",
None,
controller,
"managed",
Some(id),
&serde_json::json!({ "signal": signal_name }),
)?;
for replica in replicas {
signal_group(replica.identity.pid, value)?;
}
drop(handles);
self.inner.store.record_processes(
&snapshots,
"execution.signal",
"result",
None,
controller,
"managed",
Some(id),
&serde_json::json!({ "signal": signal_name, "applied": true }),
)?;
self.get(id)
}
pub(crate) async fn stop(
&self,
id: &str,
timeout: Duration,
controller: &ProcessController,
) -> Result<ManagedRun, String> {
let run = self.get(id)?;
let control = self.inner.handles.lock().await.get(id).cloned();
let Some(control) = control else {
return Ok(run);
};
let snapshots = control
.replicas
.iter()
.map(|control| run_process_info(&control.identity, &run.spec))
.collect::<Vec<_>>();
self.inner.store.record_processes(
&snapshots,
"execution.stop",
"intent",
None,
controller,
"managed",
Some(id),
&serde_json::json!({ "timeout_ms": timeout.as_millis() }),
)?;
let _ = control.cancel_cause.compare_exchange(
CANCEL_NONE,
CANCEL_EXPLICIT,
Ordering::AcqRel,
Ordering::Acquire,
);
control.cancel.cancel();
for replica in &control.replicas {
if process_matches(&replica.identity) {
let _ = signal_group(replica.identity.pid, libc::SIGTERM);
}
}
let deadline = tokio::time::Instant::now() + timeout;
loop {
let run = self.get(id)?;
if run.processes.is_empty() || tokio::time::Instant::now() >= deadline {
break;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
for replica in control.replicas {
if process_matches(&replica.identity) {
let _ = signal_group(replica.identity.pid, libc::SIGKILL);
}
}
self.refresh_run(id, "stopped", None).await?;
self.inner.store.record_processes(
&snapshots,
"execution.stop",
"result",
None,
controller,
"managed",
Some(id),
&serde_json::json!({ "stopped": true }),
)?;
self.get(id)
}
pub(crate) async fn input(
&self,
id: &str,
data: &[u8],
controller: &ProcessController,
) -> Result<InputReceipt, String> {
let run = self.get(id)?;
let sinks = self
.inner
.handles
.lock()
.await
.get(id)
.ok_or_else(|| format!("Run \"{id}\" is not active in this daemon"))?
.replicas
.iter()
.filter_map(|control| control.input.clone())
.collect::<Vec<_>>();
let snapshots = run
.processes
.iter()
.map(|identity| run_process_info(identity, &run.spec))
.collect::<Vec<_>>();
self.inner.store.record_processes(
&snapshots,
"execution.input",
"intent",
None,
controller,
"managed",
Some(id),
&serde_json::json!({ "bytes": data.len() }),
)?;
for sink in &sinks {
match sink {
InputSink::Pipes(stdin) => stdin
.lock()
.await
.write_all(data)
.await
.map_err(|error| error.to_string())?,
InputSink::Pty(writer) => writer
.lock()
.map_err(|error| error.to_string())?
.write_all(data)
.map_err(|error| error.to_string())?,
}
}
Ok(InputReceipt {
id: id.to_string(),
recipients: sinks.len(),
bytes: data.len(),
})
}
pub(crate) fn logs(&self, id: &str, tail_bytes: usize) -> Result<RunLogs, String> {
let run = self.get(id)?;
if redaction_pending(run.stdout_path.as_deref())
|| redaction_pending(run.stderr_path.as_deref())
{
return Err("Managed run logs are unavailable until secret redaction completes".into());
}
let (stdout, stdout_truncated) = read_tail(run.stdout_path.as_deref(), tail_bytes)?;
let (stderr, stderr_truncated) = read_tail(run.stderr_path.as_deref(), tail_bytes)?;
Ok(RunLogs {
id: id.to_string(),
stdout,
stderr,
stdout_truncated,
stderr_truncated,
tail_bytes,
})
}
pub(crate) fn slice(&self, id: &str, request: &LogSliceRequest) -> Result<LogSlice, String> {
let run = self.get(id)?;
if redaction_pending(run.stdout_path.as_deref())
|| redaction_pending(run.stderr_path.as_deref())
{
return Err("Managed run logs are unavailable until secret redaction completes".into());
}
let streams = [
("stdout", run.stdout_path.as_deref()),
("stderr", run.stderr_path.as_deref()),
];
let mut lines = Vec::new();
let mut totals = [0usize; 2];
let mut matches = 0usize;
let mut truncated = false;
for (index, (stream, path)) in streams.into_iter().enumerate() {
match &request.pattern {
Some(pattern) => {
let (found, count, total) = crate::log_file::read_matches(
path,
stream,
pattern,
request.context,
request.max_matches,
)?;
truncated |= count > request.max_matches;
matches += count;
totals[index] = total;
lines.extend(found);
}
None => {
let (found, total) =
crate::log_file::read_lines(path, stream, request.start, request.end)?;
totals[index] = total;
lines.extend(found);
}
}
}
Ok(LogSlice {
id: id.to_string(),
lines,
stdout_lines: totals[0],
stderr_lines: totals[1],
total_matches: request.pattern.as_ref().map(|_| matches),
truncated,
})
}
pub(crate) async fn wait(
&self,
id: &str,
timeout: Option<Duration>,
) -> Result<ManagedRun, String> {
let started = tokio::time::Instant::now();
loop {
let run = self.get(id)?;
if terminal(&run.status) {
return Ok(run);
}
if timeout.is_some_and(|timeout| started.elapsed() >= timeout) {
return Ok(run);
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
}
pub(crate) async fn remove(&self, id: &str) -> Result<bool, String> {
if self.inner.handles.lock().await.contains_key(id) {
return Err(format!("Run \"{id}\" is active; stop it before removal"));
}
self.inner.store.delete_run(id)
}
fn monitor(&self, monitor: Monitor, mut spawned: Spawned) {
let manager = self.clone();
tokio::spawn(async move {
let Monitor {
id,
spec,
stdout,
stderr,
replica,
cancel,
cancel_cause,
} = monitor;
let mut attempt = 1_u32;
loop {
let identity = spawned.identity.clone();
let code = tokio::select! {
code = &mut spawned.exit => code,
_ = cancel.cancelled() => {
let _ = signal_group(identity.pid, libc::SIGTERM);
match tokio::time::timeout(Duration::from_secs(5), &mut spawned.exit).await {
Ok(code) => code,
Err(_) => {
let _ = signal_group(identity.pid, libc::SIGKILL);
spawned.exit.await
}
}
}
};
for pump in spawned.pumps.drain(..) {
let _ = pump.await;
}
manager.remove_replica(&id, &identity).await;
spawned.input = None;
let restart = !cancel.is_cancelled()
&& attempt < spec.max_attempts.max(1)
&& match spec.restart {
RestartPolicy::Never => false,
RestartPolicy::OnFailure => code != 0,
RestartPolicy::Always => true,
};
let snapshot = run_process_info(&identity, &spec);
if let Err(error) = manager.inner.store.record_processes(
std::slice::from_ref(&snapshot),
"execution.exit",
"result",
None,
&run_controller(&spec),
"managed",
Some(&id),
&serde_json::json!({
"exit_code": code,
"cancelled": cancel.is_cancelled(),
"restart": restart
}),
) {
if let Some(outcome) = manager
.finish_lifecycle(
&id,
replica,
&format!("context_record_failed: {error}"),
Some(code),
)
.await
{
manager
.publish_terminal(&id, &spec, &stdout, &stderr, outcome)
.await;
}
break;
}
if !restart {
let status = if cancel_cause.load(Ordering::Acquire) == CANCEL_TIMEOUT {
"timed_out"
} else if cancel.is_cancelled() {
"stopped"
} else if code == 0 {
"completed"
} else {
"failed"
};
if let Some(outcome) = manager
.finish_lifecycle(&id, replica, status, Some(code))
.await
{
manager
.publish_terminal(&id, &spec, &stdout, &stderr, outcome)
.await;
}
break;
}
let exponent = attempt.saturating_sub(1).min(31);
let delay = spec
.retry_backoff_ms
.saturating_mul(1_u64 << exponent)
.min(spec.retry_max_backoff_ms);
if delay > 0 {
tokio::select! {
_ = tokio::time::sleep(Duration::from_millis(delay)) => {}
_ = cancel.cancelled() => {}
}
}
if cancel.is_cancelled() {
let status = if cancel_cause.load(Ordering::Acquire) == CANCEL_TIMEOUT {
"timed_out"
} else {
"stopped"
};
if let Some(outcome) = manager
.finish_lifecycle(&id, replica, status, Some(code))
.await
{
manager
.publish_terminal(&id, &spec, &stdout, &stderr, outcome)
.await;
}
break;
}
match spawn_once(&spec, &stdout, &stderr, replica).await {
Ok(next) => {
attempt = attempt.saturating_add(1);
let identity = next.identity.clone();
let input = next.input.clone();
spawned = next;
if manager.add_replica(&id, identity, input).await {
let _ = manager.refresh_run(&id, "running", Some(code)).await;
}
}
Err(error) => {
let status = if cancel_cause.load(Ordering::Acquire) == CANCEL_TIMEOUT {
"timed_out".to_string()
} else if cancel.is_cancelled() {
"stopped".to_string()
} else {
format!("restart_failed: {error}")
};
if let Some(outcome) = manager
.finish_lifecycle(&id, replica, &status, Some(code))
.await
{
manager
.publish_terminal(&id, &spec, &stdout, &stderr, outcome)
.await;
}
break;
}
}
}
});
}
async fn add_replica(
&self,
id: &str,
identity: ProcessIdentity,
input: Option<InputSink>,
) -> bool {
let mut handles = self.inner.handles.lock().await;
let Some(control) = handles.get_mut(id) else {
return false;
};
if control.cancel.is_cancelled() {
return false;
}
control.replicas.push(ReplicaControl { identity, input });
true
}
async fn remove_replica(&self, id: &str, identity: &ProcessIdentity) {
let mut handles = self.inner.handles.lock().await;
if let Some(control) = handles.get_mut(id) {
control
.replicas
.retain(|replica| replica.identity != *identity);
}
}
async fn finish_lifecycle(
&self,
id: &str,
replica: u32,
status: &str,
exit_code: Option<i32>,
) -> Option<LifecycleOutcome> {
let mut handles = self.inner.handles.lock().await;
let control = handles.get_mut(id)?;
let outcome = control.lifecycles.finish(replica, status, exit_code);
if outcome.is_some() {
handles.remove(id);
}
outcome
}
async fn publish_terminal(
&self,
id: &str,
spec: &RunSpec,
stdout: &Path,
stderr: &Path,
mut outcome: LifecycleOutcome,
) {
let values = spec.resolved_env.values().collect::<Vec<_>>();
if let Err(error) = finalize_redaction(&[stdout, stderr], &values) {
outcome.status = format!("log_redaction_failed: {error}");
}
let _ = self
.refresh_run(id, &outcome.status, outcome.exit_code)
.await;
}
async fn refresh_run(
&self,
id: &str,
status: &str,
exit_code: Option<i32>,
) -> Result<(), String> {
let processes = self
.inner
.handles
.lock()
.await
.get(id)
.map(|control| {
control
.replicas
.iter()
.map(|control| control.identity.clone())
.collect::<Vec<_>>()
})
.unwrap_or_default();
let mut run = self.get(id)?;
run.status = status.to_string();
for process in &processes {
if !run.process_history.contains(process) {
run.process_history.push(process.clone());
}
}
run.processes = processes;
run.updated_at = now_ms();
run.exit_code = exit_code;
self.inner.store.put_run(&run)
}
}
async fn spawn_once(
spec: &RunSpec,
stdout: &Path,
stderr: &Path,
_replica: u32,
) -> Result<Spawned, String> {
match spec.io {
IoMode::Pipes => spawn_pipes(spec, stdout, stderr).await,
IoMode::Pty => spawn_pty(spec, stdout).await,
}
}
async fn spawn_pipes(spec: &RunSpec, stdout: &Path, stderr: &Path) -> Result<Spawned, String> {
let mut command = command(spec)?;
command
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(unix)]
unsafe {
command.pre_exec(|| {
if libc::setpgid(0, 0) == -1 {
return Err(std::io::Error::last_os_error());
}
Ok(())
});
}
let mut child = command.spawn().map_err(|error| error.to_string())?;
let pid = child
.id()
.ok_or_else(|| "Spawned command has no process ID".to_string())?;
let identity = await_identity(pid).await?;
let stdin = child
.stdin
.take()
.map(|stdin| InputSink::Pipes(Arc::new(Mutex::new(stdin))));
let mut pumps = Vec::new();
if let Some(reader) = child.stdout.take() {
pumps.push(pump(reader, stdout.to_path_buf()));
}
if let Some(reader) = child.stderr.take() {
pumps.push(pump(reader, stderr.to_path_buf()));
}
Ok(Spawned {
identity,
input: stdin,
pumps,
exit: Box::pin(async move {
child
.wait()
.await
.map(|status| status.code().unwrap_or(128))
.unwrap_or(127)
}),
})
}
async fn spawn_pty(spec: &RunSpec, stdout: &Path) -> Result<Spawned, String> {
let spec = spec.clone();
let stdout = stdout.to_path_buf();
tokio::task::spawn_blocking(move || {
let spawn = PTY_SPAWN_LOCK
.lock()
.map_err(|error| format!("PTY spawn lock failed: {error}"))?;
let mut file = append_file(&stdout).map_err(|error| {
format!(
"Could not open PTY stdout log {}: {error}",
stdout.display()
)
})?;
let pair = native_pty_system()
.openpty(PtySize {
rows: 24,
cols: 120,
pixel_width: 0,
pixel_height: 0,
})
.map_err(|error| error.to_string())?;
let mut builder = pty_command(&spec)?;
if let Some(cwd) = &spec.cwd {
builder.cwd(cwd);
}
for name in &spec.suppressed_env {
builder.env_remove(name);
}
for (name, value) in &spec.env {
builder.env(name, value);
}
for (name, value) in &spec.resolved_env {
builder.env(name, value);
}
let master = pair.master;
let mut reader = master
.try_clone_reader()
.map_err(|error| error.to_string())?;
let writer = master.take_writer().map_err(|error| error.to_string())?;
let exited = Arc::new(AtomicBool::new(false));
let reader_exited = Arc::clone(&exited);
let pump = std::thread::Builder::new()
.name(format!("apoc-pty-log-{}", spec.name))
.spawn(move || {
let mut buffer = [0_u8; 8192];
let mut terminal_misses = 0_u32;
loop {
match reader.read(&mut buffer) {
Ok(0) => {
if reader_exited.load(Ordering::Acquire) {
terminal_misses = terminal_misses.saturating_add(1);
if terminal_misses >= 100 {
break;
}
}
std::thread::sleep(Duration::from_millis(1));
}
Ok(size) => {
terminal_misses = 0;
let _ = file.write_all(&buffer[..size]);
let _ = file.flush();
}
Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
Err(_) => {
if reader_exited.load(Ordering::Acquire) {
terminal_misses = terminal_misses.saturating_add(1);
if terminal_misses >= 100 {
break;
}
}
std::thread::sleep(Duration::from_millis(1));
}
}
}
drop(master);
})
.map_err(|error| error.to_string())?;
let mut child = pair
.slave
.spawn_command(builder)
.map_err(|error| error.to_string())?;
drop(pair.slave);
let pid = child
.process_id()
.ok_or_else(|| "PTY command has no process ID".to_string())?;
let input = InputSink::Pty(Arc::new(StdMutex::new(writer)));
let exit = Box::pin(async move {
tokio::task::spawn_blocking(move || {
let code = child
.wait()
.map(|status| status.exit_code() as i32)
.unwrap_or(127);
exited.store(true, Ordering::Release);
code
})
.await
.unwrap_or(127)
}) as ExitFuture;
drop(spawn);
Ok((pid, input, exit, pump))
})
.await
.map_err(|error| error.to_string())?
.map(|(pid, input, exit, pump)| Spawned {
identity: identity_for_pid(pid).unwrap_or_else(|_| fallback_identity(pid)),
input: Some(input),
exit,
pumps: vec![tokio::task::spawn_blocking(move || {
let _ = pump.join();
})],
})
}
fn command(spec: &RunSpec) -> Result<Command, String> {
let mut command = if spec.shell {
let mut command = Command::new("/bin/zsh");
command.args(["-lc", &spec.argv[0]]);
command
} else {
let mut command = Command::new(&spec.argv[0]);
command.args(&spec.argv[1..]);
command
};
if let Some(cwd) = &spec.cwd {
command.current_dir(cwd);
}
for name in &spec.suppressed_env {
command.env_remove(name);
}
command.envs(&spec.env);
command.envs(&spec.resolved_env);
Ok(command)
}
fn pty_command(spec: &RunSpec) -> Result<CommandBuilder, String> {
if spec.shell {
let mut command = CommandBuilder::new("/bin/zsh");
command.args(["-lc", &spec.argv[0]]);
Ok(command)
} else {
Ok(CommandBuilder::from_argv(
spec.argv.iter().map(std::ffi::OsString::from).collect(),
))
}
}
fn validate_spec(spec: &RunSpec) -> Result<(), String> {
if spec.argv.is_empty() {
return Err("argv must contain an executable".to_string());
}
if spec.shell && spec.argv.len() != 1 {
return Err("shell mode accepts one zsh command string in argv".to_string());
}
if spec.replicas > 256 {
return Err("replicas cannot exceed 256".to_string());
}
if spec.max_attempts == 0 {
return Err("max_attempts must be at least 1".to_string());
}
if spec.retry_max_backoff_ms < spec.retry_backoff_ms {
return Err("retry_max_backoff_ms cannot be less than retry_backoff_ms".to_string());
}
Ok(())
}
fn pump(
reader: impl AsyncRead + Unpin + Send + 'static,
path: PathBuf,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let Ok(mut file) = tokio::fs::OpenOptions::new()
.create(true)
.append(true)
.open(path)
.await
else {
return;
};
let mut reader = reader;
let _ = tokio::io::copy(&mut reader, &mut file).await;
let _ = file.flush().await;
})
}
fn append_file(path: &Path) -> std::io::Result<std::fs::File> {
std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(path)
}
async fn await_identity(pid: u32) -> Result<ProcessIdentity, String> {
for _ in 0..20 {
if let Ok(identity) = identity_for_pid(pid) {
return Ok(identity);
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
Ok(fallback_identity(pid))
}
fn fallback_identity(pid: u32) -> ProcessIdentity {
let started_at = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default();
ProcessIdentity {
boot_id: format!("boot-{}", sysinfo::System::boot_time()),
pid,
ppid: Some(std::process::id()),
pgid: Some(pid),
started_at: started_at.as_secs(),
started_at_ns: started_at.as_nanos().min(u64::MAX as u128) as u64,
name: "managed".to_string(),
executable: None,
}
}
fn run_controller(spec: &RunSpec) -> ProcessController {
spec.requested_by
.clone()
.unwrap_or_else(|| ProcessController::capture(false, None))
}
fn run_process_info(identity: &ProcessIdentity, spec: &RunSpec) -> ProcessInfo {
if let Ok(mut process) = process_info_for_pid(identity.pid) {
process.identity = identity.clone();
return process;
}
ProcessInfo {
identity: identity.clone(),
status: "starting_or_exited".to_string(),
user_id: None,
effective_user_id: None,
group_id: None,
session_id: None,
cpu_percent: 0.0,
cpu_time_ms: 0,
memory_bytes: 0,
virtual_memory_bytes: 0,
runtime_seconds: 0,
command: if spec.shell {
vec![
"/bin/zsh".to_string(),
"-lc".to_string(),
spec.argv.first().cloned().unwrap_or_default(),
]
} else {
spec.argv.clone()
},
cwd: spec.cwd.clone().or_else(|| {
std::env::current_dir()
.ok()
.map(|path| path.to_string_lossy().into_owned())
}),
disk_read_bytes: 0,
disk_written_bytes: 0,
}
}
fn launch_details(spec: &RunSpec) -> Result<serde_json::Value, String> {
let mut environment = std::env::vars_os().collect::<std::collections::BTreeMap<_, _>>();
for name in &spec.suppressed_env {
environment.remove(&std::ffi::OsString::from(name));
}
environment.extend(
spec.env
.iter()
.map(|(name, value)| (name.into(), value.into())),
);
let mut digest = Sha256::new();
for (name, value) in &environment {
let name = name.as_encoded_bytes();
let value = value.as_encoded_bytes();
digest.update((name.len() as u64).to_be_bytes());
digest.update(name);
digest.update((value.len() as u64).to_be_bytes());
digest.update(value);
}
let resolved_cwd = spec.cwd.clone().or_else(|| {
std::env::current_dir()
.ok()
.map(|path| path.to_string_lossy().into_owned())
});
Ok(serde_json::json!({
"spec": spec,
"resolved_argv": if spec.shell {
vec![
"/bin/zsh".to_string(),
"-lc".to_string(),
spec.argv.first().cloned().unwrap_or_default()
]
} else {
spec.argv.clone()
},
"resolved_cwd": resolved_cwd,
"inherits_environment": true,
"environment_keys": environment
.keys()
.map(|name| name.to_string_lossy())
.collect::<Vec<_>>(),
"environment_sha256": format!("{:x}", digest.finalize())
}))
}
fn signal_group(pid: u32, signal: i32) -> Result<(), String> {
let result = unsafe { libc::kill(-(pid as i32), signal) };
if result == -1 {
let group_error = std::io::Error::last_os_error();
let result = unsafe { libc::kill(pid as i32, signal) };
if result == -1 {
return Err(format!(
"Cannot signal process group or PID {pid}: {group_error}; {}",
std::io::Error::last_os_error()
));
}
}
Ok(())
}
fn signal_to_raw(signal: sysinfo::Signal) -> Result<i32, String> {
Ok(match signal {
sysinfo::Signal::Hangup => libc::SIGHUP,
sysinfo::Signal::Interrupt => libc::SIGINT,
sysinfo::Signal::Quit => libc::SIGQUIT,
sysinfo::Signal::Kill => libc::SIGKILL,
sysinfo::Signal::User1 => libc::SIGUSR1,
sysinfo::Signal::User2 => libc::SIGUSR2,
sysinfo::Signal::Term => libc::SIGTERM,
sysinfo::Signal::Continue => libc::SIGCONT,
sysinfo::Signal::Stop => libc::SIGSTOP,
_ => return Err("Signal is not supported by apoc".to_string()),
})
}
fn process_matches(identity: &ProcessIdentity) -> bool {
identity_for_pid(identity.pid).is_ok_and(|current| {
current.boot_id == identity.boot_id
&& current.pid == identity.pid
&& current.started_at_ns == identity.started_at_ns
})
}
fn arm_timeout(
cancel: &CancellationToken,
cancel_cause: &Arc<AtomicU8>,
timeout_ms: Option<u64>,
elapsed_ms: u64,
) {
let Some(timeout_ms) = timeout_ms else {
return;
};
let remaining_ms = timeout_ms.saturating_sub(elapsed_ms);
if remaining_ms == 0 {
if cancel_cause
.compare_exchange(
CANCEL_NONE,
CANCEL_TIMEOUT,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_ok()
{
cancel.cancel();
}
return;
}
let deadline = cancel.clone();
let cause = Arc::clone(cancel_cause);
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(remaining_ms)).await;
if cause
.compare_exchange(
CANCEL_NONE,
CANCEL_TIMEOUT,
Ordering::AcqRel,
Ordering::Acquire,
)
.is_ok()
{
deadline.cancel();
}
});
}
fn run_log_paths(run: &ManagedRun) -> Option<(PathBuf, PathBuf)> {
Some((
PathBuf::from(run.stdout_path.as_ref()?),
PathBuf::from(run.stderr_path.as_ref()?),
))
}
pub(crate) fn interrupt_orphaned_runs(store: &Store) -> Result<usize, String> {
let now = now_ms();
let mut closed = 0;
for mut run in store.recoverable_runs()? {
if terminal(&run.status) || run.processes.iter().any(process_matches) {
continue;
}
run.status = "interrupted".to_owned();
run.processes.clear();
run.updated_at = now;
store.put_run(&run)?;
closed += 1;
}
Ok(closed)
}
fn terminal(status: &str) -> bool {
matches!(
status,
"completed" | "failed" | "stopped" | "timed_out" | "interrupted"
) || status.starts_with("restart_failed:")
|| status.starts_with("context_record_failed:")
|| status.starts_with("log_redaction_failed:")
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use tempfile::tempdir;
use super::*;
#[tokio::test]
async fn persists_and_observes_a_managed_run() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = RunManager::new(store).unwrap();
let run = manager
.start(RunSpec {
name: "echo".to_string(),
purpose: Some("test managed context".to_string()),
labels: BTreeMap::new(),
requested_by: None,
argv: vec!["/bin/echo".to_string(), "managed".to_string()],
cwd: None,
env: BTreeMap::new(),
resolved_env: Default::default(),
suppressed_env: Vec::new(),
shell: false,
io: IoMode::Pipes,
lifetime: crate::model::RunLifetime::Persistent,
restart: RestartPolicy::Never,
timeout_ms: None,
replicas: 1,
max_attempts: 1,
retry_backoff_ms: 500,
retry_max_backoff_ms: 30_000,
})
.await
.unwrap();
let finished = manager
.wait(&run.id, Some(Duration::from_secs(2)))
.await
.unwrap();
assert_eq!(finished.status, "completed");
assert!(
manager
.logs(&run.id, 1024)
.unwrap()
.stdout
.contains("managed")
);
}
#[tokio::test]
async fn redaction_failure_never_publishes_completed_or_retains_unsanitized_logs() {
use std::os::unix::fs::PermissionsExt;
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = RunManager::new(store).unwrap();
let secret = "synthetic-redaction-secret";
let run = manager
.start(RunSpec {
name: "redaction-failure".to_string(),
purpose: Some("verify failed log redaction".to_string()),
labels: BTreeMap::new(),
requested_by: None,
argv: vec![
"/bin/sh".to_string(),
"-c".to_string(),
"printf %s \"$TEST_SECRET\"; sleep 1".to_string(),
],
cwd: None,
env: BTreeMap::new(),
resolved_env: BTreeMap::from([("TEST_SECRET".to_string(), secret.to_string())]),
suppressed_env: Vec::new(),
shell: false,
io: IoMode::Pipes,
lifetime: crate::model::RunLifetime::Persistent,
restart: RestartPolicy::Never,
timeout_ms: None,
replicas: 1,
max_attempts: 1,
retry_backoff_ms: 500,
retry_max_backoff_ms: 30_000,
})
.await
.unwrap();
let stdout = PathBuf::from(run.stdout_path.as_ref().unwrap());
let deadline = tokio::time::Instant::now() + Duration::from_secs(1);
loop {
if std::fs::read_to_string(&stdout).is_ok_and(|contents| contents.contains(secret)) {
break;
}
assert!(
tokio::time::Instant::now() < deadline,
"child never wrote the synthetic secret"
);
tokio::time::sleep(Duration::from_millis(10)).await;
}
std::fs::set_permissions(&stdout, std::fs::Permissions::from_mode(0o000)).unwrap();
let finished = manager
.wait(&run.id, Some(Duration::from_secs(3)))
.await
.unwrap();
if stdout.exists() {
std::fs::set_permissions(&stdout, std::fs::Permissions::from_mode(0o600)).unwrap();
}
let leaked = std::fs::read_to_string(&stdout).unwrap_or_default();
assert!(
finished.status.starts_with("log_redaction_failed:"),
"run published status {:?} after redaction failed; retained log {leaked:?}",
finished.status
);
assert!(
!stdout.exists(),
"failed redaction retained unsanitized log {leaked:?}"
);
}
#[tokio::test]
async fn terminal_pty_run_has_flushed_logs() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = RunManager::new(store).unwrap();
let runs = futures::future::join_all((0..64).map(|iteration| {
let manager = manager.clone();
async move {
let run = manager
.start(RunSpec {
name: format!("pty-echo-{iteration}"),
purpose: Some("verify terminal PTY logs".to_string()),
labels: BTreeMap::new(),
requested_by: None,
argv: vec!["/bin/echo".to_string(), "pty-apocalypse".to_string()],
cwd: None,
env: BTreeMap::new(),
resolved_env: Default::default(),
suppressed_env: Vec::new(),
shell: false,
io: IoMode::Pty,
lifetime: crate::model::RunLifetime::Persistent,
restart: RestartPolicy::Never,
timeout_ms: None,
replicas: 1,
max_attempts: 1,
retry_backoff_ms: 500,
retry_max_backoff_ms: 30_000,
})
.await
.unwrap();
(iteration, run)
}
}))
.await;
for (iteration, run) in runs {
let finished = manager
.wait(&run.id, Some(Duration::from_secs(2)))
.await
.unwrap();
let logs = manager.logs(&run.id, 1024).unwrap();
assert_eq!(finished.status, "completed", "iteration {iteration}");
assert!(
logs.stdout.contains("pty-apocalypse"),
"iteration {iteration}, run {}, stdout {:?}, stderr {:?}",
run.id,
logs.stdout,
logs.stderr
);
}
}
#[tokio::test]
async fn stops_a_managed_run_during_restart_backoff() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = RunManager::new(store).unwrap();
let run = manager
.start(RunSpec {
name: "restart-backoff".to_string(),
purpose: Some("verify restart cancellation".to_string()),
labels: BTreeMap::new(),
requested_by: None,
argv: vec!["/usr/bin/false".to_string()],
cwd: None,
env: BTreeMap::new(),
resolved_env: Default::default(),
suppressed_env: Vec::new(),
shell: false,
io: IoMode::Pipes,
lifetime: crate::model::RunLifetime::Persistent,
restart: RestartPolicy::OnFailure,
timeout_ms: None,
replicas: 1,
max_attempts: 2,
retry_backoff_ms: 500,
retry_max_backoff_ms: 500,
})
.await
.unwrap();
let identity = run.processes[0].clone();
tokio::time::timeout(Duration::from_secs(1), async {
while process_matches(&identity) {
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(25)).await;
let stopped = manager
.stop(
&run.id,
Duration::from_millis(250),
&ProcessController::capture(true, None),
)
.await
.unwrap();
assert_eq!(stopped.status, "stopped");
tokio::time::sleep(Duration::from_millis(600)).await;
let finished = manager.get(&run.id).unwrap();
assert_eq!(finished.status, "stopped");
assert_eq!(finished.process_history.len(), 1);
}
#[tokio::test]
async fn stopping_a_run_whose_handle_is_gone_succeeds_instead_of_erroring() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = RunManager::new(store).unwrap();
let run = manager
.start(RunSpec {
name: "already-exited".to_string(),
purpose: Some("verify idempotent stop".to_string()),
labels: BTreeMap::new(),
requested_by: None,
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
resolved_env: Default::default(),
suppressed_env: Vec::new(),
shell: false,
io: IoMode::Pipes,
lifetime: crate::model::RunLifetime::Persistent,
restart: RestartPolicy::Never,
timeout_ms: None,
replicas: 1,
max_attempts: 1,
retry_backoff_ms: 1,
retry_max_backoff_ms: 1,
})
.await
.unwrap();
manager.inner.handles.lock().await.remove(&run.id);
let stopped = manager
.stop(
&run.id,
Duration::from_millis(250),
&ProcessController::capture(true, None),
)
.await
.expect("stopping a run with no live handle must succeed");
assert_eq!(stopped.id, run.id);
}
#[tokio::test]
async fn marks_a_managed_run_failed_when_its_process_cannot_spawn() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = RunManager::new(store).unwrap();
let result = manager
.start_with_id(
"missing-run".to_string(),
RunSpec {
name: "missing".to_string(),
purpose: Some("test launch failure".to_string()),
labels: BTreeMap::new(),
requested_by: None,
argv: vec!["/definitely/missing-apoc-executable".to_string()],
cwd: None,
env: BTreeMap::new(),
resolved_env: BTreeMap::from([(
"APOC_TEST_SECRET".to_string(),
"launch-failure-secret".to_string(),
)]),
suppressed_env: Vec::new(),
shell: false,
io: IoMode::Pipes,
lifetime: crate::model::RunLifetime::Persistent,
restart: RestartPolicy::Never,
timeout_ms: None,
replicas: 1,
max_attempts: 1,
retry_backoff_ms: 500,
retry_max_backoff_ms: 30_000,
},
)
.await;
assert!(result.is_err());
assert_eq!(manager.get("missing-run").unwrap().status, "failed");
assert!(manager.logs("missing-run", 1_024).is_ok());
}
#[tokio::test]
async fn an_expired_deadline_is_armed_before_recovery_monitors_start() {
let cancel = CancellationToken::new();
let cancel_cause = Arc::new(AtomicU8::new(CANCEL_NONE));
arm_timeout(&cancel, &cancel_cause, Some(20), 20);
assert_eq!(
cancel_cause.load(Ordering::Acquire),
CANCEL_TIMEOUT,
"an already-expired deadline must not depend on the timeout task being scheduled"
);
assert!(cancel.is_cancelled());
}
#[tokio::test]
async fn recovery_preserves_the_original_wall_time_deadline() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = RunManager::new(store.clone()).unwrap();
let mut child = std::process::Command::new("/bin/sleep")
.arg("30")
.spawn()
.unwrap();
let identity = await_identity(child.id()).await.unwrap();
let mut spec = fixture_spec();
spec.argv = vec!["/bin/sleep".to_string(), "30".to_string()];
spec.timeout_ms = Some(20);
let now = now_ms();
store
.put_run(&ManagedRun {
id: "recovered-wall-time-run".to_string(),
spec,
status: "running".to_string(),
processes: vec![identity.clone()],
process_history: vec![identity],
created_at: now.saturating_sub(100),
updated_at: now,
exit_code: None,
stdout_path: None,
stderr_path: None,
})
.unwrap();
manager.recover().await.unwrap();
let reaper = tokio::task::spawn_blocking(move || child.wait().unwrap());
let result = tokio::time::timeout(
Duration::from_millis(500),
manager.wait("recovered-wall-time-run", None),
)
.await;
if result.is_err() {
manager
.stop(
"recovered-wall-time-run",
Duration::from_secs(1),
&ProcessController::capture(true, None),
)
.await
.unwrap();
}
reaper.await.unwrap();
let finished = result
.expect("recovered run outlived its original wall-time deadline")
.unwrap();
assert_eq!(finished.status, "timed_out");
assert_eq!(finished.exit_code, None);
}
#[tokio::test]
async fn validates_run_specs_and_portable_helper_boundaries() {
let directory = tempdir().unwrap();
let mut spec = fixture_spec();
validate_spec(&spec).unwrap();
spec.argv.clear();
assert!(validate_spec(&spec).unwrap_err().contains("argv"));
spec = fixture_spec();
spec.shell = true;
spec.argv.push("extra".to_string());
assert!(validate_spec(&spec).unwrap_err().contains("shell mode"));
spec = fixture_spec();
spec.replicas = 257;
assert!(validate_spec(&spec).unwrap_err().contains("replicas"));
spec = fixture_spec();
spec.max_attempts = 0;
assert!(validate_spec(&spec).unwrap_err().contains("max_attempts"));
spec = fixture_spec();
spec.retry_backoff_ms = 10;
spec.retry_max_backoff_ms = 9;
assert!(
validate_spec(&spec)
.unwrap_err()
.contains("retry_max_backoff_ms")
);
spec = fixture_spec();
spec.cwd = Some(directory.path().to_string_lossy().into_owned());
spec.env
.insert("APOC_FIXTURE".to_string(), "literal".to_string());
spec.resolved_env
.insert("APOC_SECRET_FIXTURE".to_string(), "resolved".to_string());
command(&spec).unwrap();
pty_command(&spec).unwrap();
let details = launch_details(&spec).unwrap();
assert_eq!(details["resolved_argv"][0], "/usr/bin/true");
assert_eq!(
details["resolved_cwd"],
directory.path().to_string_lossy().as_ref()
);
assert!(
details["environment_keys"]
.as_array()
.unwrap()
.iter()
.any(|name| name == "APOC_FIXTURE")
);
spec.shell = true;
spec.argv = vec!["printf shell".to_string()];
command(&spec).unwrap();
pty_command(&spec).unwrap();
let details = launch_details(&spec).unwrap();
assert_eq!(details["resolved_argv"][0], "/bin/zsh");
let requested = ProcessController::capture(true, Some("execution".to_string()));
spec.requested_by = Some(requested.clone());
assert_eq!(run_controller(&spec).invocation_id, requested.invocation_id);
spec.requested_by = None;
assert!(!run_controller(&spec).agent);
let current = identity_for_pid(std::process::id()).unwrap();
assert!(process_matches(¤t));
let mut mismatched = current.clone();
mismatched.started_at_ns = mismatched.started_at_ns.saturating_add(1);
assert!(!process_matches(&mismatched));
let current_info = run_process_info(¤t, &spec);
assert_eq!(current_info.identity, current);
let fallback = fallback_identity(u32::MAX);
let fallback_info = run_process_info(&fallback, &spec);
assert_eq!(fallback_info.status, "starting_or_exited");
assert_eq!(fallback_info.command[0], "/bin/zsh");
let output = directory.path().join("append.log");
append_file(&output).unwrap().write_all(b"fixture").unwrap();
assert_eq!(std::fs::read_to_string(&output).unwrap(), "fixture");
assert!(append_file(&directory.path().join("missing/append.log")).is_err());
let pump = pump(
tokio::io::empty(),
directory.path().join("missing/pump.log"),
);
pump.await.unwrap();
for (signal, expected) in [
(sysinfo::Signal::Hangup, libc::SIGHUP),
(sysinfo::Signal::Interrupt, libc::SIGINT),
(sysinfo::Signal::Quit, libc::SIGQUIT),
(sysinfo::Signal::Kill, libc::SIGKILL),
(sysinfo::Signal::User1, libc::SIGUSR1),
(sysinfo::Signal::User2, libc::SIGUSR2),
(sysinfo::Signal::Term, libc::SIGTERM),
(sysinfo::Signal::Continue, libc::SIGCONT),
(sysinfo::Signal::Stop, libc::SIGSTOP),
] {
assert_eq!(signal_to_raw(signal).unwrap(), expected);
}
assert!(signal_to_raw(sysinfo::Signal::Alarm).is_err());
let run = ManagedRun {
id: "logs".to_string(),
spec: fixture_spec(),
status: "completed".to_string(),
processes: Vec::new(),
process_history: Vec::new(),
created_at: 1,
updated_at: 2,
exit_code: Some(0),
stdout_path: Some(output.to_string_lossy().into_owned()),
stderr_path: Some(
directory
.path()
.join("stderr.log")
.to_string_lossy()
.into_owned(),
),
};
assert!(run_log_paths(&run).is_some());
let mut missing_log = run;
missing_log.stderr_path = None;
assert!(run_log_paths(&missing_log).is_none());
for status in [
"completed",
"failed",
"stopped",
"interrupted",
"restart_failed: fixture",
"context_record_failed: fixture",
"log_redaction_failed: fixture",
] {
assert!(terminal(status), "{status}");
}
assert!(!terminal("running"));
}
#[tokio::test]
async fn orphaned_runs_are_closed_and_live_ones_are_spared() {
let directory = tempdir().unwrap();
let store = Store::open(directory.path().join("state.sqlite3")).unwrap();
let mut child = std::process::Command::new("/bin/sleep")
.arg("30")
.spawn()
.unwrap();
let alive = await_identity(child.id()).await.unwrap();
let recycled = ProcessIdentity {
started_at_ns: alive.started_at_ns.wrapping_add(1),
..alive.clone()
};
let put = |id: &str, status: &str, processes: Vec<ProcessIdentity>| {
store
.put_run(&ManagedRun {
id: id.to_owned(),
spec: fixture_spec(),
status: status.to_owned(),
processes,
process_history: Vec::new(),
created_at: 1,
updated_at: 1,
exit_code: None,
stdout_path: None,
stderr_path: None,
})
.unwrap();
};
put("live", "running", vec![alive]);
put("recycled-pid", "running", vec![recycled]);
put("never-started", "starting", Vec::new());
put("finished", "completed", Vec::new());
put("halted", "stopped", Vec::new());
let closed = interrupt_orphaned_runs(&store).unwrap();
assert_eq!(closed, 2, "only the two orphans should be closed");
let status = |id: &str| store.get_run(id).unwrap().unwrap().status;
assert_eq!(status("live"), "running", "a live process must be spared");
assert_eq!(status("recycled-pid"), "interrupted");
assert_eq!(status("never-started"), "interrupted");
assert_eq!(status("finished"), "completed");
assert_eq!(status("halted"), "stopped");
assert!(
store
.get_run("recycled-pid")
.unwrap()
.unwrap()
.processes
.is_empty(),
"a closed run must not keep pointing at a process that is gone"
);
assert_eq!(interrupt_orphaned_runs(&store).unwrap(), 0);
child.kill().ok();
child.wait().ok();
}
fn fixture_spec() -> RunSpec {
RunSpec {
name: "fixture".to_string(),
purpose: Some("Exercise run helper boundaries".to_string()),
labels: BTreeMap::new(),
requested_by: None,
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
resolved_env: BTreeMap::new(),
suppressed_env: Vec::new(),
shell: false,
io: IoMode::Pipes,
lifetime: crate::model::RunLifetime::Persistent,
restart: RestartPolicy::Never,
timeout_ms: None,
replicas: 1,
max_attempts: 1,
retry_backoff_ms: 500,
retry_max_backoff_ms: 30_000,
}
}
}