use std::collections::HashMap;
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::process::Stdio;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex as StdMutex};
use std::time::Duration;
use portable_pty::{CommandBuilder, PtySize, native_pty_system};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use tokio::io::{AsyncRead, AsyncWriteExt};
use tokio::process::{ChildStdin, Command};
use tokio::sync::Mutex;
use tokio_util::sync::CancellationToken;
use crate::model::{
IoMode, ManagedRun, ProcessController, ProcessIdentity, ProcessInfo, RestartPolicy, RunSpec,
now_ms,
};
use crate::process::{identity_for_pid, parse_signal, process_context_key, process_info_for_pid};
use crate::store::Store;
static PTY_SPAWN_LOCK: StdMutex<()> = StdMutex::new(());
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub struct ExecOutput {
pub argv: Vec<String>,
pub process: ProcessIdentity,
pub process_context_key: String,
pub controller: ProcessController,
pub purpose: Option<String>,
pub labels: std::collections::BTreeMap<String, String>,
pub exit_code: i32,
pub stdout: String,
pub stderr: String,
pub duration_ms: u64,
pub timed_out: bool,
}
#[derive(Debug, Clone)]
pub struct ExecRequest {
pub argv: Vec<String>,
pub cwd: Option<String>,
pub env: std::collections::BTreeMap<String, String>,
pub shell: bool,
pub timeout: Option<Duration>,
pub purpose: Option<String>,
pub labels: std::collections::BTreeMap<String, String>,
pub controller: ProcessController,
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub struct RunLogs {
pub id: String,
pub stdout: String,
pub stderr: String,
pub stdout_truncated: bool,
pub stderr_truncated: bool,
pub tail_bytes: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
pub struct InputReceipt {
pub id: String,
pub recipients: usize,
pub bytes: usize,
}
#[derive(Clone)]
enum InputSink {
Pipes(Arc<Mutex<ChildStdin>>),
Pty(Arc<StdMutex<Box<dyn Write + Send>>>),
}
#[derive(Clone)]
struct ReplicaControl {
identity: ProcessIdentity,
input: Option<InputSink>,
}
#[derive(Clone)]
struct RunControl {
cancel: CancellationToken,
replicas: Vec<ReplicaControl>,
}
type ExitFuture = Pin<Box<dyn Future<Output = i32> + Send>>;
struct Spawned {
identity: ProcessIdentity,
input: Option<InputSink>,
exit: ExitFuture,
pumps: Vec<tokio::task::JoinHandle<()>>,
}
struct Monitor {
id: String,
spec: RunSpec,
stdout: PathBuf,
stderr: PathBuf,
replica: u32,
cancel: CancellationToken,
}
struct RunManagerInner {
store: Arc<Store>,
handles: Mutex<HashMap<String, RunControl>>,
logs: PathBuf,
}
#[derive(Clone)]
pub struct RunManager {
inner: Arc<RunManagerInner>,
}
impl RunManager {
pub fn open_default() -> Result<Self, String> {
Self::new(Arc::new(Store::open_default()?))
}
pub fn new(store: Arc<Store>) -> Result<Self, String> {
let logs = store
.path()
.parent()
.unwrap_or_else(|| Path::new("."))
.join("runs");
std::fs::create_dir_all(&logs).map_err(|error| error.to_string())?;
Ok(Self {
inner: Arc::new(RunManagerInner {
store,
handles: Mutex::new(HashMap::new()),
logs,
}),
})
}
pub fn store(&self) -> Arc<Store> {
Arc::clone(&self.inner.store)
}
pub async fn recover(&self) -> Result<(), String> {
for mut run in self.inner.store.list_runs()? {
if terminal(&run.status) {
continue;
}
let alive = run
.processes
.iter()
.filter(|identity| process_matches(identity))
.cloned()
.collect::<Vec<_>>();
if alive.is_empty() {
run.status = "interrupted".to_string();
run.processes.clear();
run.updated_at = now_ms();
self.inner.store.put_run(&run)?;
continue;
}
let snapshots = alive
.iter()
.map(|identity| run_process_info(identity, &run.spec))
.collect::<Vec<_>>();
self.inner.store.record_processes(
&snapshots,
"execution.recover",
"result",
None,
&run_controller(&run.spec),
"managed",
Some(&run.id),
&serde_json::json!({ "recovered": snapshots.len() }),
)?;
let cancel = CancellationToken::new();
let replicas = alive
.iter()
.cloned()
.map(|identity| {
let token = cancel.child_token();
(
ReplicaControl {
identity: identity.clone(),
input: None,
},
identity,
token,
)
})
.collect::<Vec<_>>();
self.inner.handles.lock().await.insert(
run.id.clone(),
RunControl {
cancel,
replicas: replicas
.iter()
.map(|(control, _, _)| control.clone())
.collect(),
},
);
for (_, identity, token) in replicas {
let manager = self.clone();
let id = run.id.clone();
tokio::spawn(async move {
loop {
if token.is_cancelled() {
let _ = signal_group(identity.pid, libc::SIGTERM);
}
if !process_matches(&identity) {
manager.remove_replica(&id, &identity).await;
manager.remove_control_if_idle(&id).await;
let status = if token.is_cancelled() {
"stopped"
} else {
"completed"
};
let _ = manager.refresh_run(&id, status, None).await;
break;
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
});
}
}
Ok(())
}
pub async fn is_active(&self, id: &str) -> bool {
self.inner.handles.lock().await.contains_key(id)
}
pub async fn start(&self, spec: RunSpec) -> Result<ManagedRun, String> {
self.start_with_id(uuid::Uuid::now_v7().to_string(), spec)
.await
}
pub async fn start_with_id(&self, id: String, mut spec: RunSpec) -> Result<ManagedRun, String> {
validate_spec(&spec)?;
if spec.replicas == 0 {
spec.replicas = 1;
}
let directory = self.inner.logs.join(&id);
std::fs::create_dir_all(&directory).map_err(|error| error.to_string())?;
let stdout = directory.join("stdout.log");
let stderr = directory.join("stderr.log");
let created_at = now_ms();
let run = ManagedRun {
id: id.clone(),
spec: spec.clone(),
status: "starting".to_string(),
processes: Vec::new(),
process_history: Vec::new(),
created_at,
updated_at: created_at,
exit_code: None,
stdout_path: Some(stdout.to_string_lossy().into_owned()),
stderr_path: Some(stderr.to_string_lossy().into_owned()),
};
self.inner.store.put_run(&run)?;
let cancel = CancellationToken::new();
if let Some(timeout_ms) = spec.timeout_ms {
let deadline = cancel.clone();
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(timeout_ms)).await;
deadline.cancel();
});
}
let mut controls: Vec<ReplicaControl> = Vec::new();
let mut snapshots = Vec::new();
let mut monitors = Vec::new();
for replica in 0..spec.replicas {
let spawned = match spawn_once(&spec, &stdout, &stderr, replica).await {
Ok(spawned) => spawned,
Err(error) => {
for control in &controls {
let _ = signal_group(control.identity.pid, libc::SIGKILL);
}
let mut failed = self.get(&id)?;
failed.status = "failed".to_string();
failed.process_history = controls
.iter()
.map(|control| control.identity.clone())
.collect();
failed.updated_at = now_ms();
self.inner.store.put_run(&failed)?;
return Err(error);
}
};
snapshots.push(run_process_info(&spawned.identity, &spec));
let token = cancel.child_token();
controls.push(ReplicaControl {
identity: spawned.identity.clone(),
input: spawned.input.clone(),
});
monitors.push((
Monitor {
id: id.clone(),
spec: spec.clone(),
stdout: stdout.clone(),
stderr: stderr.clone(),
replica,
cancel: token,
},
spawned,
));
}
let controller = run_controller(&spec);
let details = launch_details(&spec)?;
if let Err(error) = self.inner.store.record_processes(
&snapshots,
"execution.start",
"result",
None,
&controller,
"managed",
Some(&id),
&details,
) {
for control in &controls {
let _ = signal_group(control.identity.pid, libc::SIGKILL);
}
return Err(format!(
"Started run processes but could not persist their context: {error}"
));
}
self.inner.handles.lock().await.insert(
id.clone(),
RunControl {
cancel,
replicas: controls,
},
);
self.refresh_run(&id, "running", None).await?;
let run = self.get(&id)?;
for (monitor, spawned) in monitors {
self.monitor(monitor, spawned);
}
Ok(run)
}
pub fn list(&self) -> Result<Vec<ManagedRun>, String> {
self.inner.store.list_runs()
}
pub fn get(&self, id: &str) -> Result<ManagedRun, String> {
self.inner
.store
.get_run(id)?
.ok_or_else(|| format!("Run \"{id}\" not found"))
}
pub fn record_use(
&self,
run: &ManagedRun,
operation: &str,
phase: &str,
controller: &ProcessController,
details: &serde_json::Value,
) -> Result<(), String> {
let identities = if run.processes.is_empty() {
&run.process_history
} else {
&run.processes
};
let snapshots = identities
.iter()
.map(|identity| run_process_info(identity, &run.spec))
.collect::<Vec<_>>();
self.inner.store.record_processes(
&snapshots,
operation,
phase,
None,
controller,
"managed",
Some(&run.id),
details,
)?;
Ok(())
}
pub async fn signal(
&self,
id: &str,
signal: &str,
controller: &ProcessController,
) -> Result<ManagedRun, String> {
let signal_name = signal.to_string();
let signal = parse_signal(signal)?;
let value = signal_to_raw(signal)?;
let run = self.get(id)?;
let handles = self.inner.handles.lock().await;
let replicas = handles
.get(id)
.ok_or_else(|| format!("Run \"{id}\" is not active in this daemon"))?
.replicas
.as_slice();
let snapshots = replicas
.iter()
.map(|replica| run_process_info(&replica.identity, &run.spec))
.collect::<Vec<_>>();
self.inner.store.record_processes(
&snapshots,
"execution.signal",
"intent",
None,
controller,
"managed",
Some(id),
&serde_json::json!({ "signal": signal_name }),
)?;
for replica in replicas {
signal_group(replica.identity.pid, value)?;
}
drop(handles);
self.inner.store.record_processes(
&snapshots,
"execution.signal",
"result",
None,
controller,
"managed",
Some(id),
&serde_json::json!({ "signal": signal_name, "applied": true }),
)?;
self.get(id)
}
pub async fn stop(
&self,
id: &str,
timeout: Duration,
controller: &ProcessController,
) -> Result<ManagedRun, String> {
let run = self.get(id)?;
let control = self
.inner
.handles
.lock()
.await
.get(id)
.cloned()
.ok_or_else(|| format!("Run \"{id}\" is not active in this daemon"))?;
let snapshots = control
.replicas
.iter()
.map(|control| run_process_info(&control.identity, &run.spec))
.collect::<Vec<_>>();
self.inner.store.record_processes(
&snapshots,
"execution.stop",
"intent",
None,
controller,
"managed",
Some(id),
&serde_json::json!({ "timeout_ms": timeout.as_millis() }),
)?;
control.cancel.cancel();
for replica in &control.replicas {
let _ = signal_group(replica.identity.pid, libc::SIGTERM);
}
let deadline = tokio::time::Instant::now() + timeout;
loop {
let run = self.get(id)?;
if run.processes.is_empty() || tokio::time::Instant::now() >= deadline {
break;
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
for replica in control.replicas {
if process_matches(&replica.identity) {
let _ = signal_group(replica.identity.pid, libc::SIGKILL);
}
}
self.refresh_run(id, "stopped", None).await?;
self.inner.store.record_processes(
&snapshots,
"execution.stop",
"result",
None,
controller,
"managed",
Some(id),
&serde_json::json!({ "stopped": true }),
)?;
self.get(id)
}
pub async fn input(
&self,
id: &str,
data: &[u8],
controller: &ProcessController,
) -> Result<InputReceipt, String> {
let run = self.get(id)?;
let sinks = self
.inner
.handles
.lock()
.await
.get(id)
.ok_or_else(|| format!("Run \"{id}\" is not active in this daemon"))?
.replicas
.iter()
.filter_map(|control| control.input.clone())
.collect::<Vec<_>>();
let snapshots = run
.processes
.iter()
.map(|identity| run_process_info(identity, &run.spec))
.collect::<Vec<_>>();
self.inner.store.record_processes(
&snapshots,
"execution.input",
"intent",
None,
controller,
"managed",
Some(id),
&serde_json::json!({ "bytes": data.len() }),
)?;
for sink in &sinks {
match sink {
InputSink::Pipes(stdin) => stdin
.lock()
.await
.write_all(data)
.await
.map_err(|error| error.to_string())?,
InputSink::Pty(writer) => writer
.lock()
.map_err(|error| error.to_string())?
.write_all(data)
.map_err(|error| error.to_string())?,
}
}
Ok(InputReceipt {
id: id.to_string(),
recipients: sinks.len(),
bytes: data.len(),
})
}
pub fn logs(&self, id: &str, tail_bytes: usize) -> Result<RunLogs, String> {
let run = self.get(id)?;
let (stdout, stdout_truncated) = read_tail(run.stdout_path.as_deref(), tail_bytes)?;
let (stderr, stderr_truncated) = read_tail(run.stderr_path.as_deref(), tail_bytes)?;
Ok(RunLogs {
id: id.to_string(),
stdout,
stderr,
stdout_truncated,
stderr_truncated,
tail_bytes,
})
}
pub async fn wait(&self, id: &str, timeout: Option<Duration>) -> Result<ManagedRun, String> {
let started = tokio::time::Instant::now();
loop {
let run = self.get(id)?;
if terminal(&run.status) {
return Ok(run);
}
if timeout.is_some_and(|timeout| started.elapsed() >= timeout) {
return Err(format!("Timed out waiting for run \"{id}\""));
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
}
pub async fn remove(&self, id: &str) -> Result<bool, String> {
if self.inner.handles.lock().await.contains_key(id) {
return Err(format!("Run \"{id}\" is active; stop it before removal"));
}
self.inner.store.delete_run(id)
}
fn monitor(&self, monitor: Monitor, mut spawned: Spawned) {
let manager = self.clone();
tokio::spawn(async move {
let Monitor {
id,
spec,
stdout,
stderr,
replica,
cancel,
} = monitor;
let mut attempt = 1_u32;
loop {
let identity = spawned.identity.clone();
let code = tokio::select! {
code = &mut spawned.exit => code,
_ = cancel.cancelled() => {
let _ = signal_group(identity.pid, libc::SIGTERM);
match tokio::time::timeout(Duration::from_secs(5), &mut spawned.exit).await {
Ok(code) => code,
Err(_) => {
let _ = signal_group(identity.pid, libc::SIGKILL);
spawned.exit.await
}
}
}
};
for pump in spawned.pumps.drain(..) {
let _ = pump.await;
}
manager.remove_replica(&id, &identity).await;
spawned.input = None;
let restart = !cancel.is_cancelled()
&& attempt < spec.max_attempts.max(1)
&& match spec.restart {
RestartPolicy::Never => false,
RestartPolicy::OnFailure => code != 0,
RestartPolicy::Always => true,
};
let snapshot = run_process_info(&identity, &spec);
if let Err(error) = manager.inner.store.record_processes(
std::slice::from_ref(&snapshot),
"execution.exit",
"result",
None,
&run_controller(&spec),
"managed",
Some(&id),
&serde_json::json!({
"exit_code": code,
"cancelled": cancel.is_cancelled(),
"restart": restart
}),
) {
manager.remove_control_if_idle(&id).await;
let _ = manager
.refresh_run(&id, &format!("context_record_failed: {error}"), Some(code))
.await;
break;
}
if !restart {
let status = if cancel.is_cancelled() {
"stopped"
} else if code == 0 {
"completed"
} else {
"failed"
};
manager.remove_control_if_idle(&id).await;
let _ = manager.refresh_run(&id, status, Some(code)).await;
break;
}
let exponent = attempt.saturating_sub(1).min(31);
let delay = spec
.retry_backoff_ms
.saturating_mul(1_u64 << exponent)
.min(spec.retry_max_backoff_ms);
if delay > 0 {
tokio::select! {
_ = tokio::time::sleep(Duration::from_millis(delay)) => {}
_ = cancel.cancelled() => {}
}
}
if cancel.is_cancelled() {
manager.remove_control_if_idle(&id).await;
let _ = manager.refresh_run(&id, "stopped", Some(code)).await;
break;
}
match spawn_once(&spec, &stdout, &stderr, replica).await {
Ok(next) => {
attempt = attempt.saturating_add(1);
let identity = next.identity.clone();
let input = next.input.clone();
spawned = next;
if manager.add_replica(&id, identity, input).await {
let _ = manager.refresh_run(&id, "running", Some(code)).await;
}
}
Err(error) => {
manager.remove_control_if_idle(&id).await;
let status = if cancel.is_cancelled() {
"stopped".to_string()
} else {
format!("restart_failed: {error}")
};
let _ = manager.refresh_run(&id, &status, Some(code)).await;
break;
}
}
}
});
}
async fn add_replica(
&self,
id: &str,
identity: ProcessIdentity,
input: Option<InputSink>,
) -> bool {
let mut handles = self.inner.handles.lock().await;
let Some(control) = handles.get_mut(id) else {
return false;
};
if control.cancel.is_cancelled() {
return false;
}
control.replicas.push(ReplicaControl { identity, input });
true
}
async fn remove_replica(&self, id: &str, identity: &ProcessIdentity) {
let mut handles = self.inner.handles.lock().await;
if let Some(control) = handles.get_mut(id) {
control
.replicas
.retain(|replica| replica.identity != *identity);
}
}
async fn remove_control_if_idle(&self, id: &str) {
let mut handles = self.inner.handles.lock().await;
if handles
.get(id)
.is_some_and(|control| control.replicas.is_empty())
{
handles.remove(id);
}
}
async fn refresh_run(
&self,
id: &str,
status: &str,
exit_code: Option<i32>,
) -> Result<(), String> {
let processes = self
.inner
.handles
.lock()
.await
.get(id)
.map(|control| {
control
.replicas
.iter()
.map(|control| control.identity.clone())
.collect::<Vec<_>>()
})
.unwrap_or_default();
let mut run = self.get(id)?;
run.status = status.to_string();
for process in &processes {
if !run.process_history.contains(process) {
run.process_history.push(process.clone());
}
}
run.processes = processes;
run.updated_at = now_ms();
run.exit_code = exit_code;
self.inner.store.put_run(&run)
}
}
pub async fn exec(request: ExecRequest, store: Option<Arc<Store>>) -> Result<ExecOutput, String> {
let ExecRequest {
argv,
cwd,
env,
shell,
timeout,
purpose,
labels,
controller,
} = request;
let spec = RunSpec {
name: "exec".to_string(),
purpose: purpose.clone(),
labels: labels.clone(),
requested_by: Some(controller.clone()),
argv: argv.clone(),
cwd: cwd.clone(),
env: env.clone(),
shell,
io: IoMode::Pipes,
lifetime: crate::model::RunLifetime::Execution,
restart: RestartPolicy::Never,
timeout_ms: timeout.map(|value| value.as_millis() as u64),
replicas: 1,
max_attempts: 1,
retry_backoff_ms: 500,
retry_max_backoff_ms: 30_000,
};
validate_spec(&spec)?;
let mut command = command(&spec)?;
command
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true);
let started = tokio::time::Instant::now();
let child = command.spawn().map_err(|error| error.to_string())?;
let pid = child
.id()
.ok_or_else(|| "Spawned command has no process ID".to_string())?;
let identity = await_identity(pid).await?;
let snapshot = run_process_info(&identity, &spec);
let store = match store {
Some(store) => store,
None => Arc::new(Store::open_default()?),
};
store.record_processes(
std::slice::from_ref(&snapshot),
"execution.command",
"intent",
None,
&controller,
"foreground",
None,
&launch_details(&spec)?,
)?;
let context_key = process_context_key(&identity);
let output = if let Some(timeout) = timeout {
match tokio::time::timeout(timeout, child.wait_with_output()).await {
Ok(output) => (output.map_err(|error| error.to_string())?, false),
Err(_) => {
let output = ExecOutput {
argv,
process: identity,
process_context_key: context_key,
controller,
purpose,
labels,
exit_code: 124,
stdout: String::new(),
stderr: "Command timed out".to_string(),
duration_ms: started.elapsed().as_millis() as u64,
timed_out: true,
};
store.record_processes(
std::slice::from_ref(&snapshot),
"execution.command",
"result",
None,
&output.controller,
"foreground",
None,
&serde_json::to_value(&output).map_err(|error| error.to_string())?,
)?;
return Ok(output);
}
}
} else {
(
child
.wait_with_output()
.await
.map_err(|error| error.to_string())?,
false,
)
};
let output = ExecOutput {
argv,
process: identity,
process_context_key: context_key,
controller,
purpose,
labels,
exit_code: output.0.status.code().unwrap_or(128),
stdout: String::from_utf8_lossy(&output.0.stdout).into_owned(),
stderr: String::from_utf8_lossy(&output.0.stderr).into_owned(),
duration_ms: started.elapsed().as_millis() as u64,
timed_out: output.1,
};
store.record_processes(
std::slice::from_ref(&snapshot),
"execution.command",
"result",
None,
&output.controller,
"foreground",
None,
&serde_json::to_value(&output).map_err(|error| error.to_string())?,
)?;
Ok(output)
}
async fn spawn_once(
spec: &RunSpec,
stdout: &Path,
stderr: &Path,
_replica: u32,
) -> Result<Spawned, String> {
match spec.io {
IoMode::Pipes => spawn_pipes(spec, stdout, stderr).await,
IoMode::Pty => spawn_pty(spec, stdout).await,
}
}
async fn spawn_pipes(spec: &RunSpec, stdout: &Path, stderr: &Path) -> Result<Spawned, String> {
let mut command = command(spec)?;
command
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(unix)]
unsafe {
command.pre_exec(|| {
if libc::setpgid(0, 0) == -1 {
return Err(std::io::Error::last_os_error());
}
Ok(())
});
}
let mut child = command.spawn().map_err(|error| error.to_string())?;
let pid = child
.id()
.ok_or_else(|| "Spawned command has no process ID".to_string())?;
let identity = await_identity(pid).await?;
let stdin = child
.stdin
.take()
.map(|stdin| InputSink::Pipes(Arc::new(Mutex::new(stdin))));
let mut pumps = Vec::new();
if let Some(reader) = child.stdout.take() {
pumps.push(pump(reader, stdout.to_path_buf()));
}
if let Some(reader) = child.stderr.take() {
pumps.push(pump(reader, stderr.to_path_buf()));
}
Ok(Spawned {
identity,
input: stdin,
pumps,
exit: Box::pin(async move {
child
.wait()
.await
.map(|status| status.code().unwrap_or(128))
.unwrap_or(127)
}),
})
}
async fn spawn_pty(spec: &RunSpec, stdout: &Path) -> Result<Spawned, String> {
let spec = spec.clone();
let stdout = stdout.to_path_buf();
tokio::task::spawn_blocking(move || {
let spawn = PTY_SPAWN_LOCK
.lock()
.map_err(|error| format!("PTY spawn lock failed: {error}"))?;
let mut file = append_file(&stdout).map_err(|error| {
format!(
"Could not open PTY stdout log {}: {error}",
stdout.display()
)
})?;
let pair = native_pty_system()
.openpty(PtySize {
rows: 24,
cols: 120,
pixel_width: 0,
pixel_height: 0,
})
.map_err(|error| error.to_string())?;
let mut builder = pty_command(&spec)?;
if let Some(cwd) = &spec.cwd {
builder.cwd(cwd);
}
for (name, value) in &spec.env {
builder.env(name, value);
}
let master = pair.master;
let mut reader = master
.try_clone_reader()
.map_err(|error| error.to_string())?;
let writer = master.take_writer().map_err(|error| error.to_string())?;
let exited = Arc::new(AtomicBool::new(false));
let reader_exited = Arc::clone(&exited);
let pump = std::thread::Builder::new()
.name(format!("apoc-pty-log-{}", spec.name))
.spawn(move || {
let mut buffer = [0_u8; 8192];
let mut terminal_misses = 0_u32;
loop {
match reader.read(&mut buffer) {
Ok(0) => {
if reader_exited.load(Ordering::Acquire) {
terminal_misses = terminal_misses.saturating_add(1);
if terminal_misses >= 100 {
break;
}
}
std::thread::sleep(Duration::from_millis(1));
}
Ok(size) => {
terminal_misses = 0;
let _ = file.write_all(&buffer[..size]);
let _ = file.flush();
}
Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
Err(_) => {
if reader_exited.load(Ordering::Acquire) {
terminal_misses = terminal_misses.saturating_add(1);
if terminal_misses >= 100 {
break;
}
}
std::thread::sleep(Duration::from_millis(1));
}
}
}
drop(master);
})
.map_err(|error| error.to_string())?;
let mut child = pair
.slave
.spawn_command(builder)
.map_err(|error| error.to_string())?;
drop(pair.slave);
let pid = child
.process_id()
.ok_or_else(|| "PTY command has no process ID".to_string())?;
let input = InputSink::Pty(Arc::new(StdMutex::new(writer)));
let exit = Box::pin(async move {
tokio::task::spawn_blocking(move || {
let code = child
.wait()
.map(|status| status.exit_code() as i32)
.unwrap_or(127);
exited.store(true, Ordering::Release);
code
})
.await
.unwrap_or(127)
}) as ExitFuture;
drop(spawn);
Ok((pid, input, exit, pump))
})
.await
.map_err(|error| error.to_string())?
.map(|(pid, input, exit, pump)| Spawned {
identity: identity_for_pid(pid).unwrap_or_else(|_| fallback_identity(pid)),
input: Some(input),
exit,
pumps: vec![tokio::task::spawn_blocking(move || {
let _ = pump.join();
})],
})
}
fn command(spec: &RunSpec) -> Result<Command, String> {
let mut command = if spec.shell {
let mut command = Command::new("/bin/zsh");
command.args(["-lc", &spec.argv[0]]);
command
} else {
let mut command = Command::new(&spec.argv[0]);
command.args(&spec.argv[1..]);
command
};
if let Some(cwd) = &spec.cwd {
command.current_dir(cwd);
}
command.envs(&spec.env);
Ok(command)
}
fn pty_command(spec: &RunSpec) -> Result<CommandBuilder, String> {
if spec.shell {
let mut command = CommandBuilder::new("/bin/zsh");
command.args(["-lc", &spec.argv[0]]);
Ok(command)
} else {
Ok(CommandBuilder::from_argv(
spec.argv.iter().map(std::ffi::OsString::from).collect(),
))
}
}
fn validate_spec(spec: &RunSpec) -> Result<(), String> {
if spec.argv.is_empty() {
return Err("argv must contain an executable".to_string());
}
if spec.shell && spec.argv.len() != 1 {
return Err("shell mode accepts one zsh command string in argv".to_string());
}
if spec.replicas > 256 {
return Err("replicas cannot exceed 256".to_string());
}
if spec.max_attempts == 0 {
return Err("max_attempts must be at least 1".to_string());
}
if spec.retry_max_backoff_ms < spec.retry_backoff_ms {
return Err("retry_max_backoff_ms cannot be less than retry_backoff_ms".to_string());
}
Ok(())
}
fn pump(
reader: impl AsyncRead + Unpin + Send + 'static,
path: PathBuf,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let Ok(mut file) = tokio::fs::OpenOptions::new()
.create(true)
.append(true)
.open(path)
.await
else {
return;
};
let mut reader = reader;
let _ = tokio::io::copy(&mut reader, &mut file).await;
let _ = file.flush().await;
})
}
fn append_file(path: &Path) -> std::io::Result<std::fs::File> {
std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(path)
}
async fn await_identity(pid: u32) -> Result<ProcessIdentity, String> {
for _ in 0..20 {
if let Ok(identity) = identity_for_pid(pid) {
return Ok(identity);
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
Ok(fallback_identity(pid))
}
fn fallback_identity(pid: u32) -> ProcessIdentity {
let started_at = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default();
ProcessIdentity {
boot_id: format!("boot-{}", sysinfo::System::boot_time()),
pid,
ppid: Some(std::process::id()),
pgid: Some(pid),
started_at: started_at.as_secs(),
started_at_ns: started_at.as_nanos().min(u64::MAX as u128) as u64,
name: "managed".to_string(),
executable: None,
}
}
fn run_controller(spec: &RunSpec) -> ProcessController {
spec.requested_by
.clone()
.unwrap_or_else(|| ProcessController::capture(false, None))
}
fn run_process_info(identity: &ProcessIdentity, spec: &RunSpec) -> ProcessInfo {
if let Ok(mut process) = process_info_for_pid(identity.pid) {
process.identity = identity.clone();
return process;
}
ProcessInfo {
identity: identity.clone(),
status: "starting_or_exited".to_string(),
user_id: None,
effective_user_id: None,
group_id: None,
session_id: None,
cpu_percent: 0.0,
memory_bytes: 0,
virtual_memory_bytes: 0,
runtime_seconds: 0,
command: if spec.shell {
vec![
"/bin/zsh".to_string(),
"-lc".to_string(),
spec.argv.first().cloned().unwrap_or_default(),
]
} else {
spec.argv.clone()
},
cwd: spec.cwd.clone().or_else(|| {
std::env::current_dir()
.ok()
.map(|path| path.to_string_lossy().into_owned())
}),
disk_read_bytes: 0,
disk_written_bytes: 0,
}
}
fn launch_details(spec: &RunSpec) -> Result<serde_json::Value, String> {
let mut environment = std::env::vars_os().collect::<std::collections::BTreeMap<_, _>>();
environment.extend(
spec.env
.iter()
.map(|(name, value)| (name.into(), value.into())),
);
let mut digest = Sha256::new();
for (name, value) in &environment {
let name = name.as_encoded_bytes();
let value = value.as_encoded_bytes();
digest.update((name.len() as u64).to_be_bytes());
digest.update(name);
digest.update((value.len() as u64).to_be_bytes());
digest.update(value);
}
let resolved_cwd = spec.cwd.clone().or_else(|| {
std::env::current_dir()
.ok()
.map(|path| path.to_string_lossy().into_owned())
});
Ok(serde_json::json!({
"spec": spec,
"resolved_argv": if spec.shell {
vec![
"/bin/zsh".to_string(),
"-lc".to_string(),
spec.argv.first().cloned().unwrap_or_default()
]
} else {
spec.argv.clone()
},
"resolved_cwd": resolved_cwd,
"inherits_environment": true,
"environment_keys": environment
.keys()
.map(|name| name.to_string_lossy())
.collect::<Vec<_>>(),
"environment_sha256": format!("{:x}", digest.finalize())
}))
}
fn signal_group(pid: u32, signal: i32) -> Result<(), String> {
let result = unsafe { libc::kill(-(pid as i32), signal) };
if result == -1 {
let group_error = std::io::Error::last_os_error();
let result = unsafe { libc::kill(pid as i32, signal) };
if result == -1 {
return Err(format!(
"Cannot signal process group or PID {pid}: {group_error}; {}",
std::io::Error::last_os_error()
));
}
}
Ok(())
}
fn signal_to_raw(signal: sysinfo::Signal) -> Result<i32, String> {
Ok(match signal {
sysinfo::Signal::Hangup => libc::SIGHUP,
sysinfo::Signal::Interrupt => libc::SIGINT,
sysinfo::Signal::Quit => libc::SIGQUIT,
sysinfo::Signal::Kill => libc::SIGKILL,
sysinfo::Signal::User1 => libc::SIGUSR1,
sysinfo::Signal::User2 => libc::SIGUSR2,
sysinfo::Signal::Term => libc::SIGTERM,
sysinfo::Signal::Continue => libc::SIGCONT,
sysinfo::Signal::Stop => libc::SIGSTOP,
_ => return Err("Signal is not supported by apoc".to_string()),
})
}
fn process_matches(identity: &ProcessIdentity) -> bool {
identity_for_pid(identity.pid).is_ok_and(|current| {
current.boot_id == identity.boot_id
&& current.pid == identity.pid
&& current.started_at_ns == identity.started_at_ns
})
}
fn terminal(status: &str) -> bool {
matches!(status, "completed" | "failed" | "stopped" | "interrupted")
|| status.starts_with("restart_failed:")
|| status.starts_with("context_record_failed:")
}
fn read_tail(path: Option<&str>, bytes: usize) -> Result<(String, bool), String> {
let Some(path) = path else {
return Ok((String::new(), false));
};
let mut file = match std::fs::File::open(path) {
Ok(file) => file,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
return Ok((String::new(), false));
}
Err(error) => return Err(error.to_string()),
};
let length = file.metadata().map_err(|error| error.to_string())?.len();
use std::io::Seek;
file.seek(std::io::SeekFrom::Start(
length.saturating_sub(bytes as u64),
))
.map_err(|error| error.to_string())?;
let mut value = Vec::new();
file.read_to_end(&mut value)
.map_err(|error| error.to_string())?;
Ok((
String::from_utf8_lossy(&value).into_owned(),
length > bytes as u64,
))
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use tempfile::tempdir;
use super::*;
#[tokio::test]
async fn executes_argv_without_a_shell() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let output = exec(
ExecRequest {
argv: vec!["/bin/echo".to_string(), "apocalypse".to_string()],
cwd: None,
env: BTreeMap::new(),
shell: false,
timeout: Some(Duration::from_secs(1)),
purpose: Some("test one-shot context".to_string()),
labels: BTreeMap::new(),
controller: ProcessController::capture(true, Some("test-execution".to_string())),
},
Some(store.clone()),
)
.await
.unwrap();
assert_eq!(output.exit_code, 0);
assert_eq!(output.stdout, "apocalypse\n");
let dossier = store
.process_dossier(&output.process_context_key, 10)
.unwrap_or_else(|error| {
panic!(
"{error}; pid contexts: {:#?}",
store
.process_dossiers_for_pid(output.process.pid, 10)
.unwrap()
)
});
assert_eq!(dossier.events.len(), 2);
}
#[tokio::test]
async fn persists_and_observes_a_managed_run() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = RunManager::new(store).unwrap();
let run = manager
.start(RunSpec {
name: "echo".to_string(),
purpose: Some("test managed context".to_string()),
labels: BTreeMap::new(),
requested_by: None,
argv: vec!["/bin/echo".to_string(), "managed".to_string()],
cwd: None,
env: BTreeMap::new(),
shell: false,
io: IoMode::Pipes,
lifetime: crate::model::RunLifetime::Persistent,
restart: RestartPolicy::Never,
timeout_ms: None,
replicas: 1,
max_attempts: 1,
retry_backoff_ms: 500,
retry_max_backoff_ms: 30_000,
})
.await
.unwrap();
let finished = manager
.wait(&run.id, Some(Duration::from_secs(2)))
.await
.unwrap();
assert_eq!(finished.status, "completed");
assert!(
manager
.logs(&run.id, 1024)
.unwrap()
.stdout
.contains("managed")
);
}
#[tokio::test]
async fn terminal_pty_run_has_flushed_logs() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = RunManager::new(store).unwrap();
let runs = futures::future::join_all((0..64).map(|iteration| {
let manager = manager.clone();
async move {
let run = manager
.start(RunSpec {
name: format!("pty-echo-{iteration}"),
purpose: Some("verify terminal PTY logs".to_string()),
labels: BTreeMap::new(),
requested_by: None,
argv: vec!["/bin/echo".to_string(), "pty-apocalypse".to_string()],
cwd: None,
env: BTreeMap::new(),
shell: false,
io: IoMode::Pty,
lifetime: crate::model::RunLifetime::Persistent,
restart: RestartPolicy::Never,
timeout_ms: None,
replicas: 1,
max_attempts: 1,
retry_backoff_ms: 500,
retry_max_backoff_ms: 30_000,
})
.await
.unwrap();
(iteration, run)
}
}))
.await;
for (iteration, run) in runs {
let finished = manager
.wait(&run.id, Some(Duration::from_secs(2)))
.await
.unwrap();
let logs = manager.logs(&run.id, 1024).unwrap();
assert_eq!(finished.status, "completed", "iteration {iteration}");
assert!(
logs.stdout.contains("pty-apocalypse"),
"iteration {iteration}, run {}, stdout {:?}, stderr {:?}",
run.id,
logs.stdout,
logs.stderr
);
}
}
#[tokio::test]
async fn stops_a_managed_run_during_restart_backoff() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = RunManager::new(store).unwrap();
let run = manager
.start(RunSpec {
name: "restart-backoff".to_string(),
purpose: Some("verify restart cancellation".to_string()),
labels: BTreeMap::new(),
requested_by: None,
argv: vec!["/usr/bin/false".to_string()],
cwd: None,
env: BTreeMap::new(),
shell: false,
io: IoMode::Pipes,
lifetime: crate::model::RunLifetime::Persistent,
restart: RestartPolicy::OnFailure,
timeout_ms: None,
replicas: 1,
max_attempts: 2,
retry_backoff_ms: 500,
retry_max_backoff_ms: 500,
})
.await
.unwrap();
let identity = run.processes[0].clone();
tokio::time::timeout(Duration::from_secs(1), async {
while process_matches(&identity) {
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(25)).await;
let stopped = manager
.stop(
&run.id,
Duration::from_millis(250),
&ProcessController::capture(true, None),
)
.await
.unwrap();
assert_eq!(stopped.status, "stopped");
tokio::time::sleep(Duration::from_millis(600)).await;
let finished = manager.get(&run.id).unwrap();
assert_eq!(finished.status, "stopped");
assert_eq!(finished.process_history.len(), 1);
}
#[tokio::test]
async fn marks_a_managed_run_failed_when_its_process_cannot_spawn() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = RunManager::new(store).unwrap();
let result = manager
.start_with_id(
"missing-run".to_string(),
RunSpec {
name: "missing".to_string(),
purpose: Some("test launch failure".to_string()),
labels: BTreeMap::new(),
requested_by: None,
argv: vec!["/definitely/missing-apoc-executable".to_string()],
cwd: None,
env: BTreeMap::new(),
shell: false,
io: IoMode::Pipes,
lifetime: crate::model::RunLifetime::Persistent,
restart: RestartPolicy::Never,
timeout_ms: None,
replicas: 1,
max_attempts: 1,
retry_backoff_ms: 500,
retry_max_backoff_ms: 30_000,
},
)
.await;
assert!(result.is_err());
assert_eq!(manager.get("missing-run").unwrap().status, "failed");
}
}