use std::collections::{BTreeSet, HashMap, VecDeque};
use std::io::{Read, Seek, SeekFrom, Write};
use std::os::fd::{FromRawFd, OwnedFd};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::Arc;
use std::time::Duration;
use incurs_codemode::ArtifactStore;
use serde::Serialize;
use sha2::{Digest, Sha256};
use tokio::io::{AsyncRead, AsyncWriteExt};
use tokio::process::{Child, Command as TokioCommand};
use crate::model::{
CausalEvent, CommandTopology, CommandUnit, Execution, ExecutionAttempt, ExecutionBudget,
ExecutionListPage, ExecutionProgram, ExecutionProgramSummary, ExecutionSpec, ExecutionStatus,
ExecutionSummary, Lease, LeaseOwnerKind, ProcessController, RestartPolicy, RetryPolicy,
RunLifetime, RunSpec, SCHEMA_VERSION, WorkspaceContext, now_ms,
};
use crate::process::{ProcessManager, process_context_key};
use crate::run::{InputReceipt, RunLogs, RunManager};
use crate::store::{ExecutionClaim, Store};
#[derive(Clone)]
pub struct ExecutionManager {
store: Arc<Store>,
runs: RunManager,
}
#[derive(Debug, Serialize)]
struct CommandOutcome {
id: String,
execution_id: Option<String>,
status: String,
exit_code: Option<i32>,
stderr_tail: Option<String>,
}
struct PipelineStage {
id: String,
execution: Execution,
child: Child,
stderr: PathBuf,
stderr_pump: Option<tokio::task::JoinHandle<()>>,
stdout_pump: Option<tokio::task::JoinHandle<()>>,
deadline: Option<tokio::time::Instant>,
exit_code: Option<i32>,
}
struct BatchStage {
id: String,
execution: Execution,
child: Child,
stderr: PathBuf,
deadline: Option<tokio::time::Instant>,
timed_out: bool,
}
impl CommandOutcome {
fn launch_failure(id: String, error: String) -> Self {
Self {
id,
execution_id: None,
status: "failed".to_string(),
exit_code: None,
stderr_tail: Some(error),
}
}
}
impl ExecutionManager {
pub fn new(store: Arc<Store>, runs: RunManager) -> Self {
Self { store, runs }
}
pub fn recover(&self) -> Result<(), String> {
for parent in self.store.list_execution_records()? {
if terminal(parent.status) {
continue;
}
let ExecutionProgram::Commands { commands, topology } = &parent.spec.program else {
if self.store.get_run(&parent.id)?.is_none() {
self.fail_missing_run(parent)?;
}
continue;
};
let children = self
.store
.list_execution_records()?
.into_iter()
.filter(|execution| execution.parent_id.as_deref() == Some(&parent.id))
.filter_map(|execution| {
execution
.spec
.labels
.get("apoc.command_id")
.cloned()
.map(|id| (id, execution))
})
.collect::<HashMap<_, _>>();
let manager = self.clone();
let parent = parent.clone();
let commands = commands.clone();
let topology = topology.clone();
tokio::spawn(async move {
manager
.resume_commands(parent, commands, topology, children)
.await;
});
}
Ok(())
}
fn fail_missing_run(&self, mut execution: Execution) -> Result<(), String> {
execution.status = ExecutionStatus::Failed;
execution.updated_at = now_ms();
execution.last_progress_at = execution.updated_at;
execution.result = Some(serde_json::json!({
"error_code": "EXECUTION_RECOVERY_MISSING_RUN",
"error": "Accepted execution has no durable managed run after daemon recovery"
}));
self.store.put_execution_record(&execution)?;
self.store.release_lease(&lease_id(&execution.id))?;
let controller = ProcessController::capture(false, Some(execution.id.clone()));
self.event(
&execution,
&controller,
"execution.failed",
execution.result.clone().unwrap_or_default(),
)?;
Ok(())
}
pub async fn start(
&self,
spec: ExecutionSpec,
controller: ProcessController,
parent_id: Option<String>,
session_id: Option<String>,
) -> Result<Execution, String> {
self.start_with_replay(spec, controller, parent_id, session_id)
.await
.map(|(execution, _)| execution)
}
pub async fn start_with_replay(
&self,
mut spec: ExecutionSpec,
controller: ProcessController,
parent_id: Option<String>,
session_id: Option<String>,
) -> Result<(Execution, bool), String> {
validate_spec(&spec)?;
let fingerprint = execution_fingerprint(&spec, session_id.as_deref())?;
enrich_workspace(&mut spec.workspace);
let id = uuid::Uuid::now_v7().to_string();
let now = now_ms();
let mut execution = Execution {
id: id.clone(),
parent_id: parent_id.clone(),
session_id: session_id.clone(),
spec: spec.clone(),
status: ExecutionStatus::Queued,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
if let Some(key) = spec.idempotency_key.as_deref() {
match self.store.claim_execution(&execution, key, &fingerprint)? {
ExecutionClaim::Claimed => {}
ExecutionClaim::Replayed(execution) => return Ok((*execution, true)),
}
} else {
self.store.put_execution_record(&execution)?;
}
self.store.put_lease(&Lease {
id: lease_id(&id),
owner_kind: LeaseOwnerKind::Execution,
owner_id: id.clone(),
resources: Vec::new(),
created_at: now,
renewed_at: now,
expires_at: spec.expires_at,
released: false,
})?;
self.event(
&execution,
&controller,
"execution.accepted",
serde_json::json!({ "schema_version": spec.schema_version }),
)?;
if let ExecutionProgram::Commands { commands, topology } = &spec.program {
execution.status = ExecutionStatus::Running;
execution.updated_at = now_ms();
execution.last_progress_at = execution.updated_at;
self.store.put_execution_record(&execution)?;
self.event(
&execution,
&controller,
"execution.running",
serde_json::json!({ "commands": commands.len() }),
)?;
self.execute_commands(
execution.clone(),
commands.clone(),
topology.clone(),
controller,
);
return Ok((execution, false));
}
let run_spec = match run_spec(&spec, &controller) {
Ok(run_spec) => run_spec,
Err(error) => {
return self
.fail_start(execution, &controller, error)
.map(|execution| (execution, false));
}
};
let run = match self.runs.start_with_id(id, run_spec).await {
Ok(run) => run,
Err(error) => {
return self
.fail_start(execution, &controller, error)
.map(|execution| (execution, false));
}
};
let execution = self.sync_run(execution, &run, Some(&controller))?;
self.finalize(execution.id.clone(), controller.clone());
self.supervise_budgets(
execution.id.clone(),
execution.spec.budget.clone(),
controller,
);
Ok((execution, false))
}
fn fail_start(
&self,
mut execution: Execution,
controller: &ProcessController,
error: String,
) -> Result<Execution, String> {
execution.status = ExecutionStatus::Failed;
execution.updated_at = now_ms();
execution.last_progress_at = execution.updated_at;
execution.result = Some(serde_json::json!({
"error_code": "EXECUTION_START_FAILED",
"error": error
}));
self.store.put_execution_record(&execution)?;
self.store.release_lease(&lease_id(&execution.id))?;
self.event(
&execution,
controller,
"execution.failed",
execution.result.clone().unwrap_or_default(),
)?;
Ok(execution)
}
pub fn list(&self) -> Result<Vec<Execution>, String> {
self.store
.list_execution_records()?
.into_iter()
.map(|execution| {
let Some(run) = self.runs.store().get_run(&execution.id)? else {
return Ok(execution);
};
self.sync_run(execution, &run, None)
})
.collect()
}
pub fn list_page(
&self,
limit: usize,
cursor: Option<&str>,
status: Option<ExecutionStatus>,
) -> Result<ExecutionListPage, String> {
if !(1..=100).contains(&limit) {
return Err("limit must be between 1 and 100".to_string());
}
let before = cursor.map(parse_execution_cursor).transpose()?;
let candidates = self.store.list_execution_records_page(
before.as_ref().map(|(at, id)| (*at, id.as_str())),
status.map(status_name),
limit + 1,
)?;
let mut executions = Vec::new();
let mut bytes = 0_usize;
let mut has_more = candidates.len() > limit;
for execution in candidates.into_iter().take(limit) {
let execution = match self.runs.store().get_run(&execution.id)? {
Some(run) => self.sync_run(execution, &run, None)?,
None => execution,
};
let summary = execution_summary(&execution);
let summary_bytes = serde_json::to_vec(&summary)
.map_err(|error| error.to_string())?
.len();
if !executions.is_empty() && bytes + summary_bytes > 128 * 1024 {
has_more = true;
break;
}
bytes += summary_bytes;
executions.push(summary);
}
let next_cursor = has_more.then(|| {
let execution = executions
.last()
.expect("bounded page contains a cursor item");
format!("{}:{}", execution.updated_at, execution.id)
});
Ok(ExecutionListPage {
executions,
next_cursor,
})
}
pub fn get(&self, id: &str) -> Result<Execution, String> {
let execution = self
.store
.get_execution_record(id)?
.ok_or_else(|| format!("Execution \"{id}\" not found"))?;
let Some(run) = self.runs.store().get_run(id)? else {
return Ok(execution);
};
self.sync_run(execution, &run, None)
}
pub async fn wait(&self, id: &str, timeout: Option<Duration>) -> Result<Execution, String> {
if self.runs.store().get_run(id)?.is_none() {
let started = tokio::time::Instant::now();
loop {
let execution = self.get(id)?;
if terminal(execution.status) {
return Ok(execution);
}
if timeout.is_some_and(|timeout| started.elapsed() >= timeout) {
return Err(format!("Timed out waiting for execution \"{id}\""));
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
}
let run = self.runs.wait(id, timeout).await?;
let execution = self
.store
.get_execution_record(id)?
.ok_or_else(|| format!("Execution \"{id}\" not found"))?;
self.sync_run(execution, &run, None)
}
pub async fn cancel(
&self,
id: &str,
timeout: Duration,
controller: &ProcessController,
) -> Result<Execution, String> {
let children = self
.store
.list_execution_records()?
.into_iter()
.filter(|execution| execution.parent_id.as_deref() == Some(id))
.map(|execution| execution.id)
.collect::<Vec<_>>();
for child in children {
if self.runs.is_active(&child).await {
let _ = self.runs.stop(&child, timeout, controller).await;
}
let _ = self.store.release_lease(&lease_id(&child));
}
if self.runs.store().get_run(id)?.is_none() {
let mut execution = self
.store
.get_execution_record(id)?
.ok_or_else(|| format!("Execution \"{id}\" not found"))?;
if terminal(execution.status) {
return Ok(execution);
}
execution.status = ExecutionStatus::Canceled;
execution.updated_at = now_ms();
execution.result = Some(serde_json::json!({ "canceled": true }));
self.store.put_execution_record(&execution)?;
self.store.release_lease(&lease_id(id))?;
self.event(
&execution,
controller,
"execution.canceled",
execution.result.clone().unwrap_or_default(),
)?;
return Ok(execution);
}
let run = self.runs.stop(id, timeout, controller).await?;
let execution = self
.store
.get_execution_record(id)?
.ok_or_else(|| format!("Execution \"{id}\" not found"))?;
let execution = self.sync_run(execution, &run, Some(controller))?;
self.store.release_lease(&lease_id(id))?;
Ok(execution)
}
pub async fn signal(
&self,
id: &str,
signal: &str,
controller: &ProcessController,
) -> Result<Execution, String> {
let run = self.runs.signal(id, signal, controller).await?;
let execution = self
.store
.get_execution_record(id)?
.ok_or_else(|| format!("Execution \"{id}\" not found"))?;
self.sync_run(execution, &run, Some(controller))
}
pub async fn input(
&self,
id: &str,
data: &[u8],
controller: &ProcessController,
) -> Result<InputReceipt, String> {
self.runs.input(id, data, controller).await
}
pub fn logs(&self, id: &str, tail_bytes: usize) -> Result<RunLogs, String> {
if self.runs.store().get_run(id)?.is_some() {
return self.runs.logs(id, tail_bytes);
}
let execution = self.get(id)?;
if !matches!(execution.spec.program, ExecutionProgram::Commands { .. }) {
return Err(format!("Execution \"{id}\" has no process logs"));
}
let directory = run_directory(&self.store, id);
let (stdout, stdout_truncated) = read_tail_path(&directory.join("stdout.log"), tail_bytes)?;
let (stderr, stderr_truncated) = read_tail_path(&directory.join("stderr.log"), tail_bytes)?;
Ok(RunLogs {
id: id.to_string(),
stdout,
stderr,
stdout_truncated,
stderr_truncated,
tail_bytes,
})
}
pub async fn remove(&self, id: &str) -> Result<bool, String> {
let execution = self.get(id)?;
if !terminal(execution.status) {
return Err(format!(
"Execution \"{id}\" is active; cancel it before removal"
));
}
self.store.release_lease(&lease_id(id))?;
self.runs.remove(id).await?;
self.store.delete_execution_record(id)
}
fn sync_run(
&self,
mut execution: Execution,
run: &crate::model::ManagedRun,
controller: Option<&ProcessController>,
) -> Result<Execution, String> {
let previous = execution.status;
let next = execution_status(&run.status);
execution.status =
if previous == ExecutionStatus::Stalled && next == ExecutionStatus::Running {
ExecutionStatus::Stalled
} else {
next
};
execution.updated_at = run.updated_at;
execution.last_progress_at = run.updated_at;
for process in &run.process_history {
if execution
.attempts
.iter()
.any(|attempt| attempt.processes.contains(process))
{
continue;
}
let active = run.processes.contains(process);
execution.attempts.push(ExecutionAttempt {
id: uuid::Uuid::now_v7().to_string(),
number: execution.attempts.len() as u32 + 1,
processes: vec![process.clone()],
started_at: process.started_at_ns / 1_000_000,
completed_at: (!active).then_some(run.updated_at),
exit_code: (!active).then_some(run.exit_code).flatten(),
error_code: None,
stdout_artifact: None,
stderr_artifact: None,
});
}
if terminal(execution.status) {
for attempt in &mut execution.attempts {
if attempt.completed_at.is_none() {
attempt.completed_at = Some(run.updated_at);
attempt.exit_code = run.exit_code;
}
}
let output_truncated = execution
.result
.as_ref()
.and_then(|value| value.get("output_truncated"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false)
|| execution
.spec
.budget
.artifact_bytes
.map(|limit| enforce_log_budget(run, limit))
.transpose()?
.unwrap_or(false);
if execution
.result
.as_ref()
.and_then(|value| value.get("error_code"))
.is_none()
{
execution.result = Some(serde_json::json!({
"exit_code": run.exit_code,
"attempts": execution.attempts.len(),
"output_truncated": output_truncated
}));
} else if let Some(serde_json::Value::Object(result)) = &mut execution.result {
result.insert(
"output_truncated".to_string(),
serde_json::Value::Bool(output_truncated),
);
}
self.store.release_lease(&lease_id(&execution.id))?;
} else if let Some(mut lease) = self.store.get_lease(&lease_id(&execution.id))? {
lease.resources = run
.processes
.iter()
.map(|identity| format!("process/{}", process_context_key(identity)))
.collect();
lease.renewed_at = now_ms();
self.store.put_lease(&lease)?;
}
self.store.put_execution_record(&execution)?;
if previous != execution.status {
let controller = controller
.cloned()
.unwrap_or_else(|| ProcessController::capture(false, Some(execution.id.clone())));
self.event(
&execution,
&controller,
&format!("execution.{}", status_name(execution.status)),
execution.result.clone().unwrap_or_default(),
)?;
}
Ok(execution)
}
fn finalize(&self, id: String, controller: ProcessController) {
let manager = self.clone();
tokio::spawn(async move {
let Ok(run) = manager.runs.wait(&id, None).await else {
return;
};
tokio::time::sleep(Duration::from_millis(25)).await;
let Ok(Some(execution)) = manager.store.get_execution_record(&id) else {
return;
};
let _ = manager.sync_run(execution, &run, Some(&controller));
});
}
fn execute_commands(
&self,
parent: Execution,
commands: Vec<CommandUnit>,
topology: CommandTopology,
controller: ProcessController,
) {
let manager = self.clone();
tokio::spawn(async move {
let result = match topology {
CommandTopology::Parallel {
max_parallel,
fail_fast,
} => {
let outcomes = manager
.execute_parallel(
&parent,
commands,
max_parallel as usize,
fail_fast,
&controller,
)
.await;
let failed = outcomes
.iter()
.any(|outcome| matches!(outcome.status.as_str(), "failed" | "canceled"));
Ok((outcomes, failed))
}
CommandTopology::Pipeline { pipefail } => {
manager
.execute_pipeline(&parent, commands, pipefail, &controller)
.await
}
};
match result {
Ok((outcomes, failed)) => {
let _ = manager
.finish_commands(parent, outcomes, failed, &controller)
.await;
}
Err(error) => {
let _ = manager.fail_command_cleanup(parent, error, &controller);
}
}
});
}
async fn resume_commands(
&self,
parent: Execution,
commands: Vec<CommandUnit>,
topology: CommandTopology,
mut children: HashMap<String, Execution>,
) {
let controller = ProcessController::capture(false, Some(parent.id.clone()));
let mut outcomes = Vec::new();
let mut missing = Vec::new();
for command in commands {
if let Some(child) = children.remove(&command.id) {
outcomes.push(self.outcome_from_child(command.id, child).await);
} else {
missing.push(command);
}
}
let failed = match topology {
CommandTopology::Parallel {
max_parallel,
fail_fast,
} => {
if fail_fast && outcomes.iter().any(|outcome| outcome.status == "failed") {
outcomes.extend(
missing
.into_iter()
.map(|command| self.record_skipped_child(&parent, command)),
);
} else {
outcomes.extend(
self.execute_parallel(
&parent,
missing,
max_parallel as usize,
fail_fast,
&controller,
)
.await,
);
}
outcomes
.iter()
.any(|outcome| matches!(outcome.status.as_str(), "failed" | "canceled"))
}
CommandTopology::Pipeline { pipefail } => {
if !missing.is_empty() {
outcomes.extend(missing.into_iter().map(|command| {
CommandOutcome::launch_failure(
command.id,
"Pipeline was interrupted before every stage became durable"
.to_string(),
)
}));
true
} else if pipefail {
outcomes.iter().any(|outcome| outcome.exit_code != Some(0))
} else {
outcomes
.last()
.is_some_and(|outcome| outcome.exit_code != Some(0))
}
}
};
let _ = self
.finish_commands(parent, outcomes, failed, &controller)
.await;
}
async fn outcome_from_child(&self, id: String, child: Execution) -> CommandOutcome {
let finished = if terminal(child.status) {
Ok(child)
} else {
self.wait(&child.id, None).await
};
match finished {
Ok(finished) => CommandOutcome {
id,
execution_id: Some(finished.id.clone()),
status: if finished
.result
.as_ref()
.and_then(|result| result.get("skipped"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false)
{
"skipped".to_string()
} else {
status_name(finished.status).to_string()
},
exit_code: finished
.result
.as_ref()
.and_then(|result| result.get("exit_code"))
.and_then(serde_json::Value::as_i64)
.map(|value| value as i32),
stderr_tail: (finished.status == ExecutionStatus::Failed)
.then(|| self.logs(&finished.id, 4096).ok().map(|logs| logs.stderr))
.flatten(),
},
Err(error) => CommandOutcome::launch_failure(id, error),
}
}
async fn execute_parallel(
&self,
parent: &Execution,
commands: Vec<CommandUnit>,
max_parallel: usize,
fail_fast: bool,
controller: &ProcessController,
) -> Vec<CommandOutcome> {
let mut queued = VecDeque::from(commands);
let mut active: Vec<BatchStage> = Vec::new();
let mut outcomes = Vec::new();
let mut stop_launching = false;
while !queued.is_empty() || !active.is_empty() {
let parent_canceled = self
.store
.get_execution_record(&parent.id)
.ok()
.flatten()
.is_some_and(|execution| execution.status == ExecutionStatus::Canceled);
if parent_canceled {
stop_launching = true;
for stage in &active {
let _ = signal_pipeline(stage.child.id().unwrap_or(0), libc::SIGKILL);
}
}
while !stop_launching && active.len() < max_parallel {
let Some(command) = queued.pop_front() else {
break;
};
match self.launch_batch_stage(parent, command, controller).await {
Ok(stage) => active.push(stage),
Err(outcome) => {
if fail_fast {
stop_launching = true;
}
outcomes.push(outcome);
}
}
}
let mut finished = Vec::new();
for (index, stage) in active.iter_mut().enumerate() {
if stage
.deadline
.is_some_and(|deadline| tokio::time::Instant::now() >= deadline)
{
stage.timed_out = true;
let _ = signal_pipeline(stage.child.id().unwrap_or(0), libc::SIGKILL);
}
match stage.child.try_wait() {
Ok(Some(status)) => finished.push((index, status.code().unwrap_or(128))),
Ok(None) => {}
Err(_) => finished.push((index, 127)),
}
}
for (index, code) in finished.into_iter().rev() {
let stage = active.swap_remove(index);
let outcome = self.finish_batch_stage(stage, code);
if fail_fast && outcome.status == "failed" {
stop_launching = true;
}
outcomes.push(outcome);
}
if active.is_empty() && stop_launching {
break;
}
if !active.is_empty() {
tokio::time::sleep(Duration::from_millis(2)).await;
}
}
outcomes.extend(
queued
.into_iter()
.map(|command| self.record_skipped_child(parent, command)),
);
outcomes
}
async fn launch_batch_stage(
&self,
parent: &Execution,
command: CommandUnit,
controller: &ProcessController,
) -> Result<BatchStage, CommandOutcome> {
let command_id = command.id.clone();
let spec = child_spec(parent, &command);
let id = uuid::Uuid::now_v7().to_string();
let now = now_ms();
let mut execution = Execution {
id: id.clone(),
parent_id: Some(parent.id.clone()),
session_id: parent.session_id.clone(),
spec,
status: ExecutionStatus::Queued,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
self.store
.put_execution_record(&execution)
.map_err(|error| CommandOutcome::launch_failure(command_id.clone(), error))?;
if let Err(error) = self.put_child_lease(&execution) {
return Err(self.record_launch_failed_child(execution, command_id, error));
}
let run_spec = match run_spec(&execution.spec, controller) {
Ok(run_spec) => run_spec,
Err(error) => {
return Err(self.record_launch_failed_child(execution, command_id, error));
}
};
let directory = run_directory(&self.store, &id);
if let Err(error) = std::fs::create_dir_all(&directory) {
return Err(self.record_launch_failed_child(execution, command_id, error.to_string()));
}
let stdout = directory.join("stdout.log");
let stderr = directory.join("stderr.log");
let stdout_file = match std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&stdout)
{
Ok(file) => file,
Err(error) => {
return Err(self.record_launch_failed_child(
execution,
command_id,
error.to_string(),
));
}
};
let stderr_file = match std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&stderr)
{
Ok(file) => file,
Err(error) => {
return Err(self.record_launch_failed_child(
execution,
command_id,
error.to_string(),
));
}
};
let mut process = match pipeline_command(&run_spec) {
Ok(process) => process,
Err(error) => {
return Err(self.record_launch_failed_child(execution, command_id, error));
}
};
process
.stdin(Stdio::null())
.stdout(Stdio::from(stdout_file))
.stderr(Stdio::from(stderr_file));
#[cfg(unix)]
unsafe {
process.pre_exec(|| {
if libc::setpgid(0, 0) == -1 {
return Err(std::io::Error::last_os_error());
}
Ok(())
});
}
let child = match process.spawn() {
Ok(child) => child,
Err(error) => {
return Err(self.record_launch_failed_child(
execution,
command_id,
error.to_string(),
));
}
};
let Some(pid) = child.id() else {
return Err(self.record_launch_failed_child(
execution,
command_id,
"Spawned batch command has no process ID".to_string(),
));
};
let executable = run_spec.argv.first().cloned().unwrap_or_default();
let identity = await_process_identity_fast(pid, &executable).await;
execution.status = ExecutionStatus::Running;
execution.updated_at = now_ms();
execution.last_progress_at = execution.updated_at;
execution.attempts.push(ExecutionAttempt {
id: uuid::Uuid::now_v7().to_string(),
number: 1,
processes: vec![identity.clone()],
started_at: identity.started_at_ns / 1_000_000,
completed_at: None,
exit_code: None,
error_code: None,
stdout_artifact: None,
stderr_artifact: None,
});
self.store
.put_execution_record(&execution)
.map_err(|error| CommandOutcome::launch_failure(command_id.clone(), error))?;
self.store
.put_run(&crate::model::ManagedRun {
id,
spec: run_spec,
status: "running".to_string(),
processes: vec![identity.clone()],
process_history: vec![identity],
created_at: now,
updated_at: now_ms(),
exit_code: None,
stdout_path: Some(stdout.to_string_lossy().into_owned()),
stderr_path: Some(stderr.to_string_lossy().into_owned()),
})
.map_err(|error| CommandOutcome::launch_failure(command_id.clone(), error))?;
Ok(BatchStage {
id: command_id,
execution,
child,
stderr,
deadline: command
.timeout_ms
.map(|timeout| tokio::time::Instant::now() + Duration::from_millis(timeout)),
timed_out: false,
})
}
fn record_launch_failed_child(
&self,
mut execution: Execution,
command_id: String,
error: String,
) -> CommandOutcome {
let execution_id = execution.id.clone();
execution.status = ExecutionStatus::Failed;
execution.updated_at = now_ms();
execution.last_progress_at = execution.updated_at;
execution.result = Some(serde_json::json!({
"error_code": "EXECUTION_START_FAILED",
"error": error
}));
if let Err(lease_error) = self.put_child_lease(&execution) {
return CommandOutcome {
id: command_id,
execution_id: Some(execution_id),
status: "failed".to_string(),
exit_code: None,
stderr_tail: Some(format!(
"{error}; failed to persist child lease: {lease_error}"
)),
};
}
if let Err(persist_error) = self.store.put_execution_record(&execution) {
return CommandOutcome::launch_failure(
command_id,
format!("{error}; failed to persist launch failure: {persist_error}"),
);
}
if let Err(release_error) = self.store.release_lease(&lease_id(&execution_id)) {
return CommandOutcome {
id: command_id,
execution_id: Some(execution_id),
status: "failed".to_string(),
exit_code: None,
stderr_tail: Some(format!(
"{error}; failed to release child lease: {release_error}"
)),
};
}
CommandOutcome {
id: command_id,
execution_id: Some(execution_id),
status: "failed".to_string(),
exit_code: None,
stderr_tail: Some(error),
}
}
fn finish_batch_stage(&self, mut stage: BatchStage, code: i32) -> CommandOutcome {
let status = if code == 0 && !stage.timed_out {
ExecutionStatus::Completed
} else {
ExecutionStatus::Failed
};
stage.execution.status = status;
stage.execution.updated_at = now_ms();
stage.execution.last_progress_at = stage.execution.updated_at;
stage.execution.result = Some(serde_json::json!({
"exit_code": code,
"attempts": 1,
"timed_out": stage.timed_out
}));
if let Some(attempt) = stage.execution.attempts.first_mut() {
attempt.completed_at = Some(stage.execution.updated_at);
attempt.exit_code = Some(code);
}
let persist = self
.store
.put_execution_record(&stage.execution)
.and_then(|_| {
let mut run = self
.store
.get_run(&stage.execution.id)?
.ok_or_else(|| "Batch child run disappeared".to_string())?;
run.status = if status == ExecutionStatus::Completed {
"completed".to_string()
} else {
"failed".to_string()
};
run.processes.clear();
run.updated_at = stage.execution.updated_at;
run.exit_code = Some(code);
self.store.put_run(&run)
});
if let Err(error) = persist {
return CommandOutcome::launch_failure(stage.id, error);
}
if let Err(error) = self.store.release_lease(&lease_id(&stage.execution.id)) {
return CommandOutcome {
id: stage.id,
execution_id: Some(stage.execution.id),
status: "failed".to_string(),
exit_code: Some(code),
stderr_tail: Some(format!("Failed to release child lease: {error}")),
};
}
CommandOutcome {
id: stage.id,
execution_id: Some(stage.execution.id),
status: status_name(status).to_string(),
exit_code: Some(code),
stderr_tail: (code != 0)
.then(|| {
read_tail_path(&stage.stderr, 4096)
.ok()
.map(|(value, _)| value)
})
.flatten(),
}
}
fn record_skipped_child(&self, parent: &Execution, command: CommandUnit) -> CommandOutcome {
let id = uuid::Uuid::now_v7().to_string();
let now = now_ms();
let execution = Execution {
id: id.clone(),
parent_id: Some(parent.id.clone()),
session_id: parent.session_id.clone(),
spec: child_spec(parent, &command),
status: ExecutionStatus::Canceled,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: Some(serde_json::json!({ "skipped": true, "reason": "fail_fast" })),
};
match self
.store
.put_execution_record(&execution)
.and_then(|_| self.put_child_lease(&execution))
.and_then(|_| self.store.release_lease(&lease_id(&id)).map(|_| ()))
{
Ok(()) => CommandOutcome {
id: command.id,
execution_id: Some(id),
status: "skipped".to_string(),
exit_code: None,
stderr_tail: None,
},
Err(error) => CommandOutcome::launch_failure(command.id, error),
}
}
fn put_child_lease(&self, execution: &Execution) -> Result<(), String> {
let id = lease_id(&execution.id);
if self.store.get_lease(&id)?.is_some() {
return Ok(());
}
self.store.put_lease(&Lease {
id,
owner_kind: LeaseOwnerKind::Execution,
owner_id: execution
.parent_id
.clone()
.unwrap_or_else(|| execution.id.clone()),
resources: Vec::new(),
created_at: execution.created_at,
renewed_at: execution.updated_at,
expires_at: execution.spec.expires_at,
released: false,
})
}
async fn execute_pipeline(
&self,
parent: &Execution,
commands: Vec<CommandUnit>,
pipefail: bool,
controller: &ProcessController,
) -> Result<(Vec<CommandOutcome>, bool), String> {
match self
.run_native_pipeline(parent, commands.clone(), controller)
.await
{
Ok(outcomes) => {
let failed = if pipefail {
outcomes.iter().any(|outcome| outcome.exit_code != Some(0))
} else {
outcomes
.last()
.is_some_and(|outcome| outcome.exit_code != Some(0))
};
Ok((outcomes, failed))
}
Err(error) => self
.record_pipeline_launch_failure(parent, commands.clone(), &error)
.map(|outcomes| (outcomes, true))
.map_err(|persist_error| {
format!("{error}; failed to persist pipeline launch cleanup: {persist_error}")
}),
}
}
fn fail_command_cleanup(
&self,
mut parent: Execution,
error: String,
controller: &ProcessController,
) -> Result<(), String> {
let current = self.get(&parent.id)?;
if current.status == ExecutionStatus::Canceled {
return Ok(());
}
parent.status = ExecutionStatus::Failed;
parent.updated_at = now_ms();
parent.last_progress_at = parent.updated_at;
parent.result = Some(serde_json::json!({
"error_code": "EXECUTION_CLEANUP_FAILED",
"error": error,
"children_complete": false
}));
self.store.put_execution_record(&parent)?;
self.store.release_lease(&lease_id(&parent.id))?;
self.event(
&parent,
controller,
"execution.failed",
parent.result.clone().unwrap_or_default(),
)?;
Ok(())
}
fn record_pipeline_launch_failure(
&self,
parent: &Execution,
commands: Vec<CommandUnit>,
error: &str,
) -> Result<Vec<CommandOutcome>, String> {
let mut children = self
.store
.list_execution_records()?
.into_iter()
.filter(|execution| execution.parent_id.as_deref() == Some(&parent.id))
.filter_map(|execution| {
execution
.spec
.labels
.get("apoc.command_id")
.cloned()
.map(|id| (id, execution))
})
.collect::<HashMap<_, _>>();
let mut outcomes = Vec::with_capacity(commands.len());
for command in commands {
let command_id = command.id.clone();
let now = now_ms();
let mut execution = children.remove(&command_id).unwrap_or_else(|| Execution {
id: uuid::Uuid::now_v7().to_string(),
parent_id: Some(parent.id.clone()),
session_id: parent.session_id.clone(),
spec: child_spec(parent, &command),
status: ExecutionStatus::Queued,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
});
if let Some(mut run) = self.store.get_run(&execution.id)? {
for process in &run.processes {
let _ = signal_pipeline(process.pid, libc::SIGKILL);
}
run.status = "failed".to_string();
run.processes.clear();
run.updated_at = now;
self.store.put_run(&run)?;
}
execution.status = ExecutionStatus::Failed;
execution.updated_at = now;
execution.last_progress_at = now;
for attempt in &mut execution.attempts {
if attempt.completed_at.is_none() {
attempt.completed_at = Some(now);
attempt.error_code = Some("EXECUTION_START_FAILED".to_string());
}
}
execution.result = Some(serde_json::json!({
"error_code": "EXECUTION_START_FAILED",
"error": error
}));
self.store.put_execution_record(&execution)?;
let lease = lease_id(&execution.id);
if self.store.get_lease(&lease)?.is_none() {
self.store.put_lease(&Lease {
id: lease.clone(),
owner_kind: LeaseOwnerKind::Execution,
owner_id: parent.id.clone(),
resources: Vec::new(),
created_at: now,
renewed_at: now,
expires_at: parent.spec.expires_at,
released: false,
})?;
}
self.store.release_lease(&lease)?;
outcomes.push(CommandOutcome {
id: command_id,
execution_id: Some(execution.id),
status: "failed".to_string(),
exit_code: None,
stderr_tail: Some(error.to_string()),
});
}
Ok(outcomes)
}
async fn run_native_pipeline(
&self,
parent: &Execution,
commands: Vec<CommandUnit>,
controller: &ProcessController,
) -> Result<Vec<CommandOutcome>, String> {
let parent_directory = run_directory(&self.store, &parent.id);
std::fs::create_dir_all(&parent_directory).map_err(|error| error.to_string())?;
let parent_stdout = parent_directory.join("stdout.log");
let parent_stderr = parent_directory.join("stderr.log");
let mut pipes = (0..commands.len().saturating_sub(1))
.map(|_| native_pipe())
.collect::<Result<Vec<_>, _>>()?;
let mut stages = Vec::with_capacity(commands.len());
let mut pgid: Option<u32> = None;
for (index, command) in commands.iter().enumerate() {
let spec = child_spec(parent, command);
let run_spec = run_spec(&spec, controller)?;
let id = uuid::Uuid::now_v7().to_string();
let now = now_ms();
let mut execution = Execution {
id: id.clone(),
parent_id: Some(parent.id.clone()),
session_id: parent.session_id.clone(),
spec,
status: ExecutionStatus::Queued,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
self.store.put_execution_record(&execution)?;
self.store.put_lease(&Lease {
id: lease_id(&id),
owner_kind: LeaseOwnerKind::Execution,
owner_id: parent.id.clone(),
resources: Vec::new(),
created_at: now,
renewed_at: now,
expires_at: parent.spec.expires_at,
released: false,
})?;
let directory = run_directory(&self.store, &id);
std::fs::create_dir_all(&directory).map_err(|error| error.to_string())?;
let stderr = directory.join("stderr.log");
let stdout = (index + 1 == commands.len()).then(|| parent_stdout.clone());
let mut process = pipeline_command(&run_spec)?;
if index == 0 {
process.stdin(Stdio::null());
} else {
process.stdin(Stdio::from(pipes[index - 1].0.take().ok_or_else(|| {
"Pipeline read end was already consumed".to_string()
})?));
}
if index + 1 == commands.len() {
process.stdout(Stdio::piped());
} else {
process.stdout(Stdio::from(pipes[index].1.take().ok_or_else(|| {
"Pipeline write end was already consumed".to_string()
})?));
}
process.stderr(Stdio::piped());
let requested_group = pgid;
#[cfg(unix)]
unsafe {
process.pre_exec(move || {
let group = requested_group.unwrap_or(0) as i32;
if libc::setpgid(0, group) == -1 {
return Err(std::io::Error::last_os_error());
}
Ok(())
});
}
let mut child = process.spawn().map_err(|error| {
if let Some(pgid) = pgid {
let _ = signal_pipeline(pgid, libc::SIGKILL);
}
error.to_string()
})?;
let pid = child
.id()
.ok_or_else(|| "Spawned pipeline stage has no process ID".to_string())?;
pgid.get_or_insert(pid);
let identity = await_process_identity(pid).await;
let stderr_pump = child
.stderr
.take()
.map(|reader| pump_log(reader, stderr.clone()));
let stdout_pump = child
.stdout
.take()
.map(|reader| pump_log(reader, parent_stdout.clone()));
execution.status = ExecutionStatus::Running;
execution.updated_at = now_ms();
execution.last_progress_at = execution.updated_at;
execution.attempts.push(ExecutionAttempt {
id: uuid::Uuid::now_v7().to_string(),
number: 1,
processes: vec![identity.clone()],
started_at: identity.started_at_ns / 1_000_000,
completed_at: None,
exit_code: None,
error_code: None,
stdout_artifact: None,
stderr_artifact: None,
});
self.store.put_execution_record(&execution)?;
self.store.put_run(&crate::model::ManagedRun {
id: id.clone(),
spec: run_spec,
status: "running".to_string(),
processes: vec![identity.clone()],
process_history: vec![identity.clone()],
created_at: now,
updated_at: now_ms(),
exit_code: None,
stdout_path: stdout.map(|path| path.to_string_lossy().into_owned()),
stderr_path: Some(stderr.to_string_lossy().into_owned()),
})?;
stages.push(PipelineStage {
id: command.id.clone(),
execution,
child,
stderr,
stderr_pump,
stdout_pump,
deadline: command
.timeout_ms
.map(|timeout| tokio::time::Instant::now() + Duration::from_millis(timeout)),
exit_code: None,
});
}
drop(pipes);
let group = pgid.ok_or_else(|| "Pipeline has no process group".to_string())?;
let mut canceled = false;
while stages.iter().any(|stage| stage.exit_code.is_none()) {
if self
.store
.get_execution_record(&parent.id)?
.is_some_and(|execution| execution.status == ExecutionStatus::Canceled)
|| stages.iter().any(|stage| {
stage.exit_code.is_none()
&& stage
.deadline
.is_some_and(|deadline| tokio::time::Instant::now() >= deadline)
})
{
canceled = true;
let _ = signal_pipeline(group, libc::SIGKILL);
}
for stage in &mut stages {
if stage.exit_code.is_some() {
continue;
}
if let Some(status) = stage.child.try_wait().map_err(|error| error.to_string())? {
stage.exit_code = Some(status.code().unwrap_or(128));
}
}
if stages.iter().any(|stage| stage.exit_code.is_none()) {
tokio::time::sleep(Duration::from_millis(10)).await;
}
}
let mut outcomes = Vec::with_capacity(stages.len());
for mut stage in stages {
if let Some(pump) = stage.stderr_pump.take() {
let _ = pump.await;
}
if let Some(pump) = stage.stdout_pump.take() {
let _ = pump.await;
}
let code = stage.exit_code.unwrap_or(127);
let status = if canceled {
ExecutionStatus::Canceled
} else if code == 0 {
ExecutionStatus::Completed
} else {
ExecutionStatus::Failed
};
stage.execution.status = status;
stage.execution.updated_at = now_ms();
stage.execution.last_progress_at = stage.execution.updated_at;
stage.execution.result = Some(serde_json::json!({ "exit_code": code, "attempts": 1 }));
if let Some(attempt) = stage.execution.attempts.first_mut() {
attempt.completed_at = Some(stage.execution.updated_at);
attempt.exit_code = Some(code);
}
self.store.put_execution_record(&stage.execution)?;
if let Some(mut run) = self.store.get_run(&stage.execution.id)? {
run.status = if status == ExecutionStatus::Completed {
"completed".to_string()
} else if status == ExecutionStatus::Canceled {
"stopped".to_string()
} else {
"failed".to_string()
};
run.processes.clear();
run.updated_at = stage.execution.updated_at;
run.exit_code = Some(code);
self.store.put_run(&run)?;
}
self.store.release_lease(&lease_id(&stage.execution.id))?;
outcomes.push(CommandOutcome {
id: stage.id,
execution_id: Some(stage.execution.id),
status: status_name(status).to_string(),
exit_code: Some(code),
stderr_tail: (code != 0)
.then(|| read_tail_path(&stage.stderr, 4096).map(|(value, _)| value))
.transpose()?,
});
}
if !parent_stderr.exists() {
std::fs::File::create(parent_stderr).map_err(|error| error.to_string())?;
}
Ok(outcomes)
}
async fn finish_commands(
&self,
mut parent: Execution,
outcomes: Vec<CommandOutcome>,
parent_failed: bool,
controller: &ProcessController,
) -> Result<(), String> {
let current = self.get(&parent.id)?;
if current.status == ExecutionStatus::Canceled {
return Ok(());
}
let completed = outcomes
.iter()
.filter(|outcome| outcome.status == "completed")
.count();
let failed = outcomes
.iter()
.filter(|outcome| outcome.status == "failed")
.count();
let canceled = outcomes
.iter()
.filter(|outcome| outcome.status == "canceled")
.count();
let skipped = outcomes
.iter()
.filter(|outcome| outcome.status == "skipped")
.count();
let artifact = self
.store
.put(
&parent.id,
&serde_json::to_value(&outcomes).map_err(|error| error.to_string())?,
)
.await?;
parent.status = if parent_failed {
ExecutionStatus::Failed
} else {
ExecutionStatus::Completed
};
parent.updated_at = now_ms();
parent.last_progress_at = parent.updated_at;
parent.result = Some(serde_json::json!({
"parent_id": parent.id.clone(),
"counts": {
"total": outcomes.len(),
"completed": completed,
"failed": failed,
"canceled": canceled,
"skipped": skipped
},
"failures": outcomes.iter().filter(|outcome| outcome.status == "failed").collect::<Vec<_>>(),
"children_artifact": artifact.id
}));
self.store.put_execution_record(&parent)?;
self.store.release_lease(&lease_id(&parent.id))?;
self.event(
&parent,
controller,
&format!("execution.{}", status_name(parent.status)),
parent.result.clone().unwrap_or_default(),
)?;
Ok(())
}
fn supervise_budgets(
&self,
id: String,
budget: ExecutionBudget,
controller: ProcessController,
) {
if budget.progress_timeout_ms.is_none() && budget.child_processes.is_none() {
return;
}
let manager = self.clone();
tokio::spawn(async move {
let interval =
Duration::from_millis(budget.progress_timeout_ms.unwrap_or(100).clamp(10, 250));
let mut processes = ProcessManager::with_store(manager.store.clone());
loop {
tokio::time::sleep(interval).await;
let Ok(Some(mut execution)) = manager.store.get_execution_record(&id) else {
return;
};
if terminal(execution.status) {
return;
}
if let Some(limit) = budget.child_processes {
let Ok(run) = manager.runs.get(&id) else {
return;
};
let mut identities = BTreeSet::new();
for root in &run.processes {
let Ok(tree) = processes.tree(&format!("pid:{}", root.pid)) else {
continue;
};
if tree.first().map(|process| &process.identity) != Some(root) {
continue;
}
identities.extend(
tree.into_iter()
.map(|process| process_context_key(&process.identity)),
);
}
if identities.len() > limit as usize {
let details = serde_json::json!({
"error_code": "EXECUTION_CHILD_PROCESS_BUDGET",
"limit": limit,
"observed": identities.len()
});
let _ = manager
.cancel(&id, Duration::from_secs(5), &controller)
.await;
if let Ok(Some(mut execution)) = manager.store.get_execution_record(&id) {
execution.result = Some(details.clone());
execution.updated_at = now_ms();
let _ = manager.store.put_execution_record(&execution);
let _ = manager.event(
&execution,
&controller,
"execution.budget_exceeded",
details,
);
}
return;
}
}
if let Some(timeout_ms) = budget.progress_timeout_ms
&& now_ms().saturating_sub(execution.last_progress_at) >= timeout_ms
{
if execution.status != ExecutionStatus::Stalled {
execution.status = ExecutionStatus::Stalled;
execution.updated_at = now_ms();
execution.result = Some(serde_json::json!({
"error_code": "EXECUTION_PROGRESS_TIMEOUT",
"progress_timeout_ms": timeout_ms
}));
let _ = manager.store.put_execution_record(&execution);
let _ = manager.event(
&execution,
&controller,
"execution.stalled",
execution.result.clone().unwrap_or_default(),
);
}
if budget.cancel_on_stall {
let _ = manager
.cancel(&id, Duration::from_secs(5), &controller)
.await;
}
return;
}
}
});
}
fn event(
&self,
execution: &Execution,
controller: &ProcessController,
kind: &str,
details: serde_json::Value,
) -> Result<CausalEvent, String> {
self.store.append_causal_event(CausalEvent {
sequence: 0,
id: uuid::Uuid::now_v7().to_string(),
at: now_ms(),
kind: kind.to_string(),
subject: format!("execution/{}", execution.id),
actor: controller.actor.clone(),
correlation_id: execution.id.clone(),
causation_id: None,
execution_id: Some(execution.id.clone()),
session_id: execution.session_id.clone(),
payload_id: None,
details,
})
}
}
fn child_spec(parent: &Execution, command: &CommandUnit) -> ExecutionSpec {
let mut labels = parent.spec.labels.clone();
labels.insert("apoc.command_id".to_string(), command.id.clone());
let mut env = parent.spec.env.clone();
env.extend(command.env.clone());
let mut secrets = parent.spec.secrets.clone();
secrets.extend(command.secrets.clone());
ExecutionSpec {
schema_version: SCHEMA_VERSION,
idempotency_key: None,
name: command.id.clone(),
purpose: parent.spec.purpose.clone(),
labels,
program: ExecutionProgram::Command {
argv: command.argv.clone(),
shell: command.shell,
},
workspace: WorkspaceContext {
cwd: command
.cwd
.clone()
.or_else(|| parent.spec.workspace.cwd.clone()),
..parent.spec.workspace.clone()
},
env,
secrets,
io: crate::model::IoMode::Pipes,
owner_kind: LeaseOwnerKind::Execution,
owner_id: Some(parent.id.clone()),
expires_at: parent.spec.expires_at,
retry: RetryPolicy::default(),
budget: ExecutionBudget {
wall_time_ms: command.timeout_ms,
artifact_bytes: parent.spec.budget.artifact_bytes,
..ExecutionBudget::default()
},
}
}
fn run_directory(store: &Store, id: &str) -> PathBuf {
store
.path()
.parent()
.unwrap_or_else(|| Path::new("."))
.join("runs")
.join(id)
}
fn native_pipe() -> Result<(Option<OwnedFd>, Option<OwnedFd>), String> {
let mut descriptors = [0_i32; 2];
if unsafe { libc::pipe(descriptors.as_mut_ptr()) } == -1 {
return Err(std::io::Error::last_os_error().to_string());
}
Ok(unsafe {
(
Some(OwnedFd::from_raw_fd(descriptors[0])),
Some(OwnedFd::from_raw_fd(descriptors[1])),
)
})
}
fn pipeline_command(spec: &RunSpec) -> Result<TokioCommand, String> {
let mut command = if spec.shell {
let mut command = TokioCommand::new("/bin/zsh");
command.args(["-lc", &spec.argv[0]]);
command
} else {
let executable = spec
.argv
.first()
.ok_or_else(|| "argv must contain an executable".to_string())?;
let mut command = TokioCommand::new(executable);
command.args(&spec.argv[1..]);
command
};
if let Some(cwd) = &spec.cwd {
command.current_dir(cwd);
}
command.envs(&spec.env);
Ok(command)
}
async fn await_process_identity(pid: u32) -> crate::model::ProcessIdentity {
for _ in 0..20 {
if let Ok(identity) = crate::process::identity_for_pid(pid) {
return identity;
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
let started_at = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default();
crate::model::ProcessIdentity {
boot_id: format!("boot-{}", sysinfo::System::boot_time()),
pid,
ppid: Some(std::process::id()),
pgid: Some(pid),
started_at: started_at.as_secs(),
started_at_ns: started_at.as_nanos().min(u64::MAX as u128) as u64,
name: "pipeline".to_string(),
executable: None,
}
}
async fn await_process_identity_fast(pid: u32, executable: &str) -> crate::model::ProcessIdentity {
for _ in 0..2 {
if let Ok(identity) = crate::process::identity_for_spawn(pid, executable) {
return identity;
}
tokio::time::sleep(Duration::from_millis(1)).await;
}
let started_at = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default();
crate::model::ProcessIdentity {
boot_id: format!("boot-{}", sysinfo::System::boot_time()),
pid,
ppid: Some(std::process::id()),
pgid: Some(pid),
started_at: started_at.as_secs(),
started_at_ns: started_at.as_nanos().min(u64::MAX as u128) as u64,
name: "batch".to_string(),
executable: None,
}
}
fn pump_log(
mut reader: impl AsyncRead + Unpin + Send + 'static,
path: PathBuf,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let Ok(mut file) = tokio::fs::OpenOptions::new()
.create(true)
.append(true)
.open(path)
.await
else {
return;
};
let _ = tokio::io::copy(&mut reader, &mut file).await;
let _ = file.flush().await;
})
}
fn signal_pipeline(pgid: u32, signal: i32) -> Result<(), String> {
if unsafe { libc::kill(-(pgid as i32), signal) } == -1 {
let error = std::io::Error::last_os_error();
if error.raw_os_error() != Some(libc::ESRCH) {
return Err(error.to_string());
}
}
Ok(())
}
fn read_tail_path(path: &Path, tail_bytes: usize) -> Result<(String, bool), String> {
let mut file = match std::fs::File::open(path) {
Ok(file) => file,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
return Ok((String::new(), false));
}
Err(error) => return Err(error.to_string()),
};
let length = file.metadata().map_err(|error| error.to_string())?.len();
file.seek(SeekFrom::Start(
length.saturating_sub(tail_bytes.min(u64::MAX as usize) as u64),
))
.map_err(|error| error.to_string())?;
let mut bytes = Vec::new();
file.read_to_end(&mut bytes)
.map_err(|error| error.to_string())?;
Ok((
String::from_utf8_lossy(&bytes).into_owned(),
length > tail_bytes as u64,
))
}
fn enforce_log_budget(run: &crate::model::ManagedRun, limit: u64) -> Result<bool, String> {
let stdout = file_size(run.stdout_path.as_deref())?;
let stderr = file_size(run.stderr_path.as_deref())?;
if stdout.saturating_add(stderr) <= limit {
return Ok(false);
}
let half = limit / 2;
let (stdout_limit, stderr_limit) = if stdout <= half {
(stdout, limit.saturating_sub(stdout))
} else if stderr <= half {
(limit.saturating_sub(stderr), stderr)
} else {
(limit.saturating_sub(half), half)
};
truncate_tail(run.stdout_path.as_deref(), stdout_limit)?;
truncate_tail(run.stderr_path.as_deref(), stderr_limit)?;
Ok(true)
}
fn file_size(path: Option<&str>) -> Result<u64, String> {
let Some(path) = path else {
return Ok(0);
};
std::fs::metadata(path)
.map(|metadata| metadata.len())
.or_else(|error| {
if error.kind() == std::io::ErrorKind::NotFound {
Ok(0)
} else {
Err(error)
}
})
.map_err(|error| error.to_string())
}
fn truncate_tail(path: Option<&str>, limit: u64) -> Result<(), String> {
let Some(path) = path else {
return Ok(());
};
let mut file = match std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(path)
{
Ok(file) => file,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error.to_string()),
};
let length = file.metadata().map_err(|error| error.to_string())?.len();
if length <= limit {
return Ok(());
}
file.seek(SeekFrom::Start(length.saturating_sub(limit)))
.map_err(|error| error.to_string())?;
let mut tail = Vec::with_capacity(limit.min(usize::MAX as u64) as usize);
file.read_to_end(&mut tail)
.map_err(|error| error.to_string())?;
file.set_len(0).map_err(|error| error.to_string())?;
file.seek(SeekFrom::Start(0))
.map_err(|error| error.to_string())?;
file.write_all(&tail).map_err(|error| error.to_string())?;
file.flush().map_err(|error| error.to_string())
}
fn validate_spec(spec: &ExecutionSpec) -> Result<(), String> {
if spec.schema_version != SCHEMA_VERSION {
return Err(format!(
"Unsupported schema_version {}; expected {SCHEMA_VERSION}",
spec.schema_version
));
}
if spec.retry.max_attempts == 0 {
return Err("max_attempts must be at least 1".to_string());
}
if spec.retry.max_backoff_ms < spec.retry.backoff_ms {
return Err("max_backoff_ms cannot be less than backoff_ms".to_string());
}
if spec.budget.child_processes == Some(0) {
return Err("child_processes must allow the command process".to_string());
}
if let Some(key) = spec.idempotency_key.as_deref()
&& (key.is_empty() || key.len() > 256)
{
return Err("idempotency_key must contain between 1 and 256 bytes".to_string());
}
match &spec.program {
ExecutionProgram::Command { argv, shell } => {
if argv.is_empty() {
return Err("argv must contain an executable".to_string());
}
if *shell && argv.len() != 1 {
return Err("shell mode accepts one zsh command string".to_string());
}
}
ExecutionProgram::Commands { commands, topology } => {
if spec.retry.max_attempts != 1 {
return Err(
"Command-group parents cannot be retried; retry child commands explicitly"
.to_string(),
);
}
if commands.is_empty() {
return Err("commands must contain at least one command".to_string());
}
let mut ids = BTreeSet::new();
for command in commands {
if command.id.is_empty() {
return Err("command ids cannot be empty".to_string());
}
if !ids.insert(&command.id) {
return Err(format!("Duplicate command id \"{}\"", command.id));
}
if command.argv.is_empty() {
return Err(format!(
"Command \"{}\" argv must contain an executable",
command.id
));
}
if command.shell && command.argv.len() != 1 {
return Err(format!(
"Command \"{}\" shell mode accepts one zsh command string",
command.id
));
}
}
if let crate::model::CommandTopology::Parallel { max_parallel, .. } = topology
&& !(1..=256).contains(max_parallel)
{
return Err("max_parallel must be between 1 and 256".to_string());
}
if matches!(topology, crate::model::CommandTopology::Pipeline { .. })
&& spec.io == crate::model::IoMode::Pty
{
return Err("pipelines do not support PTY mode".to_string());
}
}
ExecutionProgram::Javascript { .. } => {
return Err("JavaScript executions must use the Code Mode runtime".to_string());
}
}
Ok(())
}
fn execution_fingerprint(spec: &ExecutionSpec, session_id: Option<&str>) -> Result<String, String> {
let mut spec = spec.clone();
spec.idempotency_key = None;
spec.workspace.repository = None;
spec.workspace.worktree = None;
spec.workspace.commit = None;
spec.workspace.branch = None;
spec.workspace.dirty_digest = None;
let value = serde_json::to_vec(&serde_json::json!({
"spec": spec,
"session_id": session_id
}))
.map_err(|error| error.to_string())?;
Ok(format!("{:x}", Sha256::digest(value)))
}
fn parse_execution_cursor(value: &str) -> Result<(u64, String), String> {
let (updated_at, id) = value
.split_once(':')
.ok_or_else(|| "Invalid execution cursor".to_string())?;
let updated_at = updated_at
.parse::<u64>()
.map_err(|_| "Invalid execution cursor".to_string())?;
if id.is_empty() {
return Err("Invalid execution cursor".to_string());
}
Ok((updated_at, id.to_string()))
}
fn execution_summary(execution: &Execution) -> ExecutionSummary {
let (name, name_truncated) = bounded_text(&execution.spec.name, 256);
let (cwd, cwd_truncated) = execution
.spec
.workspace
.cwd
.as_deref()
.map(|cwd| {
let (cwd, truncated) = bounded_text(cwd, 4096);
(Some(cwd), truncated)
})
.unwrap_or((None, false));
let program = match &execution.spec.program {
ExecutionProgram::Command { argv, .. } => ExecutionProgramSummary {
kind: "command".to_string(),
executable: argv.first().map(|value| bounded_text(value, 1024).0),
argument_count: argv.len().saturating_sub(1),
command_count: 1,
},
ExecutionProgram::Commands { commands, topology } => ExecutionProgramSummary {
kind: match topology {
CommandTopology::Parallel { .. } => "batch",
CommandTopology::Pipeline { .. } => "pipeline",
}
.to_string(),
executable: None,
argument_count: 0,
command_count: commands.len(),
},
ExecutionProgram::Javascript { .. } => ExecutionProgramSummary {
kind: "javascript".to_string(),
executable: None,
argument_count: 0,
command_count: 0,
},
};
let attempt = execution.attempts.last();
let exit_code = execution
.result
.as_ref()
.and_then(|result| result.get("exit_code"))
.and_then(serde_json::Value::as_i64)
.and_then(|value| i32::try_from(value).ok())
.or_else(|| attempt.and_then(|attempt| attempt.exit_code));
ExecutionSummary {
id: execution.id.clone(),
parent_id: execution.parent_id.clone(),
session_id: execution.session_id.clone(),
idempotency_key: execution.spec.idempotency_key.clone(),
name,
name_truncated,
status: execution.status,
program,
cwd,
cwd_truncated,
attempt_count: execution.attempts.len(),
exit_code,
error_code: attempt.and_then(|attempt| attempt.error_code.clone()),
created_at: execution.created_at,
updated_at: execution.updated_at,
}
}
fn bounded_text(value: &str, bytes: usize) -> (String, bool) {
if value.len() <= bytes {
return (value.to_string(), false);
}
let mut end = bytes;
while !value.is_char_boundary(end) {
end -= 1;
}
(value[..end].to_string(), true)
}
fn run_spec(spec: &ExecutionSpec, controller: &ProcessController) -> Result<RunSpec, String> {
let ExecutionProgram::Command { argv, shell } = &spec.program else {
return Err("JavaScript executions must use the Code Mode runtime".to_string());
};
let mut env = spec.env.clone();
for (name, secret) in &spec.secrets {
if secret.provider != "env" {
return Err(format!(
"SECRET_PROVIDER_UNAVAILABLE: provider \"{}\" is not registered",
secret.provider
));
}
let value = std::env::var(&secret.reference).map_err(|_| {
format!(
"SECRET_NOT_FOUND: environment reference \"{}\" is unavailable",
secret.reference
)
})?;
env.insert(name.clone(), value);
}
Ok(RunSpec {
name: spec.name.clone(),
purpose: spec.purpose.clone(),
labels: spec.labels.clone(),
requested_by: Some(controller.clone()),
argv: argv.clone(),
cwd: spec.workspace.cwd.clone(),
env,
shell: *shell,
io: spec.io,
lifetime: match spec.owner_kind {
LeaseOwnerKind::Execution => RunLifetime::Execution,
LeaseOwnerKind::Session | LeaseOwnerKind::Persistent => RunLifetime::Persistent,
},
restart: if spec.retry.max_attempts > 1 {
RestartPolicy::OnFailure
} else {
RestartPolicy::Never
},
timeout_ms: spec.budget.wall_time_ms,
replicas: 1,
max_attempts: spec.retry.max_attempts,
retry_backoff_ms: spec.retry.backoff_ms,
retry_max_backoff_ms: spec.retry.max_backoff_ms,
})
}
pub(crate) fn enrich_workspace(workspace: &mut WorkspaceContext) {
let Some(cwd) = workspace.cwd.as_deref() else {
return;
};
if workspace.repository.is_none() {
workspace.repository = git(cwd, &["rev-parse", "--show-toplevel"]);
}
if workspace.worktree.is_none() {
workspace.worktree = git(cwd, &["rev-parse", "--show-toplevel"]);
}
if workspace.commit.is_none() {
workspace.commit = git(cwd, &["rev-parse", "HEAD"]);
}
if workspace.branch.is_none() {
workspace.branch = git(cwd, &["branch", "--show-current"]);
}
if workspace.dirty_digest.is_none()
&& let Some(status) = git_bytes(cwd, &["status", "--porcelain=v1", "-z"])
&& !status.is_empty()
{
workspace.dirty_digest = Some(format!("{:x}", Sha256::digest(status)));
}
}
fn git(cwd: &str, args: &[&str]) -> Option<String> {
String::from_utf8(git_bytes(cwd, args)?)
.ok()
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
}
fn git_bytes(cwd: &str, args: &[&str]) -> Option<Vec<u8>> {
let output = Command::new("git")
.args(args)
.current_dir(cwd)
.output()
.ok()?;
output.status.success().then_some(output.stdout)
}
fn execution_status(status: &str) -> ExecutionStatus {
match status {
"starting" => ExecutionStatus::Queued,
"running" => ExecutionStatus::Running,
"completed" => ExecutionStatus::Completed,
"stopped" => ExecutionStatus::Canceled,
value if value.starts_with("context_record_failed:") => ExecutionStatus::Interrupted,
_ => ExecutionStatus::Failed,
}
}
fn status_name(status: ExecutionStatus) -> &'static str {
match status {
ExecutionStatus::Queued => "queued",
ExecutionStatus::Running => "running",
ExecutionStatus::Stalled => "stalled",
ExecutionStatus::Interrupted => "interrupted",
ExecutionStatus::Completed => "completed",
ExecutionStatus::Failed => "failed",
ExecutionStatus::Canceled => "canceled",
}
}
fn terminal(status: ExecutionStatus) -> bool {
matches!(
status,
ExecutionStatus::Completed | ExecutionStatus::Failed | ExecutionStatus::Canceled
)
}
fn lease_id(execution_id: &str) -> String {
format!("execution/{execution_id}")
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use tempfile::tempdir;
use super::*;
use crate::model::{
CommandTopology, CommandUnit, ExecutionBudget, IoMode, RetryPolicy, SCHEMA_VERSION,
SecretRef,
};
fn spec(argv: Vec<String>, budget: ExecutionBudget) -> ExecutionSpec {
ExecutionSpec {
schema_version: SCHEMA_VERSION,
idempotency_key: None,
name: "fixture".to_string(),
purpose: Some("verify execution budgets".to_string()),
labels: BTreeMap::new(),
program: ExecutionProgram::Command { argv, shell: false },
workspace: WorkspaceContext::default(),
env: BTreeMap::new(),
secrets: BTreeMap::new(),
io: IoMode::Pipes,
owner_kind: LeaseOwnerKind::Persistent,
owner_id: None,
expires_at: None,
retry: RetryPolicy::default(),
budget,
}
}
#[tokio::test]
async fn runs_one_command_as_a_durable_execution_and_releases_its_lease() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let execution = manager
.start(
ExecutionSpec {
schema_version: SCHEMA_VERSION,
idempotency_key: None,
name: "echo".to_string(),
purpose: Some("verify the execution runtime".to_string()),
labels: BTreeMap::new(),
program: ExecutionProgram::Command {
argv: vec!["/bin/echo".to_string(), "apocalypse".to_string()],
shell: false,
},
workspace: WorkspaceContext {
cwd: Some(directory.path().to_string_lossy().into_owned()),
..WorkspaceContext::default()
},
env: BTreeMap::new(),
secrets: BTreeMap::new(),
io: IoMode::Pipes,
owner_kind: LeaseOwnerKind::Persistent,
owner_id: None,
expires_at: None,
retry: RetryPolicy::default(),
budget: ExecutionBudget::default(),
},
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(2)))
.await
.unwrap();
assert_eq!(finished.status, ExecutionStatus::Completed);
assert_eq!(finished.attempts.len(), 1);
assert!(
store
.get_lease(&lease_id(&execution.id))
.unwrap()
.unwrap()
.released
);
assert!(
manager
.logs(&execution.id, 1024)
.unwrap()
.stdout
.contains("apocalypse")
);
}
#[tokio::test]
async fn returns_failed_execution_when_post_acceptance_spawn_fails() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let missing = directory
.path()
.join("missing")
.join("apoc-definitely-nonexistent-executable");
let execution = manager
.start(
spec(
vec![missing.to_string_lossy().into_owned()],
ExecutionBudget::default(),
),
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
assert_eq!(execution.status, ExecutionStatus::Failed);
assert_eq!(
execution.result.as_ref().unwrap()["error_code"],
"EXECUTION_START_FAILED"
);
assert_eq!(store.list_execution_records().unwrap().len(), 1);
assert!(
store
.get_lease(&lease_id(&execution.id))
.unwrap()
.unwrap()
.released
);
assert_eq!(
store
.causal_events(0, 100)
.unwrap()
.iter()
.filter(|event| event.kind == "execution.failed")
.count(),
1
);
}
#[tokio::test]
async fn returns_failed_execution_when_post_acceptance_secret_resolution_fails() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(vec!["/usr/bin/env".to_string()], ExecutionBudget::default());
execution_spec.secrets.insert(
"TEST_APOC_SECRET".to_string(),
SecretRef {
provider: "env".to_string(),
reference: "APOC_DEFINITELY_MISSING_SECRET_FOR_EXECUTION_TEST".to_string(),
version: None,
},
);
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
assert_eq!(execution.status, ExecutionStatus::Failed);
assert_eq!(
execution.result.as_ref().unwrap()["error_code"],
"EXECUTION_START_FAILED"
);
assert!(
execution.result.as_ref().unwrap()["error"]
.as_str()
.unwrap()
.contains("SECRET_NOT_FOUND")
);
assert!(
store
.get_lease(&lease_id(&execution.id))
.unwrap()
.unwrap()
.released
);
}
#[test]
fn recovery_fails_an_accepted_command_without_a_durable_run() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let now = now_ms();
let execution = Execution {
id: "accepted-without-run".to_string(),
parent_id: None,
session_id: None,
spec: spec(
vec!["/usr/bin/true".to_string()],
ExecutionBudget::default(),
),
status: ExecutionStatus::Queued,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
store.put_execution_record(&execution).unwrap();
store
.put_lease(&Lease {
id: lease_id(&execution.id),
owner_kind: LeaseOwnerKind::Execution,
owner_id: execution.id.clone(),
resources: Vec::new(),
created_at: now,
renewed_at: now,
expires_at: None,
released: false,
})
.unwrap();
manager.recover().unwrap();
manager.recover().unwrap();
let failed = manager.get(&execution.id).unwrap();
assert_eq!(failed.status, ExecutionStatus::Failed);
assert_eq!(
failed.result.unwrap()["error_code"],
"EXECUTION_RECOVERY_MISSING_RUN"
);
assert!(
store
.get_lease(&lease_id(&execution.id))
.unwrap()
.unwrap()
.released
);
assert_eq!(
store
.causal_events(0, 100)
.unwrap()
.iter()
.filter(|event| event.kind == "execution.failed")
.count(),
1
);
}
#[test]
fn cleanup_persistence_failure_is_truthful_on_the_parent() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let now = now_ms();
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![CommandUnit {
id: "child".to_string(),
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
}],
topology: CommandTopology::Pipeline { pipefail: true },
};
let parent = Execution {
id: "cleanup-parent".to_string(),
parent_id: None,
session_id: None,
spec: execution_spec,
status: ExecutionStatus::Running,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
store.put_execution_record(&parent).unwrap();
store
.put_lease(&Lease {
id: lease_id(&parent.id),
owner_kind: LeaseOwnerKind::Execution,
owner_id: parent.id.clone(),
resources: Vec::new(),
created_at: now,
renewed_at: now,
expires_at: None,
released: false,
})
.unwrap();
manager
.fail_command_cleanup(
parent.clone(),
"child cleanup could not be persisted".to_string(),
&ProcessController::capture(true, Some(parent.id.clone())),
)
.unwrap();
let failed = manager.get(&parent.id).unwrap();
assert_eq!(failed.status, ExecutionStatus::Failed);
assert_eq!(failed.result.as_ref().unwrap()["children_complete"], false);
assert_eq!(
failed.result.as_ref().unwrap()["error_code"],
"EXECUTION_CLEANUP_FAILED"
);
assert!(
store
.get_lease(&lease_id(&parent.id))
.unwrap()
.unwrap()
.released
);
}
#[tokio::test]
async fn cancels_an_execution_after_its_progress_deadline() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let execution = manager
.start(
spec(
vec!["/bin/sleep".to_string(), "30".to_string()],
ExecutionBudget {
progress_timeout_ms: Some(25),
cancel_on_stall: true,
..ExecutionBudget::default()
},
),
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(2)))
.await
.unwrap();
assert_eq!(finished.status, ExecutionStatus::Canceled);
assert!(
store
.causal_events(0, 100)
.unwrap()
.iter()
.any(|event| event.kind == "execution.stalled")
);
}
#[tokio::test]
async fn bounds_retained_output_at_terminal_state() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let execution = manager
.start(
spec(
vec![
"/usr/bin/printf".to_string(),
"abcdefghijklmnopqrstuvwxyz".to_string(),
],
ExecutionBudget {
artifact_bytes: Some(10),
..ExecutionBudget::default()
},
),
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
manager
.wait(&execution.id, Some(Duration::from_secs(2)))
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(50)).await;
let finished = manager.get(&execution.id).unwrap();
let logs = manager.logs(&execution.id, 1024).unwrap();
assert!(logs.stdout.len() + logs.stderr.len() <= 10);
assert_eq!(finished.result.unwrap()["output_truncated"], true);
}
#[tokio::test]
async fn cancels_an_execution_that_exceeds_its_child_process_budget() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let execution = manager
.start(
spec(
vec![
"/bin/zsh".to_string(),
"-c".to_string(),
"sleep 30 & wait".to_string(),
],
ExecutionBudget {
child_processes: Some(1),
..ExecutionBudget::default()
},
),
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
manager
.wait(&execution.id, Some(Duration::from_secs(2)))
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(25)).await;
let finished = manager.get(&execution.id).unwrap();
assert_eq!(finished.status, ExecutionStatus::Canceled);
assert_eq!(
finished.result.unwrap()["error_code"],
"EXECUTION_CHILD_PROCESS_BUDGET"
);
}
#[tokio::test]
async fn applies_finite_retries_and_records_every_attempt_identity() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(
vec![
"/bin/zsh".to_string(),
"-c".to_string(),
"exit 7".to_string(),
],
ExecutionBudget::default(),
);
execution_spec.retry = RetryPolicy {
max_attempts: 2,
backoff_ms: 1,
max_backoff_ms: 1,
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(2)))
.await
.unwrap();
assert_eq!(finished.status, ExecutionStatus::Failed);
assert_eq!(finished.attempts.len(), 2);
assert_ne!(
finished.attempts[0].processes[0],
finished.attempts[1].processes[0]
);
}
#[tokio::test]
async fn cancels_an_execution_during_retry_backoff() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(
vec!["/usr/bin/false".to_string()],
ExecutionBudget::default(),
);
execution_spec.retry = RetryPolicy {
max_attempts: 2,
backoff_ms: 500,
max_backoff_ms: 500,
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(100)).await;
let canceled = manager
.cancel(
&execution.id,
Duration::from_millis(250),
&ProcessController::capture(true, None),
)
.await
.unwrap();
assert_eq!(canceled.status, ExecutionStatus::Canceled);
tokio::time::sleep(Duration::from_millis(600)).await;
let finished = manager.get(&execution.id).unwrap();
assert_eq!(finished.status, ExecutionStatus::Canceled);
assert_eq!(finished.attempts.len(), 1);
}
#[tokio::test]
async fn runs_a_bounded_parallel_batch_as_child_executions() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![
CommandUnit {
id: "first".to_string(),
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
},
CommandUnit {
id: "second".to_string(),
argv: vec![
"/bin/zsh".to_string(),
"-c".to_string(),
"exit 7".to_string(),
],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
},
],
topology: CommandTopology::Parallel {
max_parallel: 2,
fail_fast: false,
},
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(5)))
.await
.unwrap();
assert_eq!(finished.status, ExecutionStatus::Failed);
assert_eq!(finished.result.as_ref().unwrap()["counts"]["total"], 2);
assert_eq!(finished.result.as_ref().unwrap()["counts"]["failed"], 1);
let children = manager
.list()
.unwrap()
.into_iter()
.filter(|child| child.parent_id.as_deref() == Some(&execution.id))
.collect::<Vec<_>>();
assert_eq!(children.len(), 2);
assert!(children.iter().all(|child| terminal(child.status)));
assert_child_leases_released(&store, &children);
}
#[tokio::test]
async fn records_parallel_launch_failures_as_terminal_children() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![
CommandUnit {
id: "valid".to_string(),
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
},
CommandUnit {
id: "missing".to_string(),
argv: vec!["/definitely/missing-apoc-executable".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
},
],
topology: CommandTopology::Parallel {
max_parallel: 2,
fail_fast: false,
},
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(5)))
.await
.unwrap();
let children = manager
.list()
.unwrap()
.into_iter()
.filter(|child| child.parent_id.as_deref() == Some(&execution.id))
.collect::<Vec<_>>();
assert_eq!(finished.status, ExecutionStatus::Failed);
assert_eq!(finished.result.as_ref().unwrap()["counts"]["failed"], 1);
assert_eq!(children.len(), 2);
assert!(children.iter().all(|child| terminal(child.status)));
assert_child_leases_released(&store, &children);
let missing = children
.iter()
.find(|child| child.spec.name == "missing")
.unwrap();
assert_eq!(missing.status, ExecutionStatus::Failed);
assert_eq!(
missing.result.as_ref().unwrap()["error_code"],
"EXECUTION_START_FAILED"
);
}
#[tokio::test]
async fn connects_a_linear_pipeline_with_native_pipes_and_pipefail() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store).unwrap());
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![
CommandUnit {
id: "produce".to_string(),
argv: vec!["/usr/bin/printf".to_string(), "apocalypse".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
},
CommandUnit {
id: "transform".to_string(),
argv: vec![
"/usr/bin/tr".to_string(),
"[:lower:]".to_string(),
"[:upper:]".to_string(),
],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
},
],
topology: CommandTopology::Pipeline { pipefail: true },
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(5)))
.await
.unwrap();
assert_eq!(finished.status, ExecutionStatus::Completed);
assert_eq!(
manager.logs(&execution.id, 1024).unwrap().stdout,
"APOCALYPSE"
);
let children = manager
.list()
.unwrap()
.into_iter()
.filter(|child| child.parent_id.as_deref() == Some(&execution.id))
.collect::<Vec<_>>();
assert_eq!(children.len(), 2);
assert!(
children
.iter()
.all(|child| child.status == ExecutionStatus::Completed)
);
}
#[tokio::test]
async fn records_pipeline_launch_failures_as_terminal_children() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![
CommandUnit {
id: "started".to_string(),
argv: vec!["/bin/sleep".to_string(), "30".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
},
CommandUnit {
id: "missing".to_string(),
argv: vec!["/definitely/missing-apoc-pipeline".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
},
],
topology: CommandTopology::Pipeline { pipefail: true },
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(5)))
.await
.unwrap();
let children = manager
.list()
.unwrap()
.into_iter()
.filter(|child| child.parent_id.as_deref() == Some(&execution.id))
.collect::<Vec<_>>();
assert_eq!(finished.status, ExecutionStatus::Failed);
assert_eq!(children.len(), 2);
assert!(children.iter().all(|child| terminal(child.status)));
assert!(children.iter().all(|child| {
store
.get_lease(&lease_id(&child.id))
.unwrap()
.is_some_and(|lease| lease.released)
}));
assert!(
finished.result.as_ref().unwrap()["failures"]
.as_array()
.unwrap()
.iter()
.all(|failure| !failure["execution_id"].is_null())
);
}
#[tokio::test]
async fn fail_fast_records_unlaunched_commands_as_skipped_children() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![
CommandUnit {
id: "fail".to_string(),
argv: vec!["/usr/bin/false".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
},
CommandUnit {
id: "never".to_string(),
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
},
],
topology: CommandTopology::Parallel {
max_parallel: 1,
fail_fast: true,
},
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(5)))
.await
.unwrap();
assert_eq!(finished.result.as_ref().unwrap()["counts"]["failed"], 1);
assert_eq!(finished.result.as_ref().unwrap()["counts"]["skipped"], 1);
let children = manager
.list()
.unwrap()
.into_iter()
.filter(|child| child.parent_id.as_deref() == Some(&execution.id))
.collect::<Vec<_>>();
assert_eq!(children.len(), 2);
assert_child_leases_released(&store, &children);
}
fn assert_child_leases_released(store: &Store, children: &[Execution]) {
assert!(children.iter().all(|child| {
store
.get_lease(&lease_id(&child.id))
.unwrap()
.is_some_and(|lease| lease.released)
}));
}
#[tokio::test]
async fn pipeline_pipefail_fails_the_parent_when_an_earlier_stage_fails() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store).unwrap());
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![
CommandUnit {
id: "fail".to_string(),
argv: vec!["/usr/bin/false".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
},
CommandUnit {
id: "succeed".to_string(),
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
},
],
topology: CommandTopology::Pipeline { pipefail: true },
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(5)))
.await
.unwrap();
assert_eq!(finished.status, ExecutionStatus::Failed);
assert_eq!(finished.result.as_ref().unwrap()["counts"]["failed"], 1);
}
}