use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque};
use std::io::{Read, Seek, SeekFrom, Write};
use std::os::fd::{FromRawFd, OwnedFd};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::Arc;
use std::time::Duration;
use incurs_codemode::ArtifactStore;
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use sha2::{Digest, Sha256};
use tokio::io::{AsyncRead, AsyncWriteExt};
use tokio::process::{Child, Command as TokioCommand};
use tokio_util::sync::CancellationToken;
use crate::log_file::{finalize_redaction, redaction_pending, require_redaction};
use crate::model::{
CausalEvent, CommandTopology, CommandUnit, Execution, ExecutionAttempt, ExecutionBackendSpec,
ExecutionBudget, ExecutionListPage, ExecutionProgram, ExecutionProgramSummary, ExecutionSpec,
ExecutionStatus, ExecutionSummary, ExecutionTree, ExecutionTreeRoot, ExecutionTreeState,
ExecutionVerdict, ExpectationState, IoMode, Lease, LeaseOwnerKind, PollCondition, PollSchedule,
PollTarget, ProcessController, RestartPolicy, RetryPolicy, RunLifetime, RunSpec,
SCHEMA_VERSION, SecretRef, TelemetryCapture, TelemetryCaptureStatus, TelemetryLine,
TelemetryPolicy, TelemetryProcessSample, TelemetrySample, TelemetrySlice, TelemetrySummary,
TelemetrySummarySample, WorkspaceContext, now_ms,
};
use crate::process::{ProcessManager, process_context_key};
use crate::run::{InputReceipt, RunLogs, RunManager};
use crate::store::{ExecutionClaim, Store};
pub(crate) const BUILTIN_MACOS_PROVIDER_NAME: &str = "apoc.isolated.macos";
pub(crate) const BUILTIN_MACOS_PROVIDER_GENERATION: &str = "builtin:apoc-macos-vz-v2";
#[derive(Clone)]
pub(crate) struct ExecutionManager {
store: Arc<Store>,
runs: RunManager,
processes: Arc<std::sync::Mutex<ProcessManager>>,
poll_cancellations: Arc<std::sync::Mutex<HashMap<String, CancellationToken>>>,
provider: Arc<dyn ProviderExecutionGateway>,
provider_cancellations: Arc<std::sync::Mutex<HashMap<String, CancellationToken>>>,
telemetry_tasks: Arc<std::sync::Mutex<HashMap<String, TelemetryTaskControl>>>,
gate: Arc<dyn crate::intent::IntentGate>,
}
pub(crate) struct ExecutionStreamChunk {
pub bytes: Vec<u8>,
pub next_offset: u64,
pub terminal: bool,
}
#[derive(Debug, Clone)]
pub struct ProviderExecutionRequest {
pub execution_id: String,
pub provider: String,
pub generation: String,
pub options: serde_json::Value,
pub argv: Vec<String>,
pub shell: bool,
pub cwd: Option<String>,
pub env: BTreeMap<String, String>,
pub secrets: BTreeMap<String, SecretRef>,
pub io: IoMode,
pub timeout_ms: Option<u64>,
pub artifact_bytes: Option<u64>,
pub retry: RetryPolicy,
}
#[derive(Debug, Clone)]
pub struct ProviderExecutionControl {
pub execution_id: String,
pub provider: String,
pub generation: String,
pub provider_execution_id: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ProviderExecutionEvent {
Started {
provider_execution_id: String,
},
Stdout(Vec<u8>),
Stderr(Vec<u8>),
Artifact(crate::model::ExecutionArtifactRef),
Retrying {
exit_code: Option<i32>,
error_code: String,
error: String,
will_retry: bool,
},
Exited {
exit_code: i32,
},
Canceled,
Error {
code: String,
message: String,
},
}
#[async_trait::async_trait]
pub trait ProviderExecutionGateway: Send + Sync {
fn validate(&self, _provider: &str, _generation: &str) -> Result<(), String> {
Ok(())
}
async fn execute(
&self,
request: ProviderExecutionRequest,
events: tokio::sync::mpsc::UnboundedSender<ProviderExecutionEvent>,
cancellation: CancellationToken,
) -> Result<(), String>;
async fn input(&self, control: ProviderExecutionControl, _data: &[u8]) -> Result<(), String> {
Err(format!(
"EXECUTION_INPUT_UNSUPPORTED: provider {}@{} does not expose live input for execution {}",
control.provider, control.generation, control.execution_id
))
}
async fn signal(&self, control: ProviderExecutionControl, signal: &str) -> Result<(), String> {
Err(format!(
"EXECUTION_SIGNAL_UNSUPPORTED: provider {}@{} does not expose {signal} for execution {}",
control.provider, control.generation, control.execution_id
))
}
}
#[cfg(test)]
struct UnavailableProviderExecutionGateway;
#[async_trait::async_trait]
#[cfg(test)]
impl ProviderExecutionGateway for UnavailableProviderExecutionGateway {
async fn execute(
&self,
request: ProviderExecutionRequest,
_events: tokio::sync::mpsc::UnboundedSender<ProviderExecutionEvent>,
_cancellation: CancellationToken,
) -> Result<(), String> {
Err(format!(
"EXECUTION_BACKEND_UNAVAILABLE: provider {}@{}",
request.provider, request.generation
))
}
}
#[derive(Clone)]
struct TelemetryTaskControl {
cancel: CancellationToken,
done: Arc<tokio::sync::Notify>,
}
#[derive(Debug, Serialize)]
struct CommandOutcome {
id: String,
execution_id: Option<String>,
status: String,
exit_code: Option<i32>,
stderr_tail: Option<String>,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
assertion_failed: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct ExecutionControlReplay {
operation: String,
fingerprint: String,
result: Option<serde_json::Value>,
}
struct PipelineStage {
id: String,
execution: Execution,
child: Child,
stderr: PathBuf,
stderr_pump: Option<tokio::task::JoinHandle<()>>,
stdout_pump: Option<tokio::task::JoinHandle<()>>,
deadline: Option<tokio::time::Instant>,
exit_code: Option<i32>,
}
struct BatchStage {
id: String,
execution: Execution,
child: Child,
stderr: PathBuf,
deadline: Option<tokio::time::Instant>,
timed_out: bool,
canceled: bool,
}
impl CommandOutcome {
fn failed(&self) -> bool {
self.status == "failed"
}
fn from_execution(id: String, execution: &Execution, stderr_tail: Option<String>) -> Self {
Self {
id,
execution_id: Some(execution.id.clone()),
status: reported_child_status(execution).to_string(),
exit_code: execution.exit_code(),
stderr_tail,
assertion_failed: matches!(
execution.expectation(),
ExpectationState::Unmet | ExpectationState::Unevaluated
),
}
}
fn failed_child(
id: String,
execution_id: Option<String>,
exit_code: Option<i32>,
stderr_tail: Option<String>,
) -> Self {
Self {
id,
execution_id,
status: "failed".to_string(),
exit_code,
stderr_tail,
assertion_failed: false,
}
}
fn launch_failure(id: String, error: String) -> Self {
Self::failed_child(id, None, None, Some(error))
}
}
impl ExecutionManager {
#[cfg(test)]
pub(crate) fn new(store: Arc<Store>, runs: RunManager) -> Self {
Self::with_provider(store, runs, Arc::new(UnavailableProviderExecutionGateway))
}
pub fn with_provider(
store: Arc<Store>,
runs: RunManager,
provider: Arc<dyn ProviderExecutionGateway>,
) -> Self {
let processes = Arc::new(std::sync::Mutex::new(ProcessManager::with_store(
store.clone(),
)));
Self {
store,
runs,
processes,
poll_cancellations: Arc::new(std::sync::Mutex::new(HashMap::new())),
provider,
provider_cancellations: Arc::new(std::sync::Mutex::new(HashMap::new())),
telemetry_tasks: Arc::new(std::sync::Mutex::new(HashMap::new())),
gate: Arc::new(crate::intent::Ungated),
}
}
pub(crate) fn with_gate(mut self, gate: Arc<dyn crate::intent::IntentGate>) -> Self {
self.gate = gate;
self
}
async fn offer_command(
&self,
run: &RunSpec,
execution_id: &str,
controller: &ProcessController,
) -> Result<(), String> {
if !self.gate.active() || controller.decider_depth > 0 {
return Ok(());
}
let path_var = run
.env
.get("PATH")
.cloned()
.or_else(|| std::env::var("PATH").ok());
let origin = if controller.hook_depth > 0 {
crate::intent::IntentOrigin::Hook
} else if controller.execution_id.is_some() {
crate::intent::IntentOrigin::Code
} else {
crate::intent::IntentOrigin::Cli
};
let intent = crate::intent::for_command(
"execution.command",
&run.argv,
run.shell,
run.cwd.as_deref(),
run.env
.keys()
.chain(run.resolved_env.keys())
.cloned()
.collect(),
path_var.as_deref(),
run.purpose.clone(),
crate::intent::Lineage {
actor: Some(controller.actor.clone()),
invocation_id: Some(controller.invocation_id.clone()),
execution_id: Some(execution_id.to_string()),
code_execution_id: controller.execution_id.clone(),
code_seq: None,
session_id: None,
cwd: run.cwd.clone(),
declaration_id: None,
},
origin,
now_ms(),
);
self.gate
.offer(intent)
.await
.map_err(|reason| format!("INTENT_REFUSED: {reason}"))
}
pub(crate) fn poll_cancellations(
&self,
) -> Arc<std::sync::Mutex<HashMap<String, CancellationToken>>> {
self.poll_cancellations.clone()
}
pub(crate) fn notify_poll_cancellations(&self, id: &str) -> Result<(), String> {
let ids = self.store.execution_family_ids(id)?;
if let Ok(active) = self.poll_cancellations.lock() {
for id in ids {
if let Some(token) = active.get(&id) {
token.cancel();
}
}
}
Ok(())
}
pub fn recover(&self) -> Result<(), String> {
for parent in self.store.recoverable_execution_records()? {
if terminal(parent.status) {
continue;
}
if matches!(parent.spec.program, ExecutionProgram::Poll { .. }) {
continue;
}
let ExecutionProgram::Commands { commands, topology } = &parent.spec.program else {
let Some(run) = self.store.get_run(&parent.id)? else {
self.fail_missing_run(parent)?;
continue;
};
let controller = ProcessController::capture(false, Some(parent.id.clone()));
let execution = self.sync_run(parent, &run, Some(&controller))?;
if !terminal(execution.status) {
self.finalize(execution.id.clone(), controller.clone());
self.supervise_budgets(
execution.id.clone(),
execution.spec.budget.clone(),
controller,
);
}
continue;
};
let children = self
.store
.execution_children(&parent.id)?
.into_iter()
.filter_map(|execution| {
execution
.spec
.labels
.get("apoc.command_id")
.cloned()
.map(|id| (id, execution))
})
.collect::<HashMap<_, _>>();
let manager = self.clone();
let parent = parent.clone();
let commands = commands.clone();
let topology = topology.clone();
let controller = ProcessController::capture(false, Some(parent.id.clone()));
self.supervise_budgets(parent.id.clone(), parent.spec.budget.clone(), controller);
tokio::spawn(async move {
manager
.resume_commands(parent, commands, topology, children)
.await;
});
}
for capture in self.store.list_telemetry_captures()? {
if capture.status == TelemetryCaptureStatus::Active {
self.spawn_telemetry(capture);
}
}
Ok(())
}
pub(crate) async fn reconcile_orphans(
&self,
apply: bool,
grace_ms: u64,
limit: usize,
) -> Result<Vec<(String, &'static str)>, String> {
let cutoff = now_ms().saturating_sub(grace_ms);
let mut found = Vec::<(String, &'static str)>::new();
for run in self.runs.orphaned_runs(cutoff).await? {
if found.len() >= limit {
return Ok(found);
}
let id = run.id.clone();
if apply {
let run = self.runs.close_vanished(run)?;
if let Some(execution) = self.store.get_execution_record(&id)?
&& !terminal(execution.status)
{
self.close_orphan(execution, &run, "process_vanished")?;
}
}
found.push((id, "process_vanished"));
}
for execution in self.store.recoverable_execution_records()? {
if found.len() >= limit {
break;
}
if terminal(execution.status)
|| execution.updated_at >= cutoff
|| !matches!(execution.spec.program, ExecutionProgram::Command { .. })
|| !matches!(execution.spec.backend, ExecutionBackendSpec::Native)
|| found.iter().any(|(id, _)| id == &execution.id)
|| self.runs.is_active(&execution.id).await
{
continue;
}
match self.store.get_run(&execution.id)? {
None => {
let id = execution.id.clone();
if apply {
self.fail_missing_run(execution)?;
}
found.push((id, "missing_run"));
}
Some(run) if terminal(execution_status(&run.status)) => {
let id = execution.id.clone();
if apply {
self.sync_run(execution, &run, None)?;
}
found.push((id, "terminal_run_unsynchronized"));
}
Some(_) => {}
}
}
Ok(found)
}
fn close_orphan(
&self,
mut execution: Execution,
run: &crate::model::ManagedRun,
cause: &str,
) -> Result<Execution, String> {
let mut result = execution
.result
.take()
.and_then(|value| value.as_object().cloned())
.unwrap_or_default();
result.insert(
"error_code".to_string(),
serde_json::Value::from("EXECUTION_INTERRUPTED"),
);
result.insert(
"error".to_string(),
serde_json::Value::from(
"Every process of this execution vanished without an observable exit status while no supervisor owned it",
),
);
result.insert("cause".to_string(), serde_json::Value::from(cause));
result.insert(
"run_status".to_string(),
serde_json::Value::from(run.status.clone()),
);
execution.result = Some(serde_json::Value::Object(result));
self.sync_run(execution, run, None)
}
pub(crate) async fn expire(
&self,
id: &str,
details: serde_json::Value,
timeout: Duration,
controller: &ProcessController,
) -> Result<Execution, String> {
self.cancel_with_cause(
id,
"EXECUTION_EXPIRED",
"execution.expired",
details,
timeout,
controller,
)
.await
}
pub(crate) fn owner_orphans(
&self,
updated_before: u64,
limit: usize,
) -> Result<Vec<(String, &'static str)>, String> {
let now = now_ms();
let mut found = Vec::new();
for execution in self.store.recoverable_execution_records()? {
if found.len() >= limit {
break;
}
if terminal(execution.status) || execution.updated_at >= updated_before {
continue;
}
let owner = execution.spec.owner_id.as_deref();
let cause = match execution.spec.owner_kind {
LeaseOwnerKind::Execution => match owner.or(execution.parent_id.as_deref()) {
Some(owner) => match self.store.get_execution_record(owner)? {
Some(parent) => terminal(parent.status).then_some("parent_gone"),
None => Some("parent_gone"),
},
None => None,
},
LeaseOwnerKind::Session => match owner.or(execution.session_id.as_deref()) {
Some(owner) => match self.store.get_session(owner)? {
Some(session) => {
(session.closed || session.closing || session.expires_at <= now)
.then_some("session_gone")
}
None => Some("session_gone"),
},
None => None,
},
LeaseOwnerKind::Persistent => None,
};
if let Some(cause) = cause {
found.push((execution.id, cause));
}
}
Ok(found)
}
pub(crate) async fn cancel_with_cause(
&self,
id: &str,
error_code: &str,
event_kind: &str,
details: serde_json::Value,
timeout: Duration,
controller: &ProcessController,
) -> Result<Execution, String> {
let mut execution = self
.store
.get_execution_record(id)?
.ok_or_else(|| format!("Execution \"{id}\" not found"))?;
if terminal(execution.status) {
return Ok(execution);
}
let mut result = execution
.result
.take()
.and_then(|value| value.as_object().cloned())
.unwrap_or_default();
result.insert(
"error_code".to_string(),
serde_json::Value::from(error_code),
);
if let serde_json::Value::Object(details) = &details {
for (key, value) in details {
result.insert(key.clone(), value.clone());
}
}
execution.result = Some(serde_json::Value::Object(result));
execution.updated_at = now_ms();
self.store.put_execution_record(&execution)?;
self.event(&execution, controller, event_kind, details)?;
self.cancel_unclaimed(id, timeout, controller).await
}
fn fail_missing_run(&self, mut execution: Execution) -> Result<(), String> {
execution.status = ExecutionStatus::Failed;
execution.updated_at = now_ms();
execution.last_progress_at = execution.updated_at;
let redaction_error = uses_backend_gateway(&execution.spec.backend)
.then(|| self.finalize_provider_redaction(&execution))
.transpose()
.err();
let (error_code, error) = if let Some(error) = redaction_error.as_deref() {
("EXECUTION_LOG_REDACTION_FAILED", error)
} else if uses_backend_gateway(&execution.spec.backend) {
(
"EXECUTION_BACKEND_LOST",
"Accepted external execution lost its supervised transport during daemon recovery",
)
} else {
(
"EXECUTION_RECOVERY_MISSING_RUN",
"Accepted execution has no durable managed run after daemon recovery",
)
};
execution.result = Some(serde_json::json!({
"error_code": error_code,
"error": error
}));
self.store.put_execution_record(&execution)?;
self.store.release_lease(&lease_id(&execution.id))?;
let controller = ProcessController::capture(false, Some(execution.id.clone()));
self.event(
&execution,
&controller,
"execution.failed",
execution.result.clone().unwrap_or_default(),
)?;
Ok(())
}
pub async fn start(
&self,
spec: ExecutionSpec,
controller: ProcessController,
parent_id: Option<String>,
session_id: Option<String>,
) -> Result<Execution, String> {
self.start_with_replay(spec, controller, parent_id, session_id)
.await
.map(|(execution, _)| execution)
}
pub async fn start_with_replay(
&self,
mut spec: ExecutionSpec,
controller: ProcessController,
parent_id: Option<String>,
session_id: Option<String>,
) -> Result<(Execution, bool), String> {
validate_spec(&spec)?;
if let Some((name, generation)) = backend_gateway_identity(&spec.backend) {
self.provider.validate(name, generation)?;
}
let fingerprint = execution_fingerprint(&spec, session_id.as_deref())?;
enrich_workspace(&mut spec.workspace);
let id = uuid::Uuid::now_v7().to_string();
let now = now_ms();
let mut execution = Execution {
id: id.clone(),
parent_id: parent_id.clone(),
session_id: session_id.clone(),
spec: spec.clone(),
status: ExecutionStatus::Queued,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
if let Some(key) = spec.idempotency_key.as_deref() {
match self.store.claim_execution(&execution, key, &fingerprint)? {
ExecutionClaim::Claimed => {}
ExecutionClaim::Replayed(execution) => return Ok((*execution, true)),
}
} else {
self.store.accept_execution(&execution)?;
}
self.store.put_lease(&Lease {
id: lease_id(&id),
owner_kind: LeaseOwnerKind::Execution,
owner_id: id.clone(),
resources: Vec::new(),
created_at: now,
renewed_at: now,
expires_at: spec.expires_at,
released: false,
})?;
self.event(
&execution,
&controller,
"execution.accepted",
serde_json::json!({ "schema_version": spec.schema_version }),
)?;
let _ = self.ensure_telemetry(&execution.id, &spec.telemetry);
if let ExecutionProgram::Commands { commands, topology } = &spec.program {
execution.status = ExecutionStatus::Running;
execution.updated_at = now_ms();
execution.last_progress_at = execution.updated_at;
self.store.put_execution_record(&execution)?;
self.event(
&execution,
&controller,
"execution.running",
serde_json::json!({ "commands": commands.len() }),
)?;
self.execute_commands(
execution.clone(),
commands.clone(),
topology.clone(),
controller.clone(),
);
self.supervise_budgets(
execution.id.clone(),
execution.spec.budget.clone(),
controller,
);
return Ok((execution, false));
}
if matches!(spec.program, ExecutionProgram::Poll { .. }) {
execution.status = ExecutionStatus::Running;
execution.updated_at = now_ms();
execution.last_progress_at = execution.updated_at;
self.store.put_execution_record(&execution)?;
self.event(
&execution,
&controller,
"execution.running",
serde_json::json!({ "program": "poll" }),
)?;
return Ok((execution, false));
}
if uses_backend_gateway(&spec.backend) {
execution.status = ExecutionStatus::Running;
execution.updated_at = now_ms();
execution.last_progress_at = execution.updated_at;
self.store.put_execution_record(&execution)?;
self.event(
&execution,
&controller,
"execution.running",
serde_json::json!({ "backend": backend_kind(&spec.backend) }),
)?;
let request = provider_execution_request(&execution)?;
if let Err(error) =
self.start_provider_execution(execution.clone(), request, controller.clone())
{
return self
.fail_start(execution, &controller, error)
.map(|execution| (execution, false));
}
self.supervise_budgets(
execution.id.clone(),
execution.spec.budget.clone(),
controller,
);
return Ok((execution, false));
}
let run_spec = match run_spec(&self.store, &spec, &controller) {
Ok(run_spec) => run_spec,
Err(error) => {
return self
.fail_start(execution, &controller, error)
.map(|execution| (execution, false));
}
};
if let Err(refusal) = self.offer_command(&run_spec, &id, &controller).await {
return self
.fail_start(execution, &controller, refusal)
.map(|execution| (execution, false));
}
let run = match self.runs.start_with_id(id, run_spec).await {
Ok(run) => run,
Err(error) => {
return self
.fail_start(execution, &controller, error)
.map(|execution| (execution, false));
}
};
let execution = self.sync_run(execution, &run, Some(&controller))?;
self.finalize(execution.id.clone(), controller.clone());
self.supervise_budgets(
execution.id.clone(),
execution.spec.budget.clone(),
controller,
);
Ok((execution, false))
}
fn fail_start(
&self,
mut execution: Execution,
controller: &ProcessController,
error: String,
) -> Result<Execution, String> {
execution.status = ExecutionStatus::Failed;
execution.updated_at = now_ms();
execution.last_progress_at = execution.updated_at;
execution.result = Some(serde_json::json!({
"error_code": "EXECUTION_START_FAILED",
"error": error
}));
self.store.put_execution_record(&execution)?;
self.store.release_lease(&lease_id(&execution.id))?;
self.event(
&execution,
controller,
"execution.failed",
execution.result.clone().unwrap_or_default(),
)?;
Ok(execution)
}
fn start_provider_execution(
&self,
execution: Execution,
request: ProviderExecutionRequest,
controller: ProcessController,
) -> Result<(), String> {
let directory = run_directory(&self.store, &execution.id);
std::fs::create_dir_all(&directory).map_err(|error| error.to_string())?;
if !execution.spec.secrets.is_empty() {
require_redaction(&directory)?;
}
let mut stdout = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(directory.join("stdout.log"))
.map_err(|error| error.to_string())?;
let mut stderr = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(directory.join("stderr.log"))
.map_err(|error| error.to_string())?;
let cancellation = CancellationToken::new();
self.provider_cancellations
.lock()
.map_err(|_| "Provider cancellation lock is poisoned".to_string())?
.insert(execution.id.clone(), cancellation.clone());
let manager = self.clone();
let provider = self.provider.clone();
let execution_id = execution.id.clone();
let (events, mut incoming) = tokio::sync::mpsc::unbounded_channel();
tokio::spawn(async move {
let call = provider.execute(request, events, cancellation.clone());
tokio::pin!(call);
let mut call_result = None;
let mut execution = execution;
let mut terminal = false;
loop {
tokio::select! {
event = incoming.recv() => {
let Some(event) = event else {
if call_result.is_some() {
break;
}
continue;
};
match manager.apply_provider_event(
&mut execution,
event,
&mut stdout,
&mut stderr,
&controller,
) {
Ok(done) => {
terminal = done;
if terminal {
break;
}
}
Err(error) => {
let (code, message) = match manager.finalize_provider_redaction(&execution) {
Ok(()) => ("PROVIDER_PROTOCOL_ERROR", error),
Err(redaction) => ("EXECUTION_LOG_REDACTION_FAILED", redaction),
};
let _ = manager.finish_provider_fault(
&mut execution,
code,
&message,
cancellation.is_cancelled(),
&controller,
);
terminal = true;
break;
}
}
}
result = &mut call, if call_result.is_none() => {
call_result = Some(result);
}
}
}
if !terminal {
let (mut code, mut message) = match call_result {
Some(Ok(())) => (
"PROVIDER_TERMINAL_EVENT_MISSING",
"Provider returned without a terminal event".to_string(),
),
Some(Err(error)) => ("PROVIDER_EXECUTION_FAILED", error),
None => (
"PROVIDER_EVENT_STREAM_CLOSED",
"Provider event stream closed before the provider returned".to_string(),
),
};
if let Err(error) = manager.finalize_provider_redaction(&execution) {
code = "EXECUTION_LOG_REDACTION_FAILED";
message = error;
}
let _ = manager.finish_provider_fault(
&mut execution,
code,
&message,
cancellation.is_cancelled(),
&controller,
);
}
if let Ok(mut cancellations) = manager.provider_cancellations.lock() {
cancellations.remove(&execution_id);
}
});
Ok(())
}
fn apply_provider_event(
&self,
execution: &mut Execution,
event: ProviderExecutionEvent,
stdout: &mut std::fs::File,
stderr: &mut std::fs::File,
controller: &ProcessController,
) -> Result<bool, String> {
match event {
ProviderExecutionEvent::Started {
provider_execution_id,
} => {
if execution
.attempts
.last()
.is_some_and(|attempt| attempt.completed_at.is_none())
{
return Err(
"Provider started a new attempt before the prior attempt completed"
.to_string(),
);
}
let now = now_ms();
execution.attempts.push(ExecutionAttempt {
id: uuid::Uuid::now_v7().to_string(),
number: execution.attempts.len() as u32 + 1,
processes: Vec::new(),
started_at: now,
completed_at: None,
exit_code: None,
error_code: None,
stdout_artifact: None,
stderr_artifact: None,
});
execution.updated_at = now;
execution.last_progress_at = now;
provider_result(execution).insert(
"provider_execution_id".to_string(),
provider_execution_id.into(),
);
self.store.put_execution_record(execution)?;
self.event(
execution,
controller,
"execution.provider_started",
execution.result.clone().unwrap_or_default(),
)?;
Ok(false)
}
ProviderExecutionEvent::Stdout(data) => {
require_provider_started(execution)?;
stdout.write_all(&data).map_err(|error| error.to_string())?;
stdout.flush().map_err(|error| error.to_string())?;
self.record_provider_progress(execution, "stdout", data.len(), controller)?;
Ok(false)
}
ProviderExecutionEvent::Stderr(data) => {
require_provider_started(execution)?;
stderr.write_all(&data).map_err(|error| error.to_string())?;
stderr.flush().map_err(|error| error.to_string())?;
self.record_provider_progress(execution, "stderr", data.len(), controller)?;
Ok(false)
}
ProviderExecutionEvent::Artifact(artifact) => {
require_provider_started(execution)?;
let value = serde_json::to_value(&artifact).map_err(|error| error.to_string())?;
provider_result(execution)
.entry("artifacts")
.or_insert_with(|| serde_json::json!([]))
.as_array_mut()
.ok_or_else(|| "Provider result artifacts is not an array".to_string())?
.push(value);
self.event(
execution,
controller,
"execution.artifact",
serde_json::to_value(artifact).map_err(|error| error.to_string())?,
)?;
Ok(false)
}
ProviderExecutionEvent::Retrying {
exit_code,
error_code,
error,
will_retry,
} => {
let now = now_ms();
if let Some(attempt) = execution.attempts.last_mut()
&& attempt.completed_at.is_none()
{
attempt.completed_at = Some(now);
attempt.exit_code = exit_code;
attempt.error_code = Some(error_code.clone());
} else {
execution.attempts.push(ExecutionAttempt {
id: uuid::Uuid::now_v7().to_string(),
number: execution.attempts.len() as u32 + 1,
processes: Vec::new(),
started_at: now,
completed_at: Some(now),
exit_code,
error_code: Some(error_code.clone()),
stdout_artifact: None,
stderr_artifact: None,
});
}
execution.updated_at = now;
execution.last_progress_at = now;
let result = provider_result(execution);
result.remove("provider_execution_id");
result.insert("retry_error_code".to_string(), error_code.into());
result.insert("retry_error".to_string(), error.into());
self.store.put_execution_record(execution)?;
self.event(
execution,
controller,
if will_retry {
"execution.retrying"
} else {
"execution.attempt_failed"
},
serde_json::json!({
"attempt": execution.attempts.len(),
"exit_code": exit_code,
"will_retry": will_retry
}),
)?;
Ok(false)
}
ProviderExecutionEvent::Exited { exit_code } => {
require_provider_started(execution)?;
stdout.flush().map_err(|error| error.to_string())?;
stderr.flush().map_err(|error| error.to_string())?;
if let Err(error) = self.finalize_provider_redaction(execution) {
self.finish_provider_fault(
execution,
"EXECUTION_LOG_REDACTION_FAILED",
&error,
false,
controller,
)?;
return Ok(true);
}
let now = now_ms();
let attempt = execution
.attempts
.last_mut()
.expect("provider start validation created an attempt");
attempt.completed_at = Some(now);
attempt.exit_code = Some(exit_code);
execution.status = if exit_code == 0 {
ExecutionStatus::Completed
} else {
ExecutionStatus::Failed
};
execution.updated_at = now;
execution.last_progress_at = now;
provider_result(execution).insert(
"exit_code".to_string(),
serde_json::Value::Number(exit_code.into()),
);
provider_result(execution).insert(
"output_truncated".to_string(),
serde_json::Value::Bool(false),
);
self.judge_provider_expectation(execution);
self.store.release_lease(&lease_id(&execution.id))?;
self.store.put_execution_record(execution)?;
self.event(
execution,
controller,
&format!("execution.{}", status_name(execution.status)),
execution.result.clone().unwrap_or_default(),
)?;
Ok(true)
}
ProviderExecutionEvent::Canceled => {
require_provider_started(execution)?;
if let Err(error) = self.finalize_provider_redaction(execution) {
self.finish_provider_fault(
execution,
"EXECUTION_LOG_REDACTION_FAILED",
&error,
false,
controller,
)?;
return Ok(true);
}
let (code, message) = match execution.spec.backend {
ExecutionBackendSpec::Provider { .. } => (
"PROVIDER_EXECUTION_CANCELED",
"Provider execution was canceled",
),
ExecutionBackendSpec::Isolated { .. } => (
"EXECUTION_BACKEND_CANCELED",
"Isolated execution was canceled",
),
ExecutionBackendSpec::Native => {
return Err(
"Native execution received an external backend event".to_string()
);
}
};
self.finish_provider_fault(execution, code, message, true, controller)?;
Ok(true)
}
ProviderExecutionEvent::Error { code, message } => {
if let Err(error) = self.finalize_provider_redaction(execution) {
self.finish_provider_fault(
execution,
"EXECUTION_LOG_REDACTION_FAILED",
&error,
false,
controller,
)?;
return Ok(true);
}
self.finish_provider_fault(execution, &code, &message, false, controller)?;
Ok(true)
}
}
}
fn record_provider_progress(
&self,
execution: &mut Execution,
stream: &str,
bytes: usize,
controller: &ProcessController,
) -> Result<(), String> {
execution.updated_at = now_ms();
execution.last_progress_at = execution.updated_at;
self.store.put_execution_record(execution)?;
self.event(
execution,
controller,
"execution.output",
serde_json::json!({ "stream": stream, "bytes": bytes }),
)
.map(|_| ())
}
fn finish_provider_fault(
&self,
execution: &mut Execution,
code: &str,
message: &str,
canceled: bool,
controller: &ProcessController,
) -> Result<(), String> {
let now = now_ms();
if let Some(attempt) = execution.attempts.last_mut() {
attempt.completed_at = Some(now);
attempt.error_code = Some(code.to_string());
}
execution.status = if canceled {
ExecutionStatus::Canceled
} else {
ExecutionStatus::Failed
};
execution.updated_at = now;
execution.last_progress_at = now;
let result = provider_result(execution);
result.insert("error_code".to_string(), code.into());
result.insert("error".to_string(), message.into());
if canceled {
result.insert("canceled".to_string(), true.into());
}
self.store.release_lease(&lease_id(&execution.id))?;
self.store.put_execution_record(execution)?;
self.event(
execution,
controller,
&format!("execution.{}", status_name(execution.status)),
execution.result.clone().unwrap_or_default(),
)
.map(|_| ())
}
fn judge_provider_expectation(&self, execution: &mut Execution) {
let Some(condition) = execution.spec.expect.clone() else {
return;
};
let exit_code = execution
.attempts
.last()
.and_then(|attempt| attempt.exit_code);
let (met, error) = match self
.logs(&execution.id, EXPECTATION_TAIL_BYTES)
.map(|logs| {
crate::poll::command_observation(exit_code, status_name(execution.status), &logs)
}) {
Ok(observation) => match crate::poll::judge(&condition, &observation) {
Ok(met) => (met, None),
Err(error) => (false, Some(error)),
},
Err(error) => (false, Some(error)),
};
execution.record_expectation(met, error);
}
fn finalize_provider_redaction(&self, execution: &Execution) -> Result<(), String> {
if execution.spec.secrets.is_empty() {
return Ok(());
}
let directory = run_directory(&self.store, &execution.id);
let stdout = directory.join("stdout.log");
let stderr = directory.join("stderr.log");
let values = execution
.spec
.secrets
.iter()
.map(|(name, secret)| {
if secret.provider != "env" {
return Err(format!("SECRET_PROVIDER_UNAVAILABLE: {name}"));
}
std::env::var(&secret.reference).map_err(|_| format!("SECRET_NOT_FOUND: {name}"))
})
.collect::<Result<Vec<_>, _>>();
match values {
Ok(values) => {
let values = values.iter().collect::<Vec<_>>();
finalize_redaction(&[&stdout, &stderr], &values)
}
Err(resolution_error) => match finalize_redaction(&[&stdout, &stderr], &[]) {
Ok(()) => Ok(()),
Err(redaction_error) => Err(format!(
"{resolution_error}; unresolved secret cleanup: {redaction_error}"
)),
},
}
}
#[cfg(test)]
pub fn list(&self) -> Result<Vec<Execution>, String> {
self.store
.list_execution_records()?
.into_iter()
.map(|execution| {
let Some(run) = self.runs.store().get_run(&execution.id)? else {
return Ok(execution);
};
self.sync_run(execution, &run, None)
})
.collect()
}
pub fn list_page(
&self,
limit: usize,
cursor: Option<&str>,
status: Option<ExecutionStatus>,
error_code: Option<&str>,
parent: Option<&str>,
) -> Result<ExecutionListPage, String> {
if !(1..=100).contains(&limit) {
return Err("limit must be between 1 and 100".to_string());
}
let before = cursor.map(parse_execution_cursor).transpose()?;
let candidates = self.store.list_execution_records_page(
before.as_ref().map(|(at, id)| (*at, id.as_str())),
status.map(status_name),
parent,
error_code,
limit + 1,
)?;
let mut executions = Vec::new();
let mut bytes = 0_usize;
let mut has_more = candidates.len() > limit;
for execution in candidates.into_iter().take(limit) {
let execution = match self.runs.store().get_run(&execution.id)? {
Some(run) => self.sync_run(execution, &run, None)?,
None => execution,
};
let summary = execution_summary(&execution);
let summary_bytes = serde_json::to_vec(&summary)
.map_err(|error| error.to_string())?
.len();
if !executions.is_empty() && bytes + summary_bytes > 128 * 1024 {
has_more = true;
break;
}
bytes += summary_bytes;
executions.push(summary);
}
let next_cursor = has_more.then(|| {
let execution = executions
.last()
.expect("bounded page contains a cursor item");
format!("{}:{}", execution.updated_at, execution.id)
});
Ok(ExecutionListPage {
executions,
next_cursor,
})
}
pub fn get(&self, id: &str) -> Result<Execution, String> {
let execution = self
.store
.get_execution_record(id)?
.ok_or_else(|| format!("Execution \"{id}\" not found"))?;
let Some(run) = self.runs.store().get_run(id)? else {
return Ok(execution);
};
self.sync_run(execution, &run, None)
}
pub fn start_telemetry(
&self,
execution_id: &str,
interval_ms: u64,
max_bytes: u64,
idempotency_key: &str,
) -> Result<TelemetryCapture, String> {
validate_telemetry_key(idempotency_key)?;
if let Some((operation, target, capture)) = self.store.telemetry_receipt(idempotency_key)? {
if operation != "start" || target != execution_id {
return Err(format!(
"TELEMETRY_IDEMPOTENCY_CONFLICT: key \"{idempotency_key}\" identifies {operation} for execution \"{target}\""
));
}
return Ok(capture);
}
let capture = self.begin_telemetry(execution_id, interval_ms, max_bytes)?;
self.store
.claim_telemetry_receipt(idempotency_key, "start", execution_id, &capture)
}
pub(crate) fn ensure_telemetry(
&self,
execution_id: &str,
policy: &TelemetryPolicy,
) -> Result<Option<TelemetryCapture>, String> {
match policy {
TelemetryPolicy::Off => Ok(None),
TelemetryPolicy::Sample {
interval_ms,
max_bytes,
} => self
.begin_telemetry(execution_id, *interval_ms, *max_bytes)
.map(Some),
}
}
pub async fn stop_telemetry(
&self,
execution_id: &str,
idempotency_key: &str,
) -> Result<TelemetryCapture, String> {
validate_telemetry_key(idempotency_key)?;
if let Some((operation, target, capture)) = self.store.telemetry_receipt(idempotency_key)? {
if operation != "stop" || target != execution_id {
return Err(format!(
"TELEMETRY_IDEMPOTENCY_CONFLICT: key \"{idempotency_key}\" identifies {operation} for execution \"{target}\""
));
}
return Ok(capture);
}
let capture = self
.store
.get_telemetry_capture(execution_id)?
.ok_or_else(|| format!("Telemetry for execution \"{execution_id}\" not found"))?;
let control = self
.telemetry_tasks
.lock()
.map_err(|_| "Telemetry task lock is poisoned".to_string())?
.get(&capture.id)
.cloned();
if let Some(control) = control {
control.cancel.cancel();
control.done.notified().await;
}
let mut capture = self
.store
.get_telemetry_capture(execution_id)?
.ok_or_else(|| format!("Telemetry for execution \"{execution_id}\" not found"))?;
if capture.status == TelemetryCaptureStatus::Active {
capture.status = TelemetryCaptureStatus::Stopped;
capture.stopped_at = Some(now_ms());
self.store.put_telemetry_capture(&capture)?;
}
self.store
.claim_telemetry_receipt(idempotency_key, "stop", execution_id, &capture)
}
pub fn telemetry_summary(&self, execution_id: &str) -> Result<TelemetrySummary, String> {
let capture = self
.store
.get_telemetry_capture(execution_id)?
.ok_or_else(|| format!("Telemetry for execution \"{execution_id}\" not found"))?;
let samples = self.store.telemetry_samples(&capture.id)?;
Ok(TelemetrySummary {
latest: samples.last().map(|sample| TelemetrySummarySample {
sequence: sample.sequence,
sampled_at: sample.sampled_at,
wall_time_ms: sample.wall_time_ms,
process_count: sample.process_count,
cpu_percent: sample.cpu_percent,
cpu_time_ms: sample.cpu_time_ms,
memory_bytes: sample.memory_bytes,
virtual_memory_bytes: sample.virtual_memory_bytes,
disk_read_bytes: sample.disk_read_bytes,
disk_written_bytes: sample.disk_written_bytes,
}),
peak_cpu_percent: samples
.iter()
.map(|sample| sample.cpu_percent)
.reduce(f32::max),
peak_memory_bytes: samples.iter().map(|sample| sample.memory_bytes).max(),
peak_process_count: samples.iter().map(|sample| sample.process_count).max(),
capture,
})
}
pub fn telemetry_slice(
&self,
execution_id: &str,
grep: Option<&str>,
context: usize,
max_matches: usize,
lines: Option<&str>,
) -> Result<TelemetrySlice, String> {
if grep.is_some() && lines.is_some() {
return Err("Use either --grep or --lines, not both".to_string());
}
let capture = self
.store
.get_telemetry_capture(execution_id)?
.ok_or_else(|| format!("Telemetry for execution \"{execution_id}\" not found"))?;
let samples = self.store.telemetry_samples(&capture.id)?;
let encoded = samples
.iter()
.map(|sample| {
serde_json::to_string(sample)
.map(|text| TelemetryLine {
sequence: sample.sequence,
text,
})
.map_err(|error| error.to_string())
})
.collect::<Result<Vec<_>, _>>()?;
let total_lines = encoded.len() as u64;
if let Some(range) = lines {
let (start, end) = parse_telemetry_range(range)?;
let selected = encoded
.into_iter()
.filter(|line| line.sequence >= start && line.sequence <= end)
.collect::<Vec<_>>();
return Ok(TelemetrySlice {
capture_id: capture.id,
total_lines,
total_matches: selected.len() as u64,
lines: selected,
truncated: false,
});
}
let pattern = grep.ok_or_else(|| "Supply --grep or --lines".to_string())?;
let regex = regex::Regex::new(pattern)
.map_err(|error| format!("Invalid telemetry --grep pattern: {error}"))?;
let matching = encoded
.iter()
.enumerate()
.filter_map(|(index, line)| regex.is_match(&line.text).then_some(index))
.collect::<Vec<_>>();
let limit = max_matches.clamp(1, 1000);
let mut selected = BTreeSet::new();
for index in matching.iter().take(limit) {
let start = index.saturating_sub(context);
let end = index
.saturating_add(context)
.min(encoded.len().saturating_sub(1));
selected.extend(start..=end);
}
Ok(TelemetrySlice {
capture_id: capture.id,
total_lines,
total_matches: matching.len() as u64,
lines: selected
.into_iter()
.filter_map(|index| encoded.get(index).cloned())
.collect(),
truncated: matching.len() > limit,
})
}
fn begin_telemetry(
&self,
execution_id: &str,
interval_ms: u64,
max_bytes: u64,
) -> Result<TelemetryCapture, String> {
let minimum_interval_ms = sysinfo::MINIMUM_CPU_UPDATE_INTERVAL.as_millis() as u64;
if interval_ms < minimum_interval_ms {
return Err(format!(
"Telemetry interval must be at least {minimum_interval_ms} ms"
));
}
if max_bytes == 0 {
return Err("Telemetry max bytes must be greater than zero".to_string());
}
if self.store.get_execution_record(execution_id)?.is_none() {
return Err(format!("Execution \"{execution_id}\" not found"));
}
if let Some(capture) = self.store.get_telemetry_capture(execution_id)? {
if capture.status == TelemetryCaptureStatus::Active {
self.spawn_telemetry(capture.clone());
}
return Ok(capture);
}
let capture = TelemetryCapture {
schema_version: SCHEMA_VERSION,
id: uuid::Uuid::now_v7().to_string(),
execution_id: execution_id.to_string(),
interval_ms,
max_bytes,
status: TelemetryCaptureStatus::Active,
started_at: now_ms(),
stopped_at: None,
next_sequence: 0,
sample_count: 0,
retained_bytes: 0,
dropped_samples: 0,
error: None,
};
match self.store.put_telemetry_capture(&capture) {
Ok(()) => {
self.spawn_telemetry(capture.clone());
Ok(capture)
}
Err(error) => match self.store.get_telemetry_capture(execution_id)? {
Some(existing) => {
if existing.status == TelemetryCaptureStatus::Active {
self.spawn_telemetry(existing.clone());
}
Ok(existing)
}
None => Err(error),
},
}
}
fn spawn_telemetry(&self, capture: TelemetryCapture) {
let cancel = CancellationToken::new();
let done = Arc::new(tokio::sync::Notify::new());
let Ok(mut tasks) = self.telemetry_tasks.lock() else {
return;
};
if tasks.contains_key(&capture.id) {
return;
}
tasks.insert(
capture.id.clone(),
TelemetryTaskControl {
cancel: cancel.clone(),
done: done.clone(),
},
);
drop(tasks);
let manager = self.clone();
tokio::spawn(async move {
let discovery_every = (1000 / capture.interval_ms).max(1);
let flush_every = (500 / capture.interval_ms).max(1) as usize;
let mut tick = 0_u64;
let mut known = Vec::<crate::model::ProcessIdentity>::new();
let mut pending = Vec::new();
loop {
if cancel.is_cancelled() {
let _ = manager
.store
.append_telemetry_samples(&capture.id, std::mem::take(&mut pending));
break;
}
let discover = tick.is_multiple_of(discovery_every);
match manager.collect_telemetry_sample(&capture, discover, &mut known) {
Ok((terminal, sample)) => {
pending.push(sample);
if (terminal || pending.len() >= flush_every)
&& let Err(error) = manager
.store
.append_telemetry_samples(&capture.id, std::mem::take(&mut pending))
{
let _ = manager.finish_telemetry(
&capture.id,
TelemetryCaptureStatus::Degraded,
Some(error),
);
break;
}
if terminal {
let _ = manager.finish_telemetry(
&capture.id,
TelemetryCaptureStatus::Completed,
None,
);
break;
}
}
Err(error) => {
let _ = manager
.store
.append_telemetry_samples(&capture.id, std::mem::take(&mut pending));
let _ = manager.finish_telemetry(
&capture.id,
TelemetryCaptureStatus::Degraded,
Some(error),
);
break;
}
}
tick = tick.saturating_add(1);
tokio::select! {
_ = cancel.cancelled() => {
let _ = manager.store.append_telemetry_samples(
&capture.id,
std::mem::take(&mut pending),
);
break;
},
_ = tokio::time::sleep(Duration::from_millis(capture.interval_ms)) => {}
}
}
if let Ok(mut tasks) = manager.telemetry_tasks.lock() {
tasks.remove(&capture.id);
}
done.notify_one();
});
}
fn collect_telemetry_sample(
&self,
capture: &TelemetryCapture,
discover: bool,
known: &mut Vec<crate::model::ProcessIdentity>,
) -> Result<(bool, TelemetrySample), String> {
let family = self.store.execution_family(&capture.execution_id)?;
if family.is_empty() {
return Err(format!(
"Execution \"{}\" disappeared while telemetry was active",
capture.execution_id
));
}
let all_terminal = family.iter().all(|execution| terminal(execution.status));
let roots = family
.iter()
.flat_map(|execution| execution.attempts.iter())
.flat_map(|attempt| attempt.processes.iter().cloned())
.collect::<Vec<_>>();
let processes = if discover {
let trees = self
.processes
.lock()
.map_err(|_| "Process discovery lock is poisoned".to_string())?
.trees_for_identities(&roots);
let mut seen = BTreeSet::new();
let mut processes = Vec::new();
for (_, tree) in trees {
for process in tree {
if seen.insert(process_context_key(&process.identity)) {
processes.push(process);
}
}
}
processes
} else {
let mut candidates = known.clone();
let mut seen = candidates
.iter()
.map(process_context_key)
.collect::<BTreeSet<_>>();
for root in roots {
if seen.insert(process_context_key(&root)) {
candidates.push(root);
}
}
self.processes
.lock()
.map_err(|_| "Process discovery lock is poisoned".to_string())?
.infos_for_identities(&candidates)
};
*known = processes
.iter()
.map(|process| process.identity.clone())
.collect();
let processes = processes
.into_iter()
.map(|process| TelemetryProcessSample {
identity: process.identity,
cpu_percent: process.cpu_percent,
cpu_time_ms: process.cpu_time_ms,
memory_bytes: process.memory_bytes,
virtual_memory_bytes: process.virtual_memory_bytes,
disk_read_bytes: process.disk_read_bytes,
disk_written_bytes: process.disk_written_bytes,
})
.collect::<Vec<_>>();
let family_live = !processes.is_empty();
let sampled_at = now_ms();
let sample = TelemetrySample {
sequence: 0,
sampled_at,
wall_time_ms: sampled_at.saturating_sub(capture.started_at),
process_count: processes.len() as u64,
cpu_percent: processes.iter().map(|process| process.cpu_percent).sum(),
cpu_time_ms: processes.iter().map(|process| process.cpu_time_ms).sum(),
memory_bytes: processes.iter().map(|process| process.memory_bytes).sum(),
virtual_memory_bytes: processes
.iter()
.map(|process| process.virtual_memory_bytes)
.sum(),
disk_read_bytes: processes
.iter()
.map(|process| process.disk_read_bytes)
.sum(),
disk_written_bytes: processes
.iter()
.map(|process| process.disk_written_bytes)
.sum(),
processes,
};
Ok((all_terminal && !family_live, sample))
}
fn finish_telemetry(
&self,
capture_id: &str,
status: TelemetryCaptureStatus,
error: Option<String>,
) -> Result<(), String> {
let mut capture = self
.store
.get_telemetry_capture(capture_id)?
.ok_or_else(|| format!("Telemetry capture \"{capture_id}\" not found"))?;
if capture.status != TelemetryCaptureStatus::Active {
return Ok(());
}
capture.status = status;
capture.stopped_at = Some(now_ms());
capture.error = error;
self.store.put_telemetry_capture(&capture)
}
pub fn tree(&self, id: &str, controller: &ProcessController) -> Result<ExecutionTree, String> {
let records = self.store.execution_family(id)?;
if !records.iter().any(|execution| execution.id == id) {
return Err(format!("Execution \"{id}\" not found"));
}
let mut children = HashMap::<String, Vec<(u64, String)>>::new();
for execution in &records {
if let Some(parent_id) = &execution.parent_id {
children
.entry(parent_id.clone())
.or_default()
.push((execution.created_at, execution.id.clone()));
}
}
for owned in children.values_mut() {
owned.sort();
}
let mut queue = VecDeque::from([id.to_string()]);
let mut seen = BTreeSet::new();
let mut family = Vec::new();
while let Some(execution_id) = queue.pop_front() {
if !seen.insert(execution_id.clone()) {
continue;
}
family.push(self.get(&execution_id)?);
if let Some(owned) = children.get(&execution_id) {
queue.extend(owned.iter().map(|(_, child_id)| child_id.clone()));
}
}
let requested = family
.first()
.expect("validated execution family contains its requested root");
let mut roots = Vec::new();
let mut unfinished = BTreeSet::new();
for execution in &family {
for attempt in &execution.attempts {
for identity in &attempt.processes {
let key = process_context_key(identity);
if attempt.completed_at.is_none() {
unfinished.insert(key);
}
roots.push(ExecutionTreeRoot {
execution_id: execution.id.clone(),
attempt_id: attempt.id.clone(),
identity: identity.clone(),
live: false,
});
}
}
}
let captured = roots
.iter()
.map(|root| root.identity.clone())
.collect::<Vec<_>>();
let trees = self
.processes
.lock()
.map_err(|_| "Process discovery lock is poisoned".to_string())?
.trees_for_identities(&captured);
let live_keys = trees
.iter()
.map(|(root, _)| process_context_key(root))
.collect::<BTreeSet<_>>();
for root in &mut roots {
root.live = live_keys.contains(&process_context_key(&root.identity));
}
let all_terminal = family.iter().all(|execution| terminal(execution.status));
let unfinished_missing = unfinished.iter().any(|key| !live_keys.contains(key));
let completed_live = roots
.iter()
.any(|root| root.live && !unfinished.contains(&process_context_key(&root.identity)));
let state =
if unfinished_missing || completed_live || (all_terminal && !live_keys.is_empty()) {
ExecutionTreeState::Inconsistent
} else if !live_keys.is_empty() {
ExecutionTreeState::Active
} else if all_terminal {
ExecutionTreeState::Terminal
} else {
ExecutionTreeState::Pending
};
let mut returned = Vec::new();
let mut returned_keys = BTreeSet::new();
let processes = self
.processes
.lock()
.map_err(|_| "Process discovery lock is poisoned".to_string())?;
for (captured_root, tree) in trees {
let owner = roots
.iter()
.find(|root| root.identity == captured_root)
.expect("live tree root originated from captured execution roots");
processes.record_with_controller(
&tree,
"execution.tree",
"observed",
None,
controller,
"managed",
Some(&owner.execution_id),
&serde_json::json!({ "requested_execution_id": id }),
)?;
for process in tree {
let key = process_context_key(&process.identity);
if returned_keys.insert(key) {
returned.push(process);
}
}
}
Ok(ExecutionTree {
execution_id: id.to_string(),
status: requested.status,
state,
observed_at: now_ms(),
roots,
processes: returned,
})
}
pub async fn wait(&self, id: &str, timeout: Option<Duration>) -> Result<Execution, String> {
if self.runs.store().get_run(id)?.is_none() {
let started = tokio::time::Instant::now();
loop {
let execution = self.get(id)?;
if terminal(execution.status) {
return Ok(execution);
}
if timeout.is_some_and(|timeout| started.elapsed() >= timeout) {
return Ok(execution);
}
tokio::time::sleep(Duration::from_millis(25)).await;
}
}
let run = self.runs.wait(id, timeout).await?;
let execution = self
.store
.get_execution_record(id)?
.ok_or_else(|| format!("Execution \"{id}\" not found"))?;
self.sync_run(execution, &run, None)
}
pub async fn cancel(
&self,
id: &str,
timeout: Duration,
idempotency_key: &str,
controller: &ProcessController,
) -> Result<Execution, String> {
let operation = "execution.cancel";
let fingerprint = execution_control_fingerprint(
operation,
&serde_json::json!({ "id": id, "timeout_ms": timeout.as_millis() }),
)?;
if let Some(replay) =
self.execution_control_replay::<Execution>(idempotency_key, operation, &fingerprint)?
{
return Ok(replay);
}
if let Some(replay) =
self.claim_execution_control::<Execution>(idempotency_key, operation, &fingerprint)?
{
return Ok(replay);
}
let result = self.cancel_unclaimed(id, timeout, controller).await?;
self.record_execution_control(idempotency_key, operation, fingerprint, &result)?;
Ok(result)
}
pub(crate) async fn cancel_unclaimed(
&self,
id: &str,
timeout: Duration,
controller: &ProcessController,
) -> Result<Execution, String> {
let mut execution = self
.store
.get_execution_record(id)?
.ok_or_else(|| format!("Execution \"{id}\" not found"))?;
if terminal(execution.status) {
return Ok(execution);
}
if !cancellation_requested(&execution) {
let preserve_progress = execution.status == ExecutionStatus::Stalled;
let mut result = execution
.result
.take()
.and_then(|value| value.as_object().cloned())
.unwrap_or_default();
result.insert(
"cancel_requested".to_string(),
serde_json::Value::Bool(true),
);
execution.updated_at = now_ms();
if !preserve_progress {
execution.last_progress_at = execution.updated_at;
}
execution.result = Some(serde_json::Value::Object(result));
self.store.put_execution_record(&execution)?;
self.event(
&execution,
controller,
"execution.cancel_requested",
execution.result.clone().unwrap_or_default(),
)?;
}
self.notify_poll_cancellations(id)?;
if uses_backend_gateway(&execution.spec.backend) {
if let Some(cancellation) = self
.provider_cancellations
.lock()
.map_err(|_| "Provider cancellation lock is poisoned".to_string())?
.get(id)
.cloned()
{
cancellation.cancel();
}
let deadline = tokio::time::Instant::now() + timeout;
loop {
let current = self.get(id)?;
if terminal(current.status) {
return Ok(current);
}
if tokio::time::Instant::now() >= deadline {
return Err(format!("Timed out canceling provider execution \"{id}\""));
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
}
if let ExecutionProgram::Commands { commands, .. } = &execution.spec.program {
let expected = commands.len();
let deadline = tokio::time::Instant::now() + timeout;
loop {
let current = self
.store
.get_execution_record(id)?
.ok_or_else(|| format!("Execution \"{id}\" not found"))?;
let children = self
.store
.execution_children(id)?
.into_iter()
.collect::<Vec<_>>();
if terminal(current.status)
&& children.len() == expected
&& children.iter().all(|child| terminal(child.status))
{
return Ok(current);
}
if tokio::time::Instant::now() >= deadline {
return Err(format!(
"Timed out canceling execution \"{id}\" while waiting for child cleanup"
));
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
}
if matches!(execution.spec.program, ExecutionProgram::Poll { .. }) {
let children = self
.store
.execution_children(id)?
.into_iter()
.collect::<Vec<_>>();
for child in &children {
if !terminal(child.status) {
let _ = Box::pin(self.cancel_unclaimed(&child.id, timeout, controller)).await;
}
}
let deadline = tokio::time::Instant::now() + timeout;
loop {
let current = self
.store
.get_execution_record(id)?
.ok_or_else(|| format!("Execution \"{id}\" not found"))?;
let children_terminal = self
.store
.execution_children(id)?
.into_iter()
.all(|child| terminal(child.status));
if terminal(current.status) && children_terminal {
return Ok(current);
}
if tokio::time::Instant::now() >= deadline {
return Err(format!(
"Timed out canceling poll execution \"{id}\" while waiting for cleanup"
));
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
}
let children = self
.store
.execution_children(id)?
.into_iter()
.map(|execution| execution.id)
.collect::<Vec<_>>();
for child in children {
Box::pin(self.cancel_unclaimed(&child, timeout, controller)).await?;
}
if self.runs.store().get_run(id)?.is_none() {
let preserve_progress = execution.status == ExecutionStatus::Stalled;
execution.status = ExecutionStatus::Canceled;
execution.updated_at = now_ms();
if !preserve_progress {
execution.last_progress_at = execution.updated_at;
}
let mut result = execution
.result
.take()
.and_then(|value| value.as_object().cloned())
.unwrap_or_default();
result.insert("canceled".to_string(), serde_json::Value::Bool(true));
execution.result = Some(serde_json::Value::Object(result));
self.store.put_execution_record(&execution)?;
self.store.release_lease(&lease_id(id))?;
self.event(
&execution,
controller,
"execution.canceled",
execution.result.clone().unwrap_or_default(),
)?;
return Ok(execution);
}
let run = self.runs.stop(id, timeout, controller).await?;
let execution = self
.store
.get_execution_record(id)?
.ok_or_else(|| format!("Execution \"{id}\" not found"))?;
let execution = self.sync_run(execution, &run, Some(controller))?;
self.store.release_lease(&lease_id(id))?;
Ok(execution)
}
pub async fn signal(
&self,
id: &str,
signal: &str,
idempotency_key: &str,
controller: &ProcessController,
) -> Result<Execution, String> {
let execution = self
.store
.get_execution_record(id)?
.ok_or_else(|| format!("Execution \"{id}\" not found"))?;
crate::process::parse_signal(signal)?;
let operation = "execution.signal";
let fingerprint = execution_control_fingerprint(
operation,
&serde_json::json!({ "id": id, "signal": signal }),
)?;
if let Some(replay) =
self.execution_control_replay::<Execution>(idempotency_key, operation, &fingerprint)?
{
return Ok(replay);
}
if terminal(execution.status) {
return Err(format!("Execution \"{id}\" is not active"));
}
let provider_control = uses_backend_gateway(&execution.spec.backend)
.then(|| provider_execution_control(&execution))
.transpose()?;
if let Some(replay) =
self.claim_execution_control::<Execution>(idempotency_key, operation, &fingerprint)?
{
return Ok(replay);
}
let result = if let Some(control) = provider_control {
self.event(
&execution,
controller,
"execution.signal.intent",
serde_json::json!({ "signal": signal }),
)?;
self.provider.signal(control, signal).await?;
let current = self.get(id)?;
self.event(
¤t,
controller,
"execution.signal.result",
serde_json::json!({ "signal": signal, "applied": true }),
)?;
current
} else {
let run = self.runs.signal(id, signal, controller).await?;
self.sync_run(execution, &run, Some(controller))?
};
self.record_execution_control(idempotency_key, operation, fingerprint, &result)?;
Ok(result)
}
pub async fn input(
&self,
id: &str,
data: &[u8],
idempotency_key: &str,
controller: &ProcessController,
) -> Result<InputReceipt, String> {
let execution = self
.store
.get_execution_record(id)?
.ok_or_else(|| format!("Execution \"{id}\" not found"))?;
let operation = "execution.input";
let fingerprint = execution_control_fingerprint(
operation,
&serde_json::json!({
"id": id,
"data_sha256": format!("sha256:{:x}", Sha256::digest(data))
}),
)?;
if let Some(replay) =
self.execution_control_replay::<InputReceipt>(idempotency_key, operation, &fingerprint)?
{
return Ok(replay);
}
if terminal(execution.status) {
return Err(format!("Execution \"{id}\" is not active"));
}
let provider_control = uses_backend_gateway(&execution.spec.backend)
.then(|| provider_execution_control(&execution))
.transpose()?;
if let Some(replay) =
self.claim_execution_control::<InputReceipt>(idempotency_key, operation, &fingerprint)?
{
return Ok(replay);
}
let receipt = if let Some(control) = provider_control {
self.event(
&execution,
controller,
"execution.input.intent",
serde_json::json!({ "bytes": data.len() }),
)?;
self.provider.input(control, data).await?;
self.event(
&self.get(id)?,
controller,
"execution.input.result",
serde_json::json!({ "bytes": data.len(), "recipients": 1 }),
)?;
InputReceipt {
id: id.to_string(),
recipients: 1,
bytes: data.len(),
}
} else {
self.runs.input(id, data, controller).await?
};
self.record_execution_control(idempotency_key, operation, fingerprint, &receipt)?;
Ok(receipt)
}
fn execution_control_replay<T: DeserializeOwned>(
&self,
idempotency_key: &str,
operation: &str,
fingerprint: &str,
) -> Result<Option<T>, String> {
if idempotency_key.is_empty() {
return Err(format!("{operation} requires idempotency_key"));
}
let Some(replay) = self.store.get_receipt::<ExecutionControlReplay>(&format!(
"execution-control:{idempotency_key}"
))?
else {
return Ok(None);
};
execution_control_replay_value(idempotency_key, operation, fingerprint, replay).map(Some)
}
fn claim_execution_control<T: DeserializeOwned>(
&self,
idempotency_key: &str,
operation: &str,
fingerprint: &str,
) -> Result<Option<T>, String> {
let intent = ExecutionControlReplay {
operation: operation.to_string(),
fingerprint: fingerprint.to_string(),
result: None,
};
let Some(replay) = self.store.claim_receipt(
&format!("execution-control:{idempotency_key}"),
"execution_control",
&intent,
)?
else {
return Ok(None);
};
execution_control_replay_value(idempotency_key, operation, fingerprint, replay).map(Some)
}
fn record_execution_control(
&self,
idempotency_key: &str,
operation: &str,
fingerprint: String,
result: &impl Serialize,
) -> Result<(), String> {
self.store.put_receipt(
&format!("execution-control:{idempotency_key}"),
"execution_control",
&ExecutionControlReplay {
operation: operation.to_string(),
fingerprint,
result: Some(serde_json::to_value(result).map_err(|error| error.to_string())?),
},
)
}
pub fn logs(&self, id: &str, tail_bytes: usize) -> Result<RunLogs, String> {
if self.runs.store().get_run(id)?.is_some() {
return self.runs.logs(id, tail_bytes);
}
let execution = self.get(id)?;
if !uses_backend_gateway(&execution.spec.backend)
&& !matches!(execution.spec.program, ExecutionProgram::Commands { .. })
{
return Err(format!("Execution \"{id}\" has no process logs"));
}
let directory = run_directory(&self.store, id);
if redaction_pending(directory.join("stdout.log").to_str())
|| redaction_pending(directory.join("stderr.log").to_str())
{
return Err("Execution logs are unavailable until secret redaction completes".into());
}
let (stdout, stdout_truncated) = read_tail_path(&directory.join("stdout.log"), tail_bytes)?;
let (stderr, stderr_truncated) = read_tail_path(&directory.join("stderr.log"), tail_bytes)?;
Ok(RunLogs {
id: id.to_string(),
stdout,
stderr,
stdout_truncated,
stderr_truncated,
tail_bytes,
})
}
pub(crate) fn stream_chunk(
&self,
id: &str,
stream: &str,
offset: u64,
max_bytes: usize,
) -> Result<ExecutionStreamChunk, String> {
let execution = self.get(id)?;
let file_name = match stream {
"stdout" => "stdout.log",
"stderr" => "stderr.log",
other => return Err(format!("Unknown execution stream \"{other}\"")),
};
let path = run_directory(&self.store, id).join(file_name);
let terminal = terminal(execution.status);
let mut file = match std::fs::File::open(&path) {
Ok(file) => file,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
return Ok(ExecutionStreamChunk {
bytes: Vec::new(),
next_offset: offset,
terminal,
});
}
Err(error) => return Err(error.to_string()),
};
let length = file.metadata().map_err(|error| error.to_string())?.len();
if offset > length {
return Err(format!(
"Execution stream offset {offset} exceeds retained length {length}"
));
}
file.seek(SeekFrom::Start(offset))
.map_err(|error| error.to_string())?;
let mut bytes = vec![0_u8; max_bytes.clamp(1, 1024 * 1024)];
let read = file.read(&mut bytes).map_err(|error| error.to_string())?;
bytes.truncate(read);
Ok(ExecutionStreamChunk {
bytes,
next_offset: offset.saturating_add(read as u64),
terminal,
})
}
pub fn log_slice(
&self,
id: &str,
request: &crate::run::LogSliceRequest,
) -> Result<crate::run::LogSlice, String> {
if self.runs.store().get_run(id)?.is_some() {
return self.runs.slice(id, request);
}
let execution = self.get(id)?;
if !matches!(execution.spec.program, ExecutionProgram::Commands { .. }) {
return Err(format!("Execution \"{id}\" has no process logs"));
}
let directory = run_directory(&self.store, id);
if redaction_pending(directory.join("stdout.log").to_str())
|| redaction_pending(directory.join("stderr.log").to_str())
{
return Err("Execution logs are unavailable until secret redaction completes".into());
}
let streams = [
("stdout", directory.join("stdout.log")),
("stderr", directory.join("stderr.log")),
];
let mut lines = Vec::new();
let mut totals = [0usize; 2];
let mut matches = 0usize;
let mut truncated = false;
for (index, (stream, path)) in streams.into_iter().enumerate() {
let path = path.to_str();
match &request.pattern {
Some(pattern) => {
let (found, count, total) = crate::log_file::read_matches(
path,
stream,
pattern,
request.context,
request.max_matches,
)?;
truncated |= count > request.max_matches;
matches += count;
totals[index] = total;
lines.extend(found);
}
None => {
let (found, total) =
crate::log_file::read_lines(path, stream, request.start, request.end)?;
totals[index] = total;
lines.extend(found);
}
}
}
Ok(crate::run::LogSlice {
id: id.to_string(),
lines,
stdout_lines: totals[0],
stderr_lines: totals[1],
total_matches: request.pattern.as_ref().map(|_| matches),
truncated,
})
}
pub async fn remove(&self, id: &str) -> Result<bool, String> {
let execution = self.get(id)?;
if !terminal(execution.status) {
return Err(format!(
"Execution \"{id}\" is active; cancel it before removal"
));
}
self.store.release_lease(&lease_id(id))?;
self.runs.remove(id).await?;
self.store.delete_execution_record(id)
}
fn renew_execution_lease(
&self,
execution: &Execution,
run: &crate::model::ManagedRun,
) -> Result<(), String> {
let Some(mut lease) = self.store.get_lease(&lease_id(&execution.id))? else {
return Ok(());
};
let resources = run
.processes
.iter()
.map(|identity| format!("process/{}", process_context_key(identity)))
.collect::<Vec<_>>();
if lease.resources == resources {
return Ok(());
}
lease.resources = resources;
lease.renewed_at = now_ms();
self.store.put_lease(&lease)
}
fn sync_run(
&self,
mut execution: Execution,
run: &crate::model::ManagedRun,
controller: Option<&ProcessController>,
) -> Result<Execution, String> {
let previous = execution.status;
let progress_timeout = execution
.result
.as_ref()
.and_then(|result| result.get("error_code"))
.and_then(serde_json::Value::as_str)
== Some("EXECUTION_PROGRESS_TIMEOUT");
let stalled_last_progress_at = execution
.result
.as_ref()
.filter(|_| progress_timeout)
.and_then(|result| result.get("last_progress_at"))
.and_then(serde_json::Value::as_u64);
let next = execution_status(&run.status);
execution.status =
if previous == ExecutionStatus::Stalled && next == ExecutionStatus::Running {
ExecutionStatus::Stalled
} else {
next
};
for process in &run.process_history {
if execution
.attempts
.iter()
.any(|attempt| attempt.processes.contains(process))
{
continue;
}
let active = run.processes.contains(process);
execution.attempts.push(ExecutionAttempt {
id: uuid::Uuid::now_v7().to_string(),
number: execution.attempts.len() as u32 + 1,
processes: vec![process.clone()],
started_at: process.started_at_ns / 1_000_000,
completed_at: (!active).then_some(run.updated_at),
exit_code: (!active).then_some(run.exit_code).flatten(),
error_code: None,
stdout_artifact: None,
stderr_artifact: None,
});
}
if !terminal(execution.status) {
self.renew_execution_lease(&execution, run)?;
if previous != execution.status {
execution.updated_at = run.updated_at;
execution.last_progress_at = execution.last_progress_at.max(run.updated_at);
self.store.put_execution_record(&execution)?;
let controller = controller.cloned().unwrap_or_else(|| {
ProcessController::capture(false, Some(execution.id.clone()))
});
self.event(
&execution,
&controller,
&format!("execution.{}", status_name(execution.status)),
execution.result.clone().unwrap_or_default(),
)?;
}
return Ok(execution);
}
execution.updated_at = run.updated_at;
if let Some(last_progress_at) = stalled_last_progress_at {
execution.last_progress_at = last_progress_at;
} else if previous != ExecutionStatus::Stalled && !progress_timeout {
execution.last_progress_at = run.updated_at;
}
for attempt in &mut execution.attempts {
if attempt.completed_at.is_none() {
attempt.completed_at = Some(run.updated_at);
attempt.exit_code = run.exit_code;
}
}
let output_truncated = execution
.result
.as_ref()
.and_then(|value| value.get("output_truncated"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false)
|| execution
.spec
.budget
.artifact_bytes
.map(|limit| enforce_log_budget(run, limit))
.transpose()?
.unwrap_or(false);
if run.status == "timed_out" {
let mut result = execution
.result
.take()
.and_then(|value| value.as_object().cloned())
.unwrap_or_default();
result.insert(
"error_code".to_string(),
serde_json::Value::from("EXECUTION_TIMEOUT"),
);
result.insert(
"timeout_ms".to_string(),
serde_json::json!(run.spec.timeout_ms),
);
result.insert("exit_code".to_string(), serde_json::json!(run.exit_code));
result.insert(
"attempts".to_string(),
serde_json::json!(execution.attempts.len()),
);
result.insert(
"output_truncated".to_string(),
serde_json::Value::Bool(output_truncated),
);
execution.result = Some(serde_json::Value::Object(result));
} else if execution
.result
.as_ref()
.and_then(|value| value.get("error_code"))
.is_none()
{
execution.result = Some(serde_json::json!({
"exit_code": run.exit_code,
"attempts": execution.attempts.len(),
"output_truncated": output_truncated
}));
} else if let Some(serde_json::Value::Object(result)) = &mut execution.result {
result.insert(
"output_truncated".to_string(),
serde_json::Value::Bool(output_truncated),
);
}
self.record_expectation(&mut execution, run);
self.store.release_lease(&lease_id(&execution.id))?;
self.store.put_execution_record(&execution)?;
if previous != execution.status {
let controller = controller
.cloned()
.unwrap_or_else(|| ProcessController::capture(false, Some(execution.id.clone())));
self.event(
&execution,
&controller,
&format!("execution.{}", status_name(execution.status)),
execution.result.clone().unwrap_or_default(),
)?;
}
Ok(execution)
}
fn finalize(&self, id: String, controller: ProcessController) {
let manager = self.clone();
tokio::spawn(async move {
let Ok(run) = manager.runs.wait(&id, None).await else {
return;
};
tokio::time::sleep(Duration::from_millis(25)).await;
let Ok(Some(execution)) = manager.store.get_execution_record(&id) else {
return;
};
let _ = manager.sync_run(execution, &run, Some(&controller));
});
}
fn execute_commands(
&self,
parent: Execution,
commands: Vec<CommandUnit>,
topology: CommandTopology,
controller: ProcessController,
) {
let manager = self.clone();
tokio::spawn(async move {
let result = match topology {
CommandTopology::Parallel {
max_parallel,
fail_fast,
} => {
let outcomes = manager
.execute_parallel(
&parent,
commands,
max_parallel as usize,
fail_fast,
&controller,
)
.await;
let failed = command_group_failed(&outcomes, None);
Ok((outcomes, failed))
}
CommandTopology::Pipeline { pipefail } => {
manager
.execute_pipeline(&parent, commands, pipefail, &controller)
.await
}
};
let parent_id = parent.id.clone();
let result = match result {
Ok((outcomes, failed)) => {
manager
.finish_or_fail_commands(parent, outcomes, failed, &controller)
.await
}
Err(error) => manager.fail_command_cleanup(parent, error, &controller, false),
};
if let Err(error) = result {
tracing::error!(
execution_id = %parent_id,
error = %error,
"command group cleanup failed"
);
}
});
}
async fn resume_commands(
&self,
parent: Execution,
commands: Vec<CommandUnit>,
topology: CommandTopology,
mut children: HashMap<String, Execution>,
) {
let controller = ProcessController::capture(false, Some(parent.id.clone()));
let mut outcomes = Vec::new();
let mut missing = Vec::new();
for command in commands {
if let Some(child) = children.remove(&command.id) {
outcomes.push(self.outcome_from_child(command.id, child).await);
} else {
missing.push(command);
}
}
let failed = match topology {
CommandTopology::Parallel {
max_parallel,
fail_fast,
} => {
if fail_fast && outcomes.iter().any(|outcome| outcome.status == "failed") {
outcomes.extend(
missing.into_iter().map(|command| {
self.record_skipped_child(&parent, command, "fail_fast")
}),
);
} else {
outcomes.extend(
self.execute_parallel(
&parent,
missing,
max_parallel as usize,
fail_fast,
&controller,
)
.await,
);
}
command_group_failed(&outcomes, None)
}
CommandTopology::Pipeline { pipefail } => {
if !missing.is_empty() {
let error = "Pipeline was interrupted before every stage became durable";
let missing = match self.record_pipeline_launch_failure(&parent, missing, error)
{
Ok(outcomes) => outcomes,
Err(persist_error) => {
let parent_id = parent.id.clone();
if let Err(cleanup_error) = self.fail_command_cleanup(
parent,
format!(
"{error}; failed to persist recovered pipeline children: {persist_error}"
),
&controller,
false,
) {
tracing::error!(
execution_id = %parent_id,
error = %cleanup_error,
"recovered pipeline child cleanup failed"
);
}
return;
}
};
outcomes.extend(missing);
true
} else {
command_group_failed(&outcomes, Some(pipefail))
}
}
};
let parent_id = parent.id.clone();
if let Err(error) = self
.finish_or_fail_commands(parent, outcomes, failed, &controller)
.await
{
tracing::error!(
execution_id = %parent_id,
error = %error,
"recovered command group cleanup failed"
);
}
}
async fn outcome_from_child(&self, id: String, child: Execution) -> CommandOutcome {
let finished = if terminal(child.status) {
Ok(child)
} else {
self.wait(&child.id, None).await
};
match finished {
Ok(finished) => CommandOutcome::from_execution(
id,
&finished,
(finished.status == ExecutionStatus::Failed)
.then(|| self.logs(&finished.id, 4096).ok().map(|logs| logs.stderr))
.flatten(),
),
Err(error) => CommandOutcome::launch_failure(id, error),
}
}
async fn execute_parallel(
&self,
parent: &Execution,
commands: Vec<CommandUnit>,
max_parallel: usize,
fail_fast: bool,
controller: &ProcessController,
) -> Vec<CommandOutcome> {
let mut queued = VecDeque::from(commands);
let mut active: Vec<BatchStage> = Vec::new();
let mut outcomes = Vec::new();
let mut stop_launching = false;
let mut canceled = false;
while !queued.is_empty() || !active.is_empty() {
let parent_canceled = self
.store
.get_execution_record(&parent.id)
.ok()
.flatten()
.is_some_and(|execution| cancellation_requested(&execution));
if parent_canceled {
canceled = true;
stop_launching = true;
for stage in &mut active {
stage.canceled = true;
let _ = signal_pipeline(stage.child.id().unwrap_or(0), libc::SIGKILL);
}
}
while !stop_launching && active.len() < max_parallel {
let Some(command) = queued.pop_front() else {
break;
};
match self.launch_batch_stage(parent, command, controller).await {
Ok(stage) => active.push(stage),
Err(outcome) => {
if fail_fast {
stop_launching = true;
}
outcomes.push(outcome);
}
}
}
let mut finished = Vec::new();
for (index, stage) in active.iter_mut().enumerate() {
if stage
.deadline
.is_some_and(|deadline| tokio::time::Instant::now() >= deadline)
{
stage.timed_out = true;
let _ = signal_pipeline(stage.child.id().unwrap_or(0), libc::SIGKILL);
}
match stage.child.try_wait() {
Ok(Some(status)) => finished.push((index, status.code().unwrap_or(128))),
Ok(None) => {}
Err(_) => finished.push((index, 127)),
}
}
for (index, code) in finished.into_iter().rev() {
let stage = active.swap_remove(index);
let outcome = self.finish_batch_stage(stage, code);
if fail_fast && outcome.status == "failed" {
stop_launching = true;
}
outcomes.push(outcome);
}
if active.is_empty() && stop_launching {
break;
}
if !active.is_empty() {
tokio::time::sleep(Duration::from_millis(2)).await;
}
}
outcomes.extend(queued.into_iter().map(|command| {
self.record_skipped_child(
parent,
command,
if canceled {
"parent_canceled"
} else {
"fail_fast"
},
)
}));
outcomes
}
async fn launch_batch_stage(
&self,
parent: &Execution,
command: CommandUnit,
controller: &ProcessController,
) -> Result<BatchStage, CommandOutcome> {
let command_id = command.id.clone();
let spec = child_spec(parent, &command);
let id = uuid::Uuid::now_v7().to_string();
let now = now_ms();
let mut execution = Execution {
id: id.clone(),
parent_id: Some(parent.id.clone()),
session_id: parent.session_id.clone(),
spec,
status: ExecutionStatus::Queued,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
self.store
.put_execution_record(&execution)
.map_err(|error| CommandOutcome::launch_failure(command_id.clone(), error))?;
if let Err(error) = self.put_child_lease(&execution) {
return Err(self.record_launch_failed_child(execution, command_id, error));
}
let run_spec = match run_spec(&self.store, &execution.spec, controller) {
Ok(run_spec) => run_spec,
Err(error) => {
return Err(self.record_launch_failed_child(execution, command_id, error));
}
};
if let Err(refusal) = self
.offer_command(&run_spec, &execution.id, controller)
.await
{
return Err(self.record_launch_failed_child(execution, command_id, refusal));
}
let directory = run_directory(&self.store, &id);
if let Err(error) = std::fs::create_dir_all(&directory) {
return Err(self.record_launch_failed_child(execution, command_id, error.to_string()));
}
let stdout = directory.join("stdout.log");
let stderr = directory.join("stderr.log");
let stdout_file = match std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&stdout)
{
Ok(file) => file,
Err(error) => {
return Err(self.record_launch_failed_child(
execution,
command_id,
error.to_string(),
));
}
};
let stderr_file = match std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&stderr)
{
Ok(file) => file,
Err(error) => {
return Err(self.record_launch_failed_child(
execution,
command_id,
error.to_string(),
));
}
};
let mut process = match pipeline_command(&run_spec) {
Ok(process) => process,
Err(error) => {
return Err(self.record_launch_failed_child(execution, command_id, error));
}
};
process
.stdin(Stdio::null())
.stdout(Stdio::from(stdout_file))
.stderr(Stdio::from(stderr_file));
#[cfg(unix)]
unsafe {
process.pre_exec(|| {
if libc::setpgid(0, 0) == -1 {
return Err(std::io::Error::last_os_error());
}
Ok(())
});
}
let child = match process.spawn() {
Ok(child) => child,
Err(error) => {
return Err(self.record_launch_failed_child(
execution,
command_id,
error.to_string(),
));
}
};
let Some(pid) = child.id() else {
return Err(self.record_launch_failed_child(
execution,
command_id,
"Spawned batch command has no process ID".to_string(),
));
};
let executable = run_spec.argv.first().cloned().unwrap_or_default();
let identity = await_process_identity_fast(pid, &executable).await;
execution.status = ExecutionStatus::Running;
execution.updated_at = now_ms();
execution.last_progress_at = execution.updated_at;
execution.attempts.push(ExecutionAttempt {
id: uuid::Uuid::now_v7().to_string(),
number: 1,
processes: vec![identity.clone()],
started_at: identity.started_at_ns / 1_000_000,
completed_at: None,
exit_code: None,
error_code: None,
stdout_artifact: None,
stderr_artifact: None,
});
self.store
.put_execution_record(&execution)
.map_err(|error| CommandOutcome::launch_failure(command_id.clone(), error))?;
self.store
.put_run(&crate::model::ManagedRun {
id,
spec: run_spec,
status: "running".to_string(),
processes: vec![identity.clone()],
process_history: vec![identity],
created_at: now,
updated_at: now_ms(),
exit_code: None,
stdout_path: Some(stdout.to_string_lossy().into_owned()),
stderr_path: Some(stderr.to_string_lossy().into_owned()),
})
.map_err(|error| CommandOutcome::launch_failure(command_id.clone(), error))?;
Ok(BatchStage {
id: command_id,
execution,
child,
stderr,
deadline: command
.timeout_ms
.map(|timeout| tokio::time::Instant::now() + Duration::from_millis(timeout)),
timed_out: false,
canceled: false,
})
}
fn record_launch_failed_child(
&self,
mut execution: Execution,
command_id: String,
error: String,
) -> CommandOutcome {
let execution_id = execution.id.clone();
execution.status = ExecutionStatus::Failed;
execution.updated_at = now_ms();
execution.last_progress_at = execution.updated_at;
execution.result = Some(serde_json::json!({
"error_code": "EXECUTION_START_FAILED",
"error": error
}));
if let Err(lease_error) = self.put_child_lease(&execution) {
return CommandOutcome::failed_child(
command_id,
Some(execution_id),
None,
Some(format!(
"{error}; failed to persist child lease: {lease_error}"
)),
);
}
if let Err(persist_error) = self.store.put_execution_record(&execution) {
return CommandOutcome::launch_failure(
command_id,
format!("{error}; failed to persist launch failure: {persist_error}"),
);
}
if let Err(release_error) = self.store.release_lease(&lease_id(&execution_id)) {
return CommandOutcome::failed_child(
command_id,
Some(execution_id),
None,
Some(format!(
"{error}; failed to release child lease: {release_error}"
)),
);
}
CommandOutcome::failed_child(command_id, Some(execution_id), None, Some(error))
}
fn finish_batch_stage(&self, mut stage: BatchStage, code: i32) -> CommandOutcome {
let status = if stage.canceled {
ExecutionStatus::Canceled
} else if code == 0 && !stage.timed_out {
ExecutionStatus::Completed
} else {
ExecutionStatus::Failed
};
stage.execution.status = status;
stage.execution.updated_at = now_ms();
stage.execution.last_progress_at = stage.execution.updated_at;
stage.execution.result = Some(serde_json::json!({
"exit_code": code,
"attempts": 1,
"timed_out": stage.timed_out
}));
if let Some(attempt) = stage.execution.attempts.first_mut() {
attempt.completed_at = Some(stage.execution.updated_at);
attempt.exit_code = Some(code);
}
let child_id = stage.execution.id.clone();
self.judge_expectation(
&mut stage.execution,
&child_id,
Some(code),
status_name(status),
);
let persist = self
.store
.put_execution_record(&stage.execution)
.and_then(|_| {
let mut run = self
.store
.get_run(&stage.execution.id)?
.ok_or_else(|| "Batch child run disappeared".to_string())?;
run.status = if status == ExecutionStatus::Completed {
"completed".to_string()
} else if status == ExecutionStatus::Canceled {
"stopped".to_string()
} else {
"failed".to_string()
};
run.processes.clear();
run.updated_at = stage.execution.updated_at;
run.exit_code = Some(code);
self.store.put_run(&run)
});
if let Err(error) = persist {
return CommandOutcome::launch_failure(stage.id, error);
}
if let Err(error) = self.store.release_lease(&lease_id(&stage.execution.id)) {
return CommandOutcome::failed_child(
stage.id,
Some(stage.execution.id),
Some(code),
Some(format!("Failed to release child lease: {error}")),
);
}
CommandOutcome::from_execution(
stage.id,
&stage.execution,
(code != 0)
.then(|| {
read_tail_path(&stage.stderr, 4096)
.ok()
.map(|(value, _)| value)
})
.flatten(),
)
}
fn record_skipped_child(
&self,
parent: &Execution,
command: CommandUnit,
reason: &str,
) -> CommandOutcome {
let id = uuid::Uuid::now_v7().to_string();
let now = now_ms();
let execution = Execution {
id: id.clone(),
parent_id: Some(parent.id.clone()),
session_id: parent.session_id.clone(),
spec: child_spec(parent, &command),
status: ExecutionStatus::Canceled,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: Some(serde_json::json!({ "skipped": true, "reason": reason })),
};
match self
.store
.put_execution_record(&execution)
.and_then(|_| self.put_child_lease(&execution))
.and_then(|_| self.store.release_lease(&lease_id(&id)).map(|_| ()))
{
Ok(()) => CommandOutcome::from_execution(command.id, &execution, None),
Err(error) => CommandOutcome::launch_failure(command.id, error),
}
}
fn put_child_lease(&self, execution: &Execution) -> Result<(), String> {
let id = lease_id(&execution.id);
if self.store.get_lease(&id)?.is_some() {
return Ok(());
}
self.store.put_lease(&Lease {
id,
owner_kind: LeaseOwnerKind::Execution,
owner_id: execution
.parent_id
.clone()
.unwrap_or_else(|| execution.id.clone()),
resources: Vec::new(),
created_at: execution.created_at,
renewed_at: execution.updated_at,
expires_at: execution.spec.expires_at,
released: false,
})
}
async fn execute_pipeline(
&self,
parent: &Execution,
commands: Vec<CommandUnit>,
pipefail: bool,
controller: &ProcessController,
) -> Result<(Vec<CommandOutcome>, bool), String> {
match self
.run_native_pipeline(parent, commands.clone(), controller)
.await
{
Ok(outcomes) => {
let failed = command_group_failed(&outcomes, Some(pipefail));
Ok((outcomes, failed))
}
Err(error) => self
.record_pipeline_launch_failure(parent, commands.clone(), &error)
.map(|outcomes| (outcomes, true))
.map_err(|persist_error| {
format!("{error}; failed to persist pipeline launch cleanup: {persist_error}")
}),
}
}
fn fail_command_cleanup(
&self,
mut parent: Execution,
error: String,
controller: &ProcessController,
children_complete: bool,
) -> Result<(), String> {
let current = self.get(&parent.id)?;
if terminal(current.status) {
return Ok(());
}
parent.status = ExecutionStatus::Failed;
parent.updated_at = now_ms();
parent.last_progress_at = parent.updated_at;
parent.result = Some(serde_json::json!({
"error_code": "EXECUTION_CLEANUP_FAILED",
"error": error,
"children_complete": children_complete
}));
self.store.put_execution_record(&parent)?;
self.store.release_lease(&lease_id(&parent.id))?;
self.event(
&parent,
controller,
"execution.failed",
parent.result.clone().unwrap_or_default(),
)?;
Ok(())
}
async fn finish_or_fail_commands(
&self,
parent: Execution,
outcomes: Vec<CommandOutcome>,
parent_failed: bool,
controller: &ProcessController,
) -> Result<(), String> {
let recovery = parent.clone();
let Err(error) = self
.finish_commands(parent, outcomes, parent_failed, controller)
.await
else {
return Ok(());
};
let current = self.get(&recovery.id).map_err(|inspect_error| {
format!(
"Command group finalization failed: {error}; failed to inspect parent state: {inspect_error}"
)
})?;
if terminal(current.status) {
self.store.release_lease(&lease_id(¤t.id))?;
let kind = format!("execution.{}", status_name(current.status));
let subject = format!("execution/{}", current.id);
if !self
.store
.causal_events_for_subject(&subject, 100)?
.iter()
.any(|event| event.kind == kind)
{
self.event(
¤t,
controller,
&kind,
current.result.clone().unwrap_or_default(),
)?;
}
return Ok(());
}
if cancellation_requested(¤t) {
let mut canceled = recovery;
canceled.status = ExecutionStatus::Canceled;
canceled.updated_at = now_ms();
canceled.last_progress_at = canceled.updated_at;
canceled.result = Some(serde_json::json!({ "canceled": true }));
self.store.put_execution_record(&canceled)?;
self.store.release_lease(&lease_id(&canceled.id))?;
self.event(
&canceled,
controller,
"execution.canceled",
canceled.result.clone().unwrap_or_default(),
)?;
return Ok(());
}
self.fail_command_cleanup(
recovery,
format!("Command group finalization failed: {error}"),
controller,
true,
)
}
fn record_pipeline_launch_failure(
&self,
parent: &Execution,
commands: Vec<CommandUnit>,
error: &str,
) -> Result<Vec<CommandOutcome>, String> {
let mut children = self
.store
.execution_children(&parent.id)?
.into_iter()
.filter_map(|execution| {
execution
.spec
.labels
.get("apoc.command_id")
.cloned()
.map(|id| (id, execution))
})
.collect::<HashMap<_, _>>();
let mut outcomes = Vec::with_capacity(commands.len());
for command in commands {
let command_id = command.id.clone();
let now = now_ms();
let mut execution = children.remove(&command_id).unwrap_or_else(|| Execution {
id: uuid::Uuid::now_v7().to_string(),
parent_id: Some(parent.id.clone()),
session_id: parent.session_id.clone(),
spec: child_spec(parent, &command),
status: ExecutionStatus::Queued,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
});
if let Some(mut run) = self.store.get_run(&execution.id)? {
for process in &run.processes {
let _ = signal_pipeline(process.pid, libc::SIGKILL);
}
run.status = "failed".to_string();
run.processes.clear();
run.updated_at = now;
self.store.put_run(&run)?;
}
execution.status = ExecutionStatus::Failed;
execution.updated_at = now;
execution.last_progress_at = now;
for attempt in &mut execution.attempts {
if attempt.completed_at.is_none() {
attempt.completed_at = Some(now);
attempt.error_code = Some("EXECUTION_START_FAILED".to_string());
}
}
execution.result = Some(serde_json::json!({
"error_code": "EXECUTION_START_FAILED",
"error": error
}));
self.store.put_execution_record(&execution)?;
let lease = lease_id(&execution.id);
if self.store.get_lease(&lease)?.is_none() {
self.store.put_lease(&Lease {
id: lease.clone(),
owner_kind: LeaseOwnerKind::Execution,
owner_id: parent.id.clone(),
resources: Vec::new(),
created_at: now,
renewed_at: now,
expires_at: parent.spec.expires_at,
released: false,
})?;
}
self.store.release_lease(&lease)?;
outcomes.push(CommandOutcome::failed_child(
command_id,
Some(execution.id),
None,
Some(error.to_string()),
));
}
Ok(outcomes)
}
async fn run_native_pipeline(
&self,
parent: &Execution,
commands: Vec<CommandUnit>,
controller: &ProcessController,
) -> Result<Vec<CommandOutcome>, String> {
let parent_directory = run_directory(&self.store, &parent.id);
std::fs::create_dir_all(&parent_directory).map_err(|error| error.to_string())?;
let parent_stdout = parent_directory.join("stdout.log");
let parent_stderr = parent_directory.join("stderr.log");
let mut pipes = (0..commands.len().saturating_sub(1))
.map(|_| native_pipe())
.collect::<Result<Vec<_>, _>>()?;
let mut stages = Vec::with_capacity(commands.len());
let mut pgid: Option<u32> = None;
for (index, command) in commands.iter().enumerate() {
let spec = child_spec(parent, command);
let run_spec = run_spec(&self.store, &spec, controller)?;
self.offer_command(&run_spec, &parent.id, controller)
.await?;
let id = uuid::Uuid::now_v7().to_string();
let now = now_ms();
let mut execution = Execution {
id: id.clone(),
parent_id: Some(parent.id.clone()),
session_id: parent.session_id.clone(),
spec,
status: ExecutionStatus::Queued,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
self.store.put_execution_record(&execution)?;
self.store.put_lease(&Lease {
id: lease_id(&id),
owner_kind: LeaseOwnerKind::Execution,
owner_id: parent.id.clone(),
resources: Vec::new(),
created_at: now,
renewed_at: now,
expires_at: parent.spec.expires_at,
released: false,
})?;
let directory = run_directory(&self.store, &id);
std::fs::create_dir_all(&directory).map_err(|error| error.to_string())?;
let stderr = directory.join("stderr.log");
let stdout = (index + 1 == commands.len()).then(|| parent_stdout.clone());
let mut process = pipeline_command(&run_spec)?;
if index == 0 {
process.stdin(Stdio::null());
} else {
process.stdin(Stdio::from(pipes[index - 1].0.take().ok_or_else(|| {
"Pipeline read end was already consumed".to_string()
})?));
}
if index + 1 == commands.len() {
process.stdout(Stdio::piped());
} else {
process.stdout(Stdio::from(pipes[index].1.take().ok_or_else(|| {
"Pipeline write end was already consumed".to_string()
})?));
}
process.stderr(Stdio::piped());
let requested_group = pgid;
#[cfg(unix)]
unsafe {
process.pre_exec(move || {
let group = requested_group.unwrap_or(0) as i32;
if libc::setpgid(0, group) == -1 {
return Err(std::io::Error::last_os_error());
}
Ok(())
});
}
let mut child = process.spawn().map_err(|error| {
if let Some(pgid) = pgid {
let _ = signal_pipeline(pgid, libc::SIGKILL);
}
error.to_string()
})?;
let pid = child
.id()
.ok_or_else(|| "Spawned pipeline stage has no process ID".to_string())?;
pgid.get_or_insert(pid);
let identity = await_process_identity(pid).await;
let stderr_pump = child
.stderr
.take()
.map(|reader| pump_log(reader, stderr.clone()));
let stdout_pump = child
.stdout
.take()
.map(|reader| pump_log(reader, parent_stdout.clone()));
execution.status = ExecutionStatus::Running;
execution.updated_at = now_ms();
execution.last_progress_at = execution.updated_at;
execution.attempts.push(ExecutionAttempt {
id: uuid::Uuid::now_v7().to_string(),
number: 1,
processes: vec![identity.clone()],
started_at: identity.started_at_ns / 1_000_000,
completed_at: None,
exit_code: None,
error_code: None,
stdout_artifact: None,
stderr_artifact: None,
});
self.store.put_execution_record(&execution)?;
self.store.put_run(&crate::model::ManagedRun {
id: id.clone(),
spec: run_spec,
status: "running".to_string(),
processes: vec![identity.clone()],
process_history: vec![identity.clone()],
created_at: now,
updated_at: now_ms(),
exit_code: None,
stdout_path: stdout.map(|path| path.to_string_lossy().into_owned()),
stderr_path: Some(stderr.to_string_lossy().into_owned()),
})?;
stages.push(PipelineStage {
id: command.id.clone(),
execution,
child,
stderr,
stderr_pump,
stdout_pump,
deadline: command
.timeout_ms
.map(|timeout| tokio::time::Instant::now() + Duration::from_millis(timeout)),
exit_code: None,
});
}
drop(pipes);
let group = pgid.ok_or_else(|| "Pipeline has no process group".to_string())?;
let mut canceled = false;
while stages.iter().any(|stage| stage.exit_code.is_none()) {
if self
.store
.get_execution_record(&parent.id)?
.is_some_and(|execution| cancellation_requested(&execution))
|| stages.iter().any(|stage| {
stage.exit_code.is_none()
&& stage
.deadline
.is_some_and(|deadline| tokio::time::Instant::now() >= deadline)
})
{
canceled = true;
let _ = signal_pipeline(group, libc::SIGKILL);
}
for stage in &mut stages {
if stage.exit_code.is_some() {
continue;
}
if let Some(status) = stage.child.try_wait().map_err(|error| error.to_string())? {
stage.exit_code = Some(status.code().unwrap_or(128));
}
}
if stages.iter().any(|stage| stage.exit_code.is_none()) {
tokio::time::sleep(Duration::from_millis(10)).await;
}
}
let mut outcomes = Vec::with_capacity(stages.len());
for mut stage in stages {
if let Some(pump) = stage.stderr_pump.take() {
let _ = pump.await;
}
if let Some(pump) = stage.stdout_pump.take() {
let _ = pump.await;
}
let code = stage.exit_code.unwrap_or(127);
let status = if canceled {
ExecutionStatus::Canceled
} else if code == 0 {
ExecutionStatus::Completed
} else {
ExecutionStatus::Failed
};
stage.execution.status = status;
stage.execution.updated_at = now_ms();
stage.execution.last_progress_at = stage.execution.updated_at;
stage.execution.result = Some(serde_json::json!({ "exit_code": code, "attempts": 1 }));
if let Some(attempt) = stage.execution.attempts.first_mut() {
attempt.completed_at = Some(stage.execution.updated_at);
attempt.exit_code = Some(code);
}
let stage_id = stage.execution.id.clone();
self.judge_expectation(
&mut stage.execution,
&stage_id,
Some(code),
status_name(status),
);
self.store.put_execution_record(&stage.execution)?;
if let Some(mut run) = self.store.get_run(&stage.execution.id)? {
run.status = if status == ExecutionStatus::Completed {
"completed".to_string()
} else if status == ExecutionStatus::Canceled {
"stopped".to_string()
} else {
"failed".to_string()
};
run.processes.clear();
run.updated_at = stage.execution.updated_at;
run.exit_code = Some(code);
self.store.put_run(&run)?;
}
self.store.release_lease(&lease_id(&stage.execution.id))?;
outcomes.push(CommandOutcome::from_execution(
stage.id,
&stage.execution,
(code != 0)
.then(|| read_tail_path(&stage.stderr, 4096).map(|(value, _)| value))
.transpose()?,
));
}
if !parent_stderr.exists() {
std::fs::File::create(parent_stderr).map_err(|error| error.to_string())?;
}
Ok(outcomes)
}
async fn finish_commands(
&self,
mut parent: Execution,
outcomes: Vec<CommandOutcome>,
parent_failed: bool,
controller: &ProcessController,
) -> Result<(), String> {
let current = self.get(&parent.id)?;
if cancellation_requested(¤t) {
parent.status = ExecutionStatus::Canceled;
parent.updated_at = now_ms();
parent.last_progress_at = parent.updated_at;
parent.result = Some(serde_json::json!({ "canceled": true }));
self.store.put_execution_record(&parent)?;
self.store.release_lease(&lease_id(&parent.id))?;
self.event(
&parent,
controller,
"execution.canceled",
parent.result.clone().unwrap_or_default(),
)?;
return Ok(());
}
let completed = outcomes
.iter()
.filter(|outcome| outcome.status == "completed")
.count();
let failed = outcomes
.iter()
.filter(|outcome| outcome.status == "failed")
.count();
let canceled = outcomes
.iter()
.filter(|outcome| outcome.status == "canceled")
.count();
let skipped = outcomes
.iter()
.filter(|outcome| outcome.status == "skipped")
.count();
let artifact = self
.store
.put(
&parent.id,
&serde_json::to_value(&outcomes).map_err(|error| error.to_string())?,
)
.await?;
parent.status = if parent_failed {
ExecutionStatus::Failed
} else {
ExecutionStatus::Completed
};
parent.updated_at = now_ms();
parent.last_progress_at = parent.updated_at;
parent.result = Some(serde_json::json!({
"parent_id": parent.id.clone(),
"counts": {
"total": outcomes.len(),
"completed": completed,
"failed": failed,
"canceled": canceled,
"skipped": skipped
},
"failures": outcomes.iter().filter(|outcome| outcome.status == "failed").collect::<Vec<_>>(),
"children_artifact": artifact.id
}));
self.store.put_execution_record(&parent)?;
self.store.release_lease(&lease_id(&parent.id))?;
self.event(
&parent,
controller,
&format!("execution.{}", status_name(parent.status)),
parent.result.clone().unwrap_or_default(),
)?;
Ok(())
}
fn supervise_budgets(
&self,
id: String,
budget: ExecutionBudget,
controller: ProcessController,
) {
let resource_limits = budget.child_processes.is_some()
|| budget.memory_bytes.is_some()
|| budget.cpu_percent.is_some()
|| budget.cpu_time_ms.is_some();
if budget.progress_timeout_ms.is_none() && !resource_limits {
return;
}
let manager = self.clone();
tokio::spawn(async move {
let mut interval_ms = budget.progress_timeout_ms.unwrap_or(100).clamp(10, 250);
if resource_limits {
interval_ms = interval_ms.min(100);
}
let interval = Duration::from_millis(interval_ms);
let capture = TelemetryCapture {
schema_version: SCHEMA_VERSION,
id: format!("budget:{id}"),
execution_id: id.clone(),
interval_ms: interval.as_millis() as u64,
max_bytes: 0,
status: TelemetryCaptureStatus::Active,
started_at: now_ms(),
stopped_at: None,
next_sequence: 0,
sample_count: 0,
retained_bytes: 0,
dropped_samples: 0,
error: None,
};
let mut known = Vec::new();
let mut tick = 0_u64;
let mut observed_output_bytes = manager.retained_output_bytes(&id).unwrap_or(0);
loop {
tokio::time::sleep(interval).await;
let Ok(Some(mut execution)) = manager.store.get_execution_record(&id) else {
return;
};
if terminal(execution.status) {
return;
}
if let Ok(output_bytes) = manager.retained_output_bytes(&id)
&& output_bytes > observed_output_bytes
{
observed_output_bytes = output_bytes;
execution.updated_at = now_ms();
execution.last_progress_at = execution.updated_at;
let _ = manager.store.put_execution_record(&execution);
}
if resource_limits {
let discover = tick.is_multiple_of(10);
tick = tick.saturating_add(1);
let Ok((_, sample)) =
manager.collect_telemetry_sample(&capture, discover, &mut known)
else {
continue;
};
if let Some(details) = resource_budget_violation(&budget, &sample) {
execution.result = Some(details.clone());
execution.updated_at = now_ms();
execution.last_progress_at = execution.updated_at;
let _ = manager.store.put_execution_record(&execution);
let _ = manager.event(
&execution,
&controller,
"execution.budget_exceeded",
details,
);
let _ = manager
.cancel_unclaimed(&id, Duration::from_secs(5), &controller)
.await;
return;
}
}
if let Some(timeout_ms) = budget.progress_timeout_ms
&& now_ms().saturating_sub(execution.last_progress_at) >= timeout_ms
{
if execution.status != ExecutionStatus::Stalled {
let stalled_at = now_ms();
execution.status = ExecutionStatus::Stalled;
execution.updated_at = stalled_at;
execution.result = Some(serde_json::json!({
"error_code": "EXECUTION_PROGRESS_TIMEOUT",
"progress_timeout_ms": timeout_ms,
"last_progress_at": execution.last_progress_at,
"stalled_at": stalled_at
}));
let _ = manager.store.put_execution_record(&execution);
let _ = manager.event(
&execution,
&controller,
"execution.stalled",
execution.result.clone().unwrap_or_default(),
);
}
if budget.cancel_on_stall {
let _ = manager
.cancel_unclaimed(&id, Duration::from_secs(5), &controller)
.await;
}
return;
}
}
});
}
fn retained_output_bytes(&self, id: &str) -> Result<u64, String> {
if let Some(run) = self.runs.store().get_run(id)? {
return Ok(file_size(run.stdout_path.as_deref())?
.saturating_add(file_size(run.stderr_path.as_deref())?));
}
let directory = run_directory(&self.store, id);
Ok(file_size(directory.join("stdout.log").to_str())?
.saturating_add(file_size(directory.join("stderr.log").to_str())?))
}
fn event(
&self,
execution: &Execution,
controller: &ProcessController,
kind: &str,
details: serde_json::Value,
) -> Result<CausalEvent, String> {
self.store.append_causal_event(CausalEvent {
sequence: 0,
id: uuid::Uuid::now_v7().to_string(),
at: now_ms(),
kind: kind.to_string(),
subject: format!("execution/{}", execution.id),
actor: controller.actor.clone(),
correlation_id: execution.id.clone(),
causation_id: None,
execution_id: Some(execution.id.clone()),
session_id: execution.session_id.clone(),
payload_id: None,
details,
})
}
}
fn resource_budget_violation(
budget: &ExecutionBudget,
sample: &TelemetrySample,
) -> Option<serde_json::Value> {
if budget
.child_processes
.is_some_and(|limit| sample.process_count > u64::from(limit))
{
return Some(serde_json::json!({
"error_code": "EXECUTION_CHILD_PROCESS_BUDGET",
"limit": budget.child_processes,
"observed": sample.process_count
}));
}
if budget
.memory_bytes
.is_some_and(|limit| sample.memory_bytes > limit)
{
return Some(serde_json::json!({
"error_code": "EXECUTION_MEMORY_BUDGET",
"limit": budget.memory_bytes,
"observed": sample.memory_bytes
}));
}
if budget
.cpu_time_ms
.is_some_and(|limit| sample.cpu_time_ms > limit)
{
return Some(serde_json::json!({
"error_code": "EXECUTION_CPU_TIME_BUDGET",
"limit": budget.cpu_time_ms,
"observed": sample.cpu_time_ms
}));
}
if budget
.cpu_percent
.is_some_and(|limit| sample.cpu_percent > limit)
{
return Some(serde_json::json!({
"error_code": "EXECUTION_CPU_PERCENT_BUDGET",
"limit": budget.cpu_percent,
"observed": sample.cpu_percent
}));
}
None
}
fn child_spec(parent: &Execution, command: &CommandUnit) -> ExecutionSpec {
let mut labels = parent.spec.labels.clone();
labels.insert("apoc.command_id".to_string(), command.id.clone());
let mut env = parent.spec.env.clone();
for name in command.secrets.keys() {
env.remove(name);
}
env.extend(command.env.clone());
let mut secrets = parent.spec.secrets.clone();
for name in command.env.keys() {
secrets.remove(name);
}
secrets.extend(command.secrets.clone());
ExecutionSpec {
schema_version: SCHEMA_VERSION,
idempotency_key: None,
name: command.id.clone(),
purpose: parent.spec.purpose.clone(),
labels,
program: ExecutionProgram::Command {
argv: command.argv.clone(),
shell: command.shell,
},
backend: parent.spec.backend.clone(),
workspace: WorkspaceContext {
cwd: command
.cwd
.clone()
.or_else(|| parent.spec.workspace.cwd.clone()),
..parent.spec.workspace.clone()
},
env,
secrets,
profile_selection: parent.spec.profile_selection.clone(),
io: crate::model::IoMode::Pipes,
owner_kind: LeaseOwnerKind::Execution,
owner_id: Some(parent.id.clone()),
expires_at: parent.spec.expires_at,
retry: RetryPolicy::default(),
budget: ExecutionBudget {
wall_time_ms: command.timeout_ms,
artifact_bytes: parent.spec.budget.artifact_bytes,
..ExecutionBudget::default()
},
telemetry: crate::model::TelemetryPolicy::default(),
expect: command.expect.clone(),
}
}
fn run_directory(store: &Store, id: &str) -> PathBuf {
store
.path()
.parent()
.unwrap_or_else(|| Path::new("."))
.join("runs")
.join(id)
}
fn native_pipe() -> Result<(Option<OwnedFd>, Option<OwnedFd>), String> {
let mut descriptors = [0_i32; 2];
if unsafe { libc::pipe(descriptors.as_mut_ptr()) } == -1 {
return Err(std::io::Error::last_os_error().to_string());
}
Ok(unsafe {
(
Some(OwnedFd::from_raw_fd(descriptors[0])),
Some(OwnedFd::from_raw_fd(descriptors[1])),
)
})
}
fn pipeline_command(spec: &RunSpec) -> Result<TokioCommand, String> {
let mut command = if spec.shell {
let mut command = TokioCommand::new("/bin/zsh");
command.args(["-lc", &spec.argv[0]]);
command
} else {
let executable = spec
.argv
.first()
.ok_or_else(|| "argv must contain an executable".to_string())?;
let mut command = TokioCommand::new(executable);
command.args(&spec.argv[1..]);
command
};
if let Some(cwd) = &spec.cwd {
command.current_dir(cwd);
}
command.envs(&spec.env);
Ok(command)
}
async fn await_process_identity(pid: u32) -> crate::model::ProcessIdentity {
for _ in 0..20 {
if let Ok(identity) = crate::process::identity_for_pid(pid) {
return identity;
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
let started_at = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default();
crate::model::ProcessIdentity {
boot_id: format!("boot-{}", sysinfo::System::boot_time()),
pid,
ppid: Some(std::process::id()),
pgid: Some(pid),
started_at: started_at.as_secs(),
started_at_ns: started_at.as_nanos().min(u64::MAX as u128) as u64,
name: "pipeline".to_string(),
executable: None,
}
}
async fn await_process_identity_fast(pid: u32, executable: &str) -> crate::model::ProcessIdentity {
for _ in 0..2 {
if let Ok(identity) = crate::process::identity_for_spawn(pid, executable) {
return identity;
}
tokio::time::sleep(Duration::from_millis(1)).await;
}
let started_at = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default();
crate::model::ProcessIdentity {
boot_id: format!("boot-{}", sysinfo::System::boot_time()),
pid,
ppid: Some(std::process::id()),
pgid: Some(pid),
started_at: started_at.as_secs(),
started_at_ns: started_at.as_nanos().min(u64::MAX as u128) as u64,
name: "batch".to_string(),
executable: None,
}
}
fn pump_log(
mut reader: impl AsyncRead + Unpin + Send + 'static,
path: PathBuf,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let Ok(mut file) = tokio::fs::OpenOptions::new()
.create(true)
.append(true)
.open(path)
.await
else {
return;
};
let _ = tokio::io::copy(&mut reader, &mut file).await;
let _ = file.flush().await;
})
}
fn signal_pipeline(pgid: u32, signal: i32) -> Result<(), String> {
if unsafe { libc::kill(-(pgid as i32), signal) } == -1 {
let error = std::io::Error::last_os_error();
if error.raw_os_error() != Some(libc::ESRCH) {
return Err(error.to_string());
}
}
Ok(())
}
fn read_tail_path(path: &Path, tail_bytes: usize) -> Result<(String, bool), String> {
let mut file = match std::fs::File::open(path) {
Ok(file) => file,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
return Ok((String::new(), false));
}
Err(error) => return Err(error.to_string()),
};
let length = file.metadata().map_err(|error| error.to_string())?.len();
file.seek(SeekFrom::Start(
length.saturating_sub(tail_bytes.min(u64::MAX as usize) as u64),
))
.map_err(|error| error.to_string())?;
let mut bytes = Vec::new();
file.read_to_end(&mut bytes)
.map_err(|error| error.to_string())?;
Ok((
String::from_utf8_lossy(&bytes).into_owned(),
length > tail_bytes as u64,
))
}
const EXPECTATION_TAIL_BYTES: usize = 64 * 1024;
impl ExecutionManager {
fn judge_expectation(
&self,
execution: &mut Execution,
run_id: &str,
exit_code: Option<i32>,
status: &str,
) {
let Some(condition) = execution.spec.expect.clone() else {
return;
};
let (met, error) = match self
.runs
.logs(run_id, EXPECTATION_TAIL_BYTES)
.map(|logs| crate::poll::command_observation(exit_code, status, &logs))
{
Ok(observation) => match crate::poll::judge(&condition, &observation) {
Ok(met) => (met, None),
Err(error) => (false, Some(error)),
},
Err(error) => (false, Some(error)),
};
execution.record_expectation(met, error);
}
fn record_expectation(&self, execution: &mut Execution, run: &crate::model::ManagedRun) {
self.judge_expectation(execution, &run.id, run.exit_code, &run.status);
}
}
fn reported_child_status(execution: &Execution) -> &'static str {
match ExecutionVerdict::of_execution(execution) {
ExecutionVerdict::Skipped => "skipped",
ExecutionVerdict::Passed => "completed",
ExecutionVerdict::Failed => "failed",
ExecutionVerdict::Error if execution.status == ExecutionStatus::Canceled => "canceled",
ExecutionVerdict::Error => "failed",
ExecutionVerdict::Pending => status_name(execution.status),
}
}
fn command_group_failed(outcomes: &[CommandOutcome], pipefail: Option<bool>) -> bool {
if outcomes
.iter()
.any(|outcome| outcome.assertion_failed || outcome.status == "canceled")
{
return true;
}
match pipefail {
None | Some(true) => outcomes.iter().any(CommandOutcome::failed),
Some(false) => outcomes.last().is_some_and(CommandOutcome::failed),
}
}
fn enforce_log_budget(run: &crate::model::ManagedRun, limit: u64) -> Result<bool, String> {
let stdout = file_size(run.stdout_path.as_deref())?;
let stderr = file_size(run.stderr_path.as_deref())?;
if stdout.saturating_add(stderr) <= limit {
return Ok(false);
}
let half = limit / 2;
let (stdout_limit, stderr_limit) = if stdout <= half {
(stdout, limit.saturating_sub(stdout))
} else if stderr <= half {
(limit.saturating_sub(stderr), stderr)
} else {
(limit.saturating_sub(half), half)
};
truncate_tail(run.stdout_path.as_deref(), stdout_limit)?;
truncate_tail(run.stderr_path.as_deref(), stderr_limit)?;
Ok(true)
}
fn file_size(path: Option<&str>) -> Result<u64, String> {
let Some(path) = path else {
return Ok(0);
};
std::fs::metadata(path)
.map(|metadata| metadata.len())
.or_else(|error| {
if error.kind() == std::io::ErrorKind::NotFound {
Ok(0)
} else {
Err(error)
}
})
.map_err(|error| error.to_string())
}
fn truncate_tail(path: Option<&str>, limit: u64) -> Result<(), String> {
let Some(path) = path else {
return Ok(());
};
let mut file = match std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(path)
{
Ok(file) => file,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error.to_string()),
};
let length = file.metadata().map_err(|error| error.to_string())?.len();
if length <= limit {
return Ok(());
}
file.seek(SeekFrom::Start(length.saturating_sub(limit)))
.map_err(|error| error.to_string())?;
let mut tail = Vec::with_capacity(limit.min(usize::MAX as u64) as usize);
file.read_to_end(&mut tail)
.map_err(|error| error.to_string())?;
file.set_len(0).map_err(|error| error.to_string())?;
file.seek(SeekFrom::Start(0))
.map_err(|error| error.to_string())?;
file.write_all(&tail).map_err(|error| error.to_string())?;
file.flush().map_err(|error| error.to_string())
}
fn validate_spec(spec: &ExecutionSpec) -> Result<(), String> {
if spec.schema_version != SCHEMA_VERSION {
return Err(format!(
"Unsupported schema_version {}; expected {SCHEMA_VERSION}",
spec.schema_version
));
}
if spec.retry.max_attempts == 0 {
return Err("max_attempts must be at least 1".to_string());
}
if spec.retry.max_backoff_ms < spec.retry.backoff_ms {
return Err("max_backoff_ms cannot be less than backoff_ms".to_string());
}
if spec.budget.child_processes == Some(0) {
return Err("child_processes must allow the command process".to_string());
}
if spec.budget.memory_bytes == Some(0) {
return Err("memory_bytes must be greater than zero".to_string());
}
if spec.budget.cpu_time_ms == Some(0) {
return Err("cpu_time_ms must be greater than zero".to_string());
}
if spec
.budget
.cpu_percent
.is_some_and(|value| !value.is_finite() || value <= 0.0)
{
return Err("cpu_percent must be finite and greater than zero".to_string());
}
if !matches!(spec.backend, ExecutionBackendSpec::Native)
&& (spec.budget.memory_bytes.is_some()
|| spec.budget.cpu_percent.is_some()
|| spec.budget.cpu_time_ms.is_some())
{
return Err(
"Native resource budgets require the native backend; use memory_mb, cpu_count, and pids_max for isolated executions"
.to_string(),
);
}
if let TelemetryPolicy::Sample {
interval_ms,
max_bytes,
} = &spec.telemetry
{
let minimum_interval_ms = sysinfo::MINIMUM_CPU_UPDATE_INTERVAL.as_millis() as u64;
if *interval_ms < minimum_interval_ms {
return Err(format!(
"telemetry interval_ms must be at least {minimum_interval_ms}"
));
}
if *max_bytes == 0 {
return Err("telemetry max_bytes must be greater than zero".to_string());
}
}
if let Some(key) = spec.idempotency_key.as_deref()
&& (key.is_empty() || key.len() > 256)
{
return Err("idempotency_key must contain between 1 and 256 bytes".to_string());
}
if uses_backend_gateway(&spec.backend)
&& !matches!(spec.program, ExecutionProgram::Command { .. })
{
return Err(match spec.backend {
ExecutionBackendSpec::Provider { .. } => {
"Provider execution backends currently support one command program".to_string()
}
ExecutionBackendSpec::Isolated { .. } => {
"Isolated execution currently supports one command program".to_string()
}
ExecutionBackendSpec::Native => unreachable!("native execution does not use gateway"),
});
}
if matches!(spec.backend, ExecutionBackendSpec::Provider { .. }) && !spec.secrets.is_empty() {
return Err(
"Provider execution backends do not yet support secret environment values".to_string(),
);
}
match &spec.program {
ExecutionProgram::Command { argv, shell } => {
if argv.is_empty() {
return Err("argv must contain an executable".to_string());
}
if *shell && argv.len() != 1 {
return Err("shell mode accepts one zsh command string".to_string());
}
}
ExecutionProgram::Commands { commands, topology } => {
if spec.retry.max_attempts != 1 {
return Err(
"Command-group parents cannot be retried; retry child commands explicitly"
.to_string(),
);
}
if commands.is_empty() {
return Err("commands must contain at least one command".to_string());
}
let mut ids = BTreeSet::new();
for command in commands {
if command.id.is_empty() {
return Err("command ids cannot be empty".to_string());
}
if !ids.insert(&command.id) {
return Err(format!("Duplicate command id \"{}\"", command.id));
}
if command.argv.is_empty() {
return Err(format!(
"Command \"{}\" argv must contain an executable",
command.id
));
}
if command.shell && command.argv.len() != 1 {
return Err(format!(
"Command \"{}\" shell mode accepts one zsh command string",
command.id
));
}
}
if let crate::model::CommandTopology::Parallel { max_parallel, .. } = topology
&& !(1..=256).contains(max_parallel)
{
return Err("max_parallel must be between 1 and 256".to_string());
}
if matches!(topology, crate::model::CommandTopology::Pipeline { .. })
&& spec.io == crate::model::IoMode::Pty
{
return Err("pipelines do not support PTY mode".to_string());
}
}
ExecutionProgram::Javascript { .. } => {
return Err("JavaScript executions must use the Code Mode runtime".to_string());
}
ExecutionProgram::Poll {
target,
schedule,
condition,
max_iterations,
duration_ms,
iteration_timeout_ms,
} => {
if spec.retry.max_attempts != 1 {
return Err("Poll parents cannot use execution retry policy".to_string());
}
if spec.io == crate::model::IoMode::Pty {
return Err("Poll targets do not support PTY mode".to_string());
}
if condition.is_none() && max_iterations.is_none() && duration_ms.is_none() {
return Err("Poll requires a condition, max_iterations, or duration_ms".to_string());
}
if max_iterations == &Some(0) {
return Err("max_iterations must be at least 1".to_string());
}
if duration_ms == &Some(0) {
return Err("duration_ms must be at least 1".to_string());
}
if iteration_timeout_ms == &Some(0) {
return Err("iteration_timeout_ms must be at least 1".to_string());
}
match target {
PollTarget::Command { argv, shell } => {
if argv.is_empty() {
return Err("Poll command argv must contain an executable".to_string());
}
if *shell && argv.len() != 1 {
return Err("Poll shell mode accepts one zsh command string".to_string());
}
}
PollTarget::ApocCall { method, arguments } => {
if method.trim().is_empty() {
return Err("Poll apoc method cannot be empty".to_string());
}
if !arguments.is_object() {
return Err("Poll apoc arguments must be a JSON object".to_string());
}
}
}
match schedule {
PollSchedule::FixedDelay { interval_ms }
| PollSchedule::FixedRate { interval_ms }
if *interval_ms == 0 =>
{
return Err("Poll interval_ms must be at least 1".to_string());
}
PollSchedule::Backoff {
initial_interval_ms,
multiplier,
max_interval_ms,
} => {
if *initial_interval_ms == 0 {
return Err("Poll initial_interval_ms must be at least 1".to_string());
}
if !multiplier.is_finite() || *multiplier < 1.0 {
return Err("Poll multiplier must be finite and at least 1.0".to_string());
}
if max_interval_ms < initial_interval_ms {
return Err(
"Poll max_interval_ms cannot be less than initial_interval_ms"
.to_string(),
);
}
}
_ => {}
}
if let Some(condition) = condition {
match condition {
PollCondition::ExitCode { codes } if codes.is_empty() => {
return Err(
"Poll exit-code condition requires at least one code".to_string()
);
}
PollCondition::StdoutRegex { pattern }
| PollCondition::StderrRegex { pattern } => {
regex::Regex::new(pattern)
.map_err(|error| format!("Invalid poll regex: {error}"))?;
}
PollCondition::Javascript { expression } if expression.trim().is_empty() => {
return Err("Poll JavaScript expression cannot be empty".to_string());
}
_ => {}
}
}
}
}
Ok(())
}
fn provider_execution_request(execution: &Execution) -> Result<ProviderExecutionRequest, String> {
let (provider, generation, options, cwd) = match &execution.spec.backend {
ExecutionBackendSpec::Provider {
name,
generation,
options,
} => (
name.clone(),
generation.clone(),
options.clone(),
execution.spec.workspace.cwd.clone(),
),
ExecutionBackendSpec::Isolated { isolation } => {
let workspace = match isolation.workspace {
crate::model::IsolatedWorkspaceMode::None => None,
crate::model::IsolatedWorkspaceMode::ReadWrite => execution
.spec
.workspace
.worktree
.clone()
.or_else(|| execution.spec.workspace.repository.clone())
.or_else(|| execution.spec.workspace.cwd.clone()),
};
let mut options = serde_json::Map::new();
options.insert(
"image_ref".to_string(),
serde_json::Value::String(isolation.image.clone()),
);
if std::path::Path::new(&isolation.image).is_absolute() {
options.insert(
"vm_bundle".to_string(),
serde_json::Value::String(isolation.image.clone()),
);
}
if let Some(workspace) = workspace {
options.insert(
"workspace".to_string(),
serde_json::Value::String(workspace),
);
}
if let Some(memory_mb) = isolation.memory_mb {
options.insert("memory_mb".to_string(), memory_mb.into());
}
if let Some(cpu_count) = isolation.cpu_count {
options.insert("cpu_count".to_string(), cpu_count.into());
}
if let Some(pids_max) = isolation.pids_max {
options.insert("pids_max".to_string(), pids_max.into());
}
options.insert(
"network".to_string(),
serde_json::Value::String(
match isolation.network {
crate::model::IsolatedNetworkPolicy::None => "none",
crate::model::IsolatedNetworkPolicy::Nat => "nat",
}
.to_string(),
),
);
options.insert(
"exports".to_string(),
serde_json::to_value(&isolation.exports).map_err(|error| error.to_string())?,
);
let cwd = isolation.guest_cwd.clone().or_else(|| {
(isolation.workspace == crate::model::IsolatedWorkspaceMode::ReadWrite)
.then(|| "/workspace".to_string())
});
(
BUILTIN_MACOS_PROVIDER_NAME.to_string(),
BUILTIN_MACOS_PROVIDER_GENERATION.to_string(),
serde_json::Value::Object(options),
cwd.or_else(|| Some("/".to_string())),
)
}
ExecutionBackendSpec::Native => {
return Err("Execution does not declare an external backend".to_string());
}
};
let ExecutionProgram::Command { argv, shell } = &execution.spec.program else {
return Err("External execution requires one command program".to_string());
};
Ok(ProviderExecutionRequest {
execution_id: execution.id.clone(),
provider,
generation,
options,
argv: argv.clone(),
shell: *shell,
cwd,
env: execution.spec.env.clone(),
secrets: execution.spec.secrets.clone(),
io: execution.spec.io,
timeout_ms: execution.spec.budget.wall_time_ms,
artifact_bytes: execution.spec.budget.artifact_bytes,
retry: execution.spec.retry.clone(),
})
}
fn require_provider_started(execution: &Execution) -> Result<(), String> {
if execution.attempts.is_empty() {
Err("Provider emitted output or a terminal event before started".to_string())
} else {
Ok(())
}
}
fn provider_result(execution: &mut Execution) -> &mut serde_json::Map<String, serde_json::Value> {
if !matches!(execution.result, Some(serde_json::Value::Object(_))) {
execution.result = Some(serde_json::json!({}));
}
execution
.result
.as_mut()
.and_then(serde_json::Value::as_object_mut)
.expect("provider result was normalized to an object")
}
fn execution_fingerprint(spec: &ExecutionSpec, session_id: Option<&str>) -> Result<String, String> {
let mut spec = spec.clone();
let profile = spec.profile_selection.take().map(|selection| {
for name in selection.variables {
spec.env.remove(&name);
spec.secrets.remove(&name);
}
selection.name
});
spec.idempotency_key = None;
spec.workspace.repository = None;
spec.workspace.worktree = None;
spec.workspace.commit = None;
spec.workspace.branch = None;
spec.workspace.dirty_digest = None;
let value = serde_json::to_vec(&serde_json::json!({
"spec": spec,
"profile": profile,
"session_id": session_id
}))
.map_err(|error| error.to_string())?;
Ok(format!("{:x}", Sha256::digest(value)))
}
fn execution_control_fingerprint(
operation: &str,
material: &impl Serialize,
) -> Result<String, String> {
let encoded = serde_json::to_vec(&(operation, material)).map_err(|error| error.to_string())?;
Ok(format!("sha256:{:x}", Sha256::digest(encoded)))
}
fn execution_control_replay_value<T: DeserializeOwned>(
idempotency_key: &str,
operation: &str,
fingerprint: &str,
replay: ExecutionControlReplay,
) -> Result<T, String> {
if replay.operation != operation || replay.fingerprint != fingerprint {
return Err(format!(
"EXECUTION_IDEMPOTENCY_CONFLICT: key \"{idempotency_key}\" was already used for a different request"
));
}
let result = replay.result.ok_or_else(|| {
format!(
"EXECUTION_IDEMPOTENCY_INCOMPLETE: key \"{idempotency_key}\" has durable intent without a terminal result"
)
})?;
serde_json::from_value(result).map_err(|error| error.to_string())
}
fn parse_execution_cursor(value: &str) -> Result<(u64, String), String> {
let (updated_at, id) = value
.split_once(':')
.ok_or_else(|| "Invalid execution cursor".to_string())?;
let updated_at = updated_at
.parse::<u64>()
.map_err(|_| "Invalid execution cursor".to_string())?;
if id.is_empty() {
return Err("Invalid execution cursor".to_string());
}
Ok((updated_at, id.to_string()))
}
fn execution_summary(execution: &Execution) -> ExecutionSummary {
let (name, name_truncated) = bounded_text(&execution.spec.name, 256);
let (cwd, cwd_truncated) = execution
.spec
.workspace
.cwd
.as_deref()
.map(|cwd| {
let (cwd, truncated) = bounded_text(cwd, 4096);
(Some(cwd), truncated)
})
.unwrap_or((None, false));
let program = match &execution.spec.program {
ExecutionProgram::Command { argv, .. } => ExecutionProgramSummary {
kind: "command".to_string(),
executable: argv.first().map(|value| bounded_text(value, 1024).0),
argument_count: argv.len().saturating_sub(1),
command_count: 1,
},
ExecutionProgram::Commands { commands, topology } => ExecutionProgramSummary {
kind: match topology {
CommandTopology::Parallel { .. } => "batch",
CommandTopology::Pipeline { .. } => "pipeline",
}
.to_string(),
executable: None,
argument_count: 0,
command_count: commands.len(),
},
ExecutionProgram::Javascript { .. } => ExecutionProgramSummary {
kind: "javascript".to_string(),
executable: None,
argument_count: 0,
command_count: 0,
},
ExecutionProgram::Poll { target, .. } => ExecutionProgramSummary {
kind: "poll".to_string(),
executable: match target {
PollTarget::Command { argv, .. } => {
argv.first().map(|value| bounded_text(value, 1024).0)
}
PollTarget::ApocCall { method, .. } => Some(bounded_text(method, 1024).0),
},
argument_count: match target {
PollTarget::Command { argv, .. } => argv.len().saturating_sub(1),
PollTarget::ApocCall { arguments, .. } => {
arguments.as_object().map_or(0, serde_json::Map::len)
}
},
command_count: 0,
},
};
let attempt = execution.attempts.last();
let exit_code = execution
.result
.as_ref()
.and_then(|result| result.get("exit_code"))
.and_then(serde_json::Value::as_i64)
.and_then(|value| i32::try_from(value).ok())
.or_else(|| attempt.and_then(|attempt| attempt.exit_code));
ExecutionSummary {
id: execution.id.clone(),
parent_id: execution.parent_id.clone(),
session_id: execution.session_id.clone(),
idempotency_key: execution.spec.idempotency_key.clone(),
name,
name_truncated,
status: execution.status,
program,
cwd,
cwd_truncated,
attempt_count: execution.attempts.len(),
exit_code,
error_code: execution.recorded_error_code().map(ToString::to_string),
created_at: execution.created_at,
updated_at: execution.updated_at,
}
}
fn bounded_text(value: &str, bytes: usize) -> (String, bool) {
if value.len() <= bytes {
return (value.to_string(), false);
}
let mut end = bytes;
while !value.is_char_boundary(end) {
end -= 1;
}
(value[..end].to_string(), true)
}
fn run_spec(
store: &Store,
spec: &ExecutionSpec,
controller: &ProcessController,
) -> Result<RunSpec, String> {
let ExecutionProgram::Command { argv, shell } = &spec.program else {
return Err("JavaScript executions must use the Code Mode runtime".to_string());
};
if let crate::model::ExecutionBackendSpec::Provider {
name, generation, ..
} = &spec.backend
{
return Err(format!(
"EXECUTION_BACKEND_UNAVAILABLE: provider {name}@{generation}"
));
}
if matches!(
spec.backend,
crate::model::ExecutionBackendSpec::Isolated { .. }
) {
return Err("EXECUTION_BACKEND_UNAVAILABLE: built-in isolated backend".to_string());
}
let env = spec.env.clone();
let mut resolved_env = std::collections::BTreeMap::new();
for (name, secret) in &spec.secrets {
if secret.provider != "env" {
return Err(format!("SECRET_PROVIDER_UNAVAILABLE: {name}"));
}
let value =
std::env::var(&secret.reference).map_err(|_| format!("SECRET_NOT_FOUND: {name}"))?;
resolved_env.insert(name.clone(), value);
}
Ok(RunSpec {
name: spec.name.clone(),
purpose: spec.purpose.clone(),
labels: spec.labels.clone(),
requested_by: Some(controller.clone()),
argv: argv.clone(),
cwd: spec.workspace.cwd.clone(),
env,
resolved_env,
suppressed_env: store.controlled_profile_keys()?,
shell: *shell,
io: spec.io,
lifetime: match spec.owner_kind {
LeaseOwnerKind::Execution => RunLifetime::Execution,
LeaseOwnerKind::Session | LeaseOwnerKind::Persistent => RunLifetime::Persistent,
},
restart: if spec.retry.max_attempts > 1 {
RestartPolicy::OnFailure
} else {
RestartPolicy::Never
},
timeout_ms: spec.budget.wall_time_ms,
replicas: 1,
max_attempts: spec.retry.max_attempts,
retry_backoff_ms: spec.retry.backoff_ms,
retry_max_backoff_ms: spec.retry.max_backoff_ms,
})
}
fn uses_backend_gateway(backend: &ExecutionBackendSpec) -> bool {
matches!(
backend,
ExecutionBackendSpec::Provider { .. } | ExecutionBackendSpec::Isolated { .. }
)
}
fn backend_gateway_identity(backend: &ExecutionBackendSpec) -> Option<(&str, &str)> {
match backend {
ExecutionBackendSpec::Provider {
name, generation, ..
} => Some((name, generation)),
ExecutionBackendSpec::Isolated { .. } => Some((
BUILTIN_MACOS_PROVIDER_NAME,
BUILTIN_MACOS_PROVIDER_GENERATION,
)),
ExecutionBackendSpec::Native => None,
}
}
fn provider_execution_control(execution: &Execution) -> Result<ProviderExecutionControl, String> {
let (provider, generation) = backend_gateway_identity(&execution.spec.backend)
.ok_or_else(|| format!("Execution \"{}\" is not provider-owned", execution.id))?;
let provider_execution_id = execution
.result
.as_ref()
.and_then(|result| result.get("provider_execution_id"))
.and_then(serde_json::Value::as_str)
.ok_or_else(|| {
format!(
"Execution \"{}\" has not received its provider execution identity",
execution.id
)
})?;
Ok(ProviderExecutionControl {
execution_id: execution.id.clone(),
provider: provider.to_string(),
generation: generation.to_string(),
provider_execution_id: provider_execution_id.to_string(),
})
}
fn backend_kind(backend: &ExecutionBackendSpec) -> &'static str {
match backend {
ExecutionBackendSpec::Native => "native",
ExecutionBackendSpec::Isolated { .. } => "isolated",
ExecutionBackendSpec::Provider { .. } => "provider",
}
}
pub(crate) fn enrich_workspace(workspace: &mut WorkspaceContext) {
let Some(cwd) = workspace.cwd.as_deref() else {
return;
};
if workspace.repository.is_none() {
workspace.repository = git(cwd, &["rev-parse", "--show-toplevel"]);
}
if workspace.worktree.is_none() {
workspace.worktree = git(cwd, &["rev-parse", "--show-toplevel"]);
}
if workspace.commit.is_none() {
workspace.commit = git(cwd, &["rev-parse", "HEAD"]);
}
if workspace.branch.is_none() {
workspace.branch = git(cwd, &["branch", "--show-current"]);
}
if workspace.dirty_digest.is_none()
&& let Some(status) = git_bytes(cwd, &["status", "--porcelain=v1", "-z"])
&& !status.is_empty()
{
workspace.dirty_digest = Some(format!("{:x}", Sha256::digest(status)));
}
}
fn git(cwd: &str, args: &[&str]) -> Option<String> {
String::from_utf8(git_bytes(cwd, args)?)
.ok()
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
}
fn git_bytes(cwd: &str, args: &[&str]) -> Option<Vec<u8>> {
let output = Command::new("git")
.args(args)
.current_dir(cwd)
.output()
.ok()?;
output.status.success().then_some(output.stdout)
}
fn execution_status(status: &str) -> ExecutionStatus {
match status {
"starting" => ExecutionStatus::Queued,
"running" => ExecutionStatus::Running,
"completed" => ExecutionStatus::Completed,
"stopped" | "timed_out" => ExecutionStatus::Canceled,
value if value.starts_with("context_record_failed:") => ExecutionStatus::Interrupted,
_ => ExecutionStatus::Failed,
}
}
fn status_name(status: ExecutionStatus) -> &'static str {
match status {
ExecutionStatus::Queued => "queued",
ExecutionStatus::Running => "running",
ExecutionStatus::Stalled => "stalled",
ExecutionStatus::Interrupted => "interrupted",
ExecutionStatus::Completed => "completed",
ExecutionStatus::Failed => "failed",
ExecutionStatus::Canceled => "canceled",
}
}
fn terminal(status: ExecutionStatus) -> bool {
matches!(
status,
ExecutionStatus::Completed | ExecutionStatus::Failed | ExecutionStatus::Canceled
)
}
fn parse_telemetry_range(value: &str) -> Result<(u64, u64), String> {
let (start, end) = value.split_once('-').ok_or_else(|| {
"Telemetry --lines must be an inclusive sequence range such as 40-80".to_string()
})?;
let start = start
.parse::<u64>()
.map_err(|_| format!("Invalid telemetry sequence \"{start}\""))?;
let end = end
.parse::<u64>()
.map_err(|_| format!("Invalid telemetry sequence \"{end}\""))?;
if start > end {
return Err("Telemetry --lines start must not exceed its end".to_string());
}
Ok((start, end))
}
fn validate_telemetry_key(value: &str) -> Result<(), String> {
if value.is_empty() || value.len() > 256 {
return Err("Telemetry idempotency keys must contain between 1 and 256 bytes".to_string());
}
Ok(())
}
fn cancellation_requested(execution: &Execution) -> bool {
execution.status == ExecutionStatus::Canceled
|| execution
.result
.as_ref()
.and_then(|result| result.get("cancel_requested"))
.and_then(serde_json::Value::as_bool)
.unwrap_or(false)
}
fn lease_id(execution_id: &str) -> String {
format!("execution/{execution_id}")
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use tempfile::tempdir;
use super::*;
use crate::model::{
CommandTopology, CommandUnit, ExecutionBudget, ExecutionPass, IoMode, RetryPolicy,
SCHEMA_VERSION, SecretRef,
};
fn spec(argv: Vec<String>, budget: ExecutionBudget) -> ExecutionSpec {
ExecutionSpec {
schema_version: SCHEMA_VERSION,
idempotency_key: None,
name: "fixture".to_string(),
purpose: Some("verify execution budgets".to_string()),
labels: BTreeMap::new(),
program: ExecutionProgram::Command { argv, shell: false },
backend: crate::model::ExecutionBackendSpec::default(),
workspace: WorkspaceContext::default(),
env: BTreeMap::new(),
secrets: BTreeMap::new(),
profile_selection: None,
io: IoMode::Pipes,
owner_kind: LeaseOwnerKind::Persistent,
owner_id: None,
expires_at: None,
retry: RetryPolicy::default(),
budget,
telemetry: crate::model::TelemetryPolicy::default(),
expect: None,
}
}
#[test]
fn cancellation_notifications_ignore_unrelated_history_and_reach_descendants() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let root = Execution {
id: "root".to_string(),
parent_id: None,
session_id: None,
spec: spec(
vec!["/usr/bin/true".to_string()],
ExecutionBudget::default(),
),
status: ExecutionStatus::Running,
passes: Vec::new(),
attempts: Vec::new(),
created_at: 1,
updated_at: 1,
last_progress_at: 1,
result: None,
};
for (id, parent) in [
("root", None),
("child", Some("root")),
("grandchild", Some("child")),
] {
store
.put_execution_record(&Execution {
id: id.to_string(),
parent_id: parent.map(str::to_string),
..root.clone()
})
.unwrap();
}
let connection = rusqlite::Connection::open(store.path()).unwrap();
connection.execute(
"INSERT INTO execution_records(id,value,updated_at) VALUES ('unrelated','{\"status\":\"completed\"}',2)", []
).unwrap();
let tokens = ["root", "child", "grandchild", "unrelated", "missing"]
.map(|id| (id.to_string(), CancellationToken::new()));
manager
.poll_cancellations
.lock()
.unwrap()
.extend(tokens.iter().cloned());
manager.notify_poll_cancellations("root").unwrap();
for (id, token) in &tokens {
assert_eq!(
token.is_cancelled(),
["root", "child", "grandchild"].contains(&id.as_str()),
"{id}"
);
}
manager.notify_poll_cancellations("missing").unwrap();
assert!(tokens[4].1.is_cancelled());
assert!(!tokens[3].1.is_cancelled());
}
#[test]
fn an_ordinary_command_spec_keeps_its_idempotency_fingerprint() {
let spec = spec(
vec!["/bin/echo".to_string(), "pinned".to_string()],
ExecutionBudget::default(),
);
assert_eq!(
execution_fingerprint(&spec, None).unwrap(),
"4334bfbe70cc4c55349eed26038e0538a4ce172d2d23a57852cbd1bc0d027ca0"
);
assert_eq!(
execution_fingerprint(&spec, Some("session-1")).unwrap(),
"cd30e3e94097dab7c2ab3beaef17f72f300eb034d805df507996b264777739de"
);
}
#[test]
fn provider_backend_never_falls_through_to_native_launch() {
let directory = tempdir().unwrap();
let store = Store::open(directory.path().join("state.sqlite3")).unwrap();
let mut spec = spec(
vec!["/bin/echo".to_string(), "must-not-run".to_string()],
ExecutionBudget::default(),
);
spec.backend = crate::model::ExecutionBackendSpec::Provider {
name: "exeggutor".to_string(),
generation: "sha256:provider-v1".to_string(),
options: serde_json::json!({ "image": "local" }),
};
let error = run_spec(&store, &spec, &ProcessController::capture(false, None)).unwrap_err();
assert_eq!(
error,
"EXECUTION_BACKEND_UNAVAILABLE: provider exeggutor@sha256:provider-v1"
);
}
#[test]
fn provider_backend_rejects_command_groups_until_the_backend_can_own_every_child() {
let mut spec = spec(
vec!["/bin/echo".to_string(), "must-not-run".to_string()],
ExecutionBudget::default(),
);
spec.backend = crate::model::ExecutionBackendSpec::Provider {
name: "exeggutor".to_string(),
generation: "sha256:provider-v1".to_string(),
options: serde_json::json!({ "image": "local" }),
};
spec.program = ExecutionProgram::Commands {
commands: vec![CommandUnit {
id: "child".to_string(),
argv: vec!["/bin/echo".to_string(), "must-not-run".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
}],
topology: CommandTopology::Parallel {
max_parallel: 1,
fail_fast: false,
},
};
let error = validate_spec(&spec).unwrap_err();
assert_eq!(
error,
"Provider execution backends currently support one command program"
);
}
#[derive(Clone, Default)]
struct FixtureProvider {
requests: Arc<std::sync::Mutex<Vec<ProviderExecutionRequest>>>,
}
#[async_trait::async_trait]
impl ProviderExecutionGateway for FixtureProvider {
async fn execute(
&self,
request: ProviderExecutionRequest,
events: tokio::sync::mpsc::UnboundedSender<ProviderExecutionEvent>,
_cancellation: tokio_util::sync::CancellationToken,
) -> Result<(), String> {
self.requests.lock().unwrap().push(request);
events
.send(ProviderExecutionEvent::Started {
provider_execution_id: "sandbox-1".to_string(),
})
.unwrap();
events
.send(ProviderExecutionEvent::Stdout(
b"hello from guest\n".to_vec(),
))
.unwrap();
events
.send(ProviderExecutionEvent::Stderr(b"guest warning\n".to_vec()))
.unwrap();
events
.send(ProviderExecutionEvent::Exited { exit_code: 0 })
.unwrap();
Ok(())
}
}
#[tokio::test]
async fn provider_events_become_one_durable_apoc_execution() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let provider = FixtureProvider::default();
let requests = provider.requests.clone();
let manager = ExecutionManager::with_provider(
store.clone(),
RunManager::new(store).unwrap(),
Arc::new(provider),
);
let mut execution_spec = spec(
vec!["/bin/echo".to_string(), "hello".to_string()],
ExecutionBudget::default(),
);
execution_spec.workspace.cwd = Some("/workspace".to_string());
execution_spec
.env
.insert("MODE".to_string(), "test".to_string());
execution_spec.backend = crate::model::ExecutionBackendSpec::Provider {
name: "exeggutor".to_string(),
generation: "sha256:provider-v1".to_string(),
options: serde_json::json!({ "image": "local" }),
};
let started = manager
.start(
execution_spec,
ProcessController::capture(false, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&started.id, Some(Duration::from_secs(2)))
.await
.unwrap();
let logs = manager.logs(&started.id, 1024).unwrap();
assert_eq!(finished.status, ExecutionStatus::Completed);
assert_eq!(finished.attempts.len(), 1);
assert_eq!(finished.attempts[0].exit_code, Some(0));
assert_eq!(logs.stdout, "hello from guest\n");
assert_eq!(logs.stderr, "guest warning\n");
assert_eq!(
finished
.result
.as_ref()
.and_then(|result| result.get("provider_execution_id"))
.and_then(serde_json::Value::as_str),
Some("sandbox-1")
);
let requests = requests.lock().unwrap();
assert_eq!(requests.len(), 1);
assert_eq!(requests[0].execution_id, started.id);
assert_eq!(requests[0].provider, "exeggutor");
assert_eq!(requests[0].generation, "sha256:provider-v1");
assert_eq!(requests[0].argv, ["/bin/echo", "hello"]);
assert_eq!(requests[0].cwd.as_deref(), Some("/workspace"));
assert_eq!(
requests[0].env.get("MODE").map(String::as_str),
Some("test")
);
assert_eq!(requests[0].options, serde_json::json!({ "image": "local" }));
}
#[tokio::test]
async fn isolated_backend_uses_the_gateway_without_provider_registration() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let provider = FixtureProvider::default();
let requests = provider.requests.clone();
let manager = ExecutionManager::with_provider(
store.clone(),
RunManager::new(store).unwrap(),
Arc::new(provider),
);
let mut execution_spec = spec(
vec!["/bin/echo".to_string(), "isolated".to_string()],
ExecutionBudget::default(),
);
execution_spec.workspace.cwd = Some("/host/project".to_string());
execution_spec.backend = crate::model::ExecutionBackendSpec::Isolated {
isolation: crate::model::IsolatedExecutionSpec {
image: "local".to_string(),
workspace: crate::model::IsolatedWorkspaceMode::ReadWrite,
guest_cwd: Some("/workspace/src".to_string()),
memory_mb: Some(2048),
cpu_count: Some(4),
pids_max: Some(256),
network: crate::model::IsolatedNetworkPolicy::None,
exports: Vec::new(),
},
};
let started = manager
.start(
execution_spec,
ProcessController::capture(false, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&started.id, Some(Duration::from_secs(2)))
.await
.unwrap();
assert_eq!(finished.status, ExecutionStatus::Completed);
let requests = requests.lock().unwrap();
assert_eq!(
requests.len(),
1,
"isolated command ran outside the gateway"
);
assert_eq!(requests[0].provider, BUILTIN_MACOS_PROVIDER_NAME);
assert_eq!(requests[0].generation, BUILTIN_MACOS_PROVIDER_GENERATION);
assert_eq!(requests[0].cwd.as_deref(), Some("/workspace/src"));
assert_eq!(
requests[0].options,
serde_json::json!({
"image_ref": "local",
"workspace": "/host/project",
"memory_mb": 2048,
"cpu_count": 4,
"pids_max": 256,
"network": "none",
"exports": []
})
);
}
struct CancelableFixtureProvider;
#[async_trait::async_trait]
impl ProviderExecutionGateway for CancelableFixtureProvider {
async fn execute(
&self,
_request: ProviderExecutionRequest,
events: tokio::sync::mpsc::UnboundedSender<ProviderExecutionEvent>,
cancellation: CancellationToken,
) -> Result<(), String> {
events
.send(ProviderExecutionEvent::Started {
provider_execution_id: "sandbox-cancel".to_string(),
})
.unwrap();
cancellation.cancelled().await;
events.send(ProviderExecutionEvent::Canceled).unwrap();
Ok(())
}
}
#[tokio::test]
async fn canceling_external_execution_waits_for_backend_terminalization() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::with_provider(
store.clone(),
RunManager::new(store).unwrap(),
Arc::new(CancelableFixtureProvider),
);
let cases = [
(
ExecutionBackendSpec::Provider {
name: "exeggutor".to_string(),
generation: "sha256:provider-v1".to_string(),
options: serde_json::json!({ "image": "local" }),
},
"PROVIDER_EXECUTION_CANCELED",
),
(
ExecutionBackendSpec::Isolated {
isolation: crate::model::IsolatedExecutionSpec {
image: "local".to_string(),
workspace: crate::model::IsolatedWorkspaceMode::None,
guest_cwd: None,
memory_mb: None,
cpu_count: None,
pids_max: None,
network: crate::model::IsolatedNetworkPolicy::None,
exports: Vec::new(),
},
},
"EXECUTION_BACKEND_CANCELED",
),
];
for (backend, error_code) in cases {
let mut execution_spec = spec(
vec!["/bin/sleep".to_string(), "30".to_string()],
ExecutionBudget::default(),
);
execution_spec.backend = backend;
let controller = ProcessController::capture(false, None);
let started = manager
.start(execution_spec, controller.clone(), None, None)
.await
.unwrap();
let cancel_key = format!("test/provider-cancel/{error_code}");
let canceled = manager
.cancel(
&started.id,
Duration::from_secs(2),
&cancel_key,
&controller,
)
.await
.unwrap();
assert_eq!(canceled.status, ExecutionStatus::Canceled);
assert_eq!(canceled.attempts.len(), 1);
assert_eq!(
canceled
.result
.as_ref()
.and_then(|result| result.get("error_code"))
.and_then(serde_json::Value::as_str),
Some(error_code)
);
assert!(
manager
.store
.get_lease(&lease_id(&started.id))
.unwrap()
.unwrap()
.released
);
}
}
type ControlCalls<T> = Arc<std::sync::Mutex<Vec<T>>>;
#[derive(Default)]
struct ControlFixtureProvider {
inputs: ControlCalls<(ProviderExecutionControl, Vec<u8>)>,
signals: ControlCalls<(ProviderExecutionControl, String)>,
}
#[async_trait::async_trait]
impl ProviderExecutionGateway for ControlFixtureProvider {
async fn execute(
&self,
_request: ProviderExecutionRequest,
events: tokio::sync::mpsc::UnboundedSender<ProviderExecutionEvent>,
cancellation: CancellationToken,
) -> Result<(), String> {
events
.send(ProviderExecutionEvent::Started {
provider_execution_id: "sandbox-control".to_string(),
})
.unwrap();
cancellation.cancelled().await;
events.send(ProviderExecutionEvent::Canceled).unwrap();
Ok(())
}
async fn input(
&self,
control: ProviderExecutionControl,
data: &[u8],
) -> Result<(), String> {
self.inputs.lock().unwrap().push((control, data.to_vec()));
Ok(())
}
async fn signal(
&self,
control: ProviderExecutionControl,
signal: &str,
) -> Result<(), String> {
self.signals
.lock()
.unwrap()
.push((control, signal.to_string()));
Ok(())
}
}
#[tokio::test]
async fn provider_owned_execution_routes_input_and_signal_by_accepted_identity() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let provider = Arc::new(ControlFixtureProvider::default());
let manager = ExecutionManager::with_provider(
store.clone(),
RunManager::new(store).unwrap(),
provider.clone(),
);
let mut execution_spec = spec(vec!["/bin/cat".to_string()], ExecutionBudget::default());
execution_spec.backend = ExecutionBackendSpec::Isolated {
isolation: crate::model::IsolatedExecutionSpec {
image: "local".to_string(),
workspace: crate::model::IsolatedWorkspaceMode::None,
guest_cwd: None,
memory_mb: None,
cpu_count: None,
pids_max: None,
network: crate::model::IsolatedNetworkPolicy::None,
exports: Vec::new(),
},
};
let controller = ProcessController::capture(false, None);
let started = manager
.start(execution_spec, controller.clone(), None, None)
.await
.unwrap();
for _ in 0..100 {
if manager
.get(&started.id)
.unwrap()
.result
.as_ref()
.and_then(|result| result.get("provider_execution_id"))
.is_some()
{
break;
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
let receipt = manager
.input(
&started.id,
b"hello\n",
"provider-control/input-1",
&controller,
)
.await
.unwrap();
let replayed_receipt = manager
.input(
&started.id,
b"hello\n",
"provider-control/input-1",
&controller,
)
.await
.unwrap();
let input_conflict = manager
.input(
&started.id,
b"changed\n",
"provider-control/input-1",
&controller,
)
.await
.unwrap_err();
let signaled = manager
.signal(
&started.id,
"TERM",
"provider-control/signal-1",
&controller,
)
.await
.unwrap();
let replayed_signal = manager
.signal(
&started.id,
"TERM",
"provider-control/signal-1",
&controller,
)
.await
.unwrap();
assert_eq!(receipt.id, started.id);
assert_eq!(receipt.recipients, 1);
assert_eq!(receipt.bytes, 6);
assert_eq!(replayed_receipt.id, receipt.id);
assert_eq!(replayed_receipt.recipients, receipt.recipients);
assert_eq!(replayed_receipt.bytes, receipt.bytes);
assert!(
input_conflict.contains("EXECUTION_IDEMPOTENCY_CONFLICT"),
"{input_conflict}"
);
assert_eq!(signaled.status, ExecutionStatus::Running);
assert_eq!(replayed_signal.id, signaled.id);
assert_eq!(replayed_signal.status, signaled.status);
{
let inputs = provider.inputs.lock().unwrap();
assert_eq!(inputs.len(), 1);
assert_eq!(inputs[0].0.execution_id, started.id);
assert_eq!(inputs[0].0.provider_execution_id, "sandbox-control");
assert_eq!(inputs[0].1, b"hello\n");
}
{
let signals = provider.signals.lock().unwrap();
assert_eq!(signals.len(), 1);
assert_eq!(signals[0].0.provider, BUILTIN_MACOS_PROVIDER_NAME);
assert_eq!(signals[0].0.provider_execution_id, "sandbox-control");
assert_eq!(signals[0].1, "TERM");
}
manager
.cancel(
&started.id,
Duration::from_secs(2),
"test/provider-control-cancel",
&controller,
)
.await
.unwrap();
}
#[derive(Default)]
struct DelayedControlFixtureProvider {
release: Arc<tokio::sync::Notify>,
inputs: Arc<std::sync::atomic::AtomicUsize>,
}
#[async_trait::async_trait]
impl ProviderExecutionGateway for DelayedControlFixtureProvider {
async fn execute(
&self,
_request: ProviderExecutionRequest,
events: tokio::sync::mpsc::UnboundedSender<ProviderExecutionEvent>,
cancellation: CancellationToken,
) -> Result<(), String> {
self.release.notified().await;
events
.send(ProviderExecutionEvent::Started {
provider_execution_id: "sandbox-delayed-control".to_string(),
})
.unwrap();
cancellation.cancelled().await;
events.send(ProviderExecutionEvent::Canceled).unwrap();
Ok(())
}
async fn input(
&self,
_control: ProviderExecutionControl,
_data: &[u8],
) -> Result<(), String> {
self.inputs
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Ok(())
}
}
#[tokio::test]
async fn control_rejected_before_provider_acceptance_does_not_claim_the_key() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let provider = Arc::new(DelayedControlFixtureProvider::default());
let manager = ExecutionManager::with_provider(
store.clone(),
RunManager::new(store.clone()).unwrap(),
provider.clone(),
);
let mut execution_spec = spec(vec!["/bin/cat".to_string()], ExecutionBudget::default());
execution_spec.backend = ExecutionBackendSpec::Isolated {
isolation: crate::model::IsolatedExecutionSpec {
image: "local".to_string(),
workspace: crate::model::IsolatedWorkspaceMode::None,
guest_cwd: None,
memory_mb: None,
cpu_count: None,
pids_max: None,
network: crate::model::IsolatedNetworkPolicy::None,
exports: Vec::new(),
},
};
let controller = ProcessController::capture(false, None);
let started = manager
.start(execution_spec, controller.clone(), None, None)
.await
.unwrap();
let key = "provider-control/delayed-input-1";
let early = manager
.input(&started.id, b"hello\n", key, &controller)
.await
.unwrap_err();
assert!(early.contains("has not received its provider execution identity"));
assert!(
store
.get_receipt::<ExecutionControlReplay>(&format!("execution-control:{key}"))
.unwrap()
.is_none()
);
provider.release.notify_one();
for _ in 0..100 {
if manager
.get(&started.id)
.unwrap()
.result
.as_ref()
.and_then(|result| result.get("provider_execution_id"))
.is_some()
{
break;
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
manager
.input(&started.id, b"hello\n", key, &controller)
.await
.unwrap();
assert_eq!(provider.inputs.load(std::sync::atomic::Ordering::SeqCst), 1);
manager
.cancel(
&started.id,
Duration::from_secs(2),
"test/delayed-provider-control-cancel",
&controller,
)
.await
.unwrap();
}
#[derive(Default)]
struct UnknownInputFixtureProvider {
calls: Arc<std::sync::atomic::AtomicUsize>,
}
#[async_trait::async_trait]
impl ProviderExecutionGateway for UnknownInputFixtureProvider {
async fn execute(
&self,
_request: ProviderExecutionRequest,
events: tokio::sync::mpsc::UnboundedSender<ProviderExecutionEvent>,
cancellation: CancellationToken,
) -> Result<(), String> {
events
.send(ProviderExecutionEvent::Started {
provider_execution_id: "sandbox-unknown-input".to_string(),
})
.unwrap();
cancellation.cancelled().await;
events.send(ProviderExecutionEvent::Canceled).unwrap();
Ok(())
}
async fn input(
&self,
_control: ProviderExecutionControl,
_data: &[u8],
) -> Result<(), String> {
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Err("transport disconnected after dispatch".to_string())
}
}
#[tokio::test]
async fn ambiguous_execution_input_is_not_dispatched_again() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let provider = Arc::new(UnknownInputFixtureProvider::default());
let manager = ExecutionManager::with_provider(
store.clone(),
RunManager::new(store).unwrap(),
provider.clone(),
);
let mut execution_spec = spec(vec!["/bin/cat".to_string()], ExecutionBudget::default());
execution_spec.backend = ExecutionBackendSpec::Isolated {
isolation: crate::model::IsolatedExecutionSpec {
image: "local".to_string(),
workspace: crate::model::IsolatedWorkspaceMode::None,
guest_cwd: None,
memory_mb: None,
cpu_count: None,
pids_max: None,
network: crate::model::IsolatedNetworkPolicy::None,
exports: Vec::new(),
},
};
let controller = ProcessController::capture(false, None);
let started = manager
.start(execution_spec, controller.clone(), None, None)
.await
.unwrap();
for _ in 0..100 {
if manager
.get(&started.id)
.unwrap()
.result
.as_ref()
.and_then(|result| result.get("provider_execution_id"))
.is_some()
{
break;
}
tokio::time::sleep(Duration::from_millis(5)).await;
}
let first = manager
.input(
&started.id,
b"hello\n",
"provider-control/unknown-input-1",
&controller,
)
.await
.unwrap_err();
let replay = manager
.input(
&started.id,
b"hello\n",
"provider-control/unknown-input-1",
&controller,
)
.await
.unwrap_err();
assert!(first.contains("disconnected after dispatch"), "{first}");
assert!(
replay.contains("EXECUTION_IDEMPOTENCY_INCOMPLETE"),
"{replay}"
);
assert_eq!(provider.calls.load(std::sync::atomic::Ordering::SeqCst), 1);
manager
.cancel(
&started.id,
Duration::from_secs(2),
"test/unknown-provider-control-cancel",
&controller,
)
.await
.unwrap();
}
fn reject_artifact_writes(store: &Store) {
rusqlite::Connection::open(store.path())
.unwrap()
.execute_batch(
"CREATE TRIGGER reject_artifact_writes
BEFORE INSERT ON artifacts
BEGIN
SELECT RAISE(FAIL, 'forced artifact failure');
END;",
)
.unwrap();
}
fn put_execution_lease(store: &Store, execution: &Execution, released: bool) {
let now = now_ms();
store
.put_lease(&Lease {
id: lease_id(&execution.id),
owner_kind: LeaseOwnerKind::Execution,
owner_id: execution.id.clone(),
resources: Vec::new(),
created_at: now,
renewed_at: now,
expires_at: None,
released,
})
.unwrap();
if released {
store.release_lease(&lease_id(&execution.id)).unwrap();
}
}
#[test]
fn compact_summary_uses_latest_pass_error_without_an_attempt() {
let execution = Execution {
id: "code/timeout".to_string(),
parent_id: Some("code/outer".to_string()),
session_id: None,
spec: spec(vec!["code".to_string()], ExecutionBudget::default()),
status: ExecutionStatus::Failed,
passes: vec![ExecutionPass {
id: "code/timeout/pass/1".to_string(),
number: 1,
started_at: 1,
completed_at: Some(2),
status: ExecutionStatus::Failed,
error_code: Some("CODE_EXECUTION_TIMEOUT".to_string()),
error: Some("maximum uninterrupted JavaScript interval exceeded".to_string()),
}],
attempts: Vec::new(),
created_at: 1,
updated_at: 2,
last_progress_at: 2,
result: None,
};
assert_eq!(
execution_summary(&execution).error_code.as_deref(),
Some("CODE_EXECUTION_TIMEOUT")
);
}
#[test]
fn validates_every_execution_program_boundary() {
let valid = spec(
vec!["/usr/bin/true".to_string()],
ExecutionBudget::default(),
);
validate_spec(&valid).unwrap();
let mut candidate = valid.clone();
candidate.schema_version = SCHEMA_VERSION + 1;
assert!(validate_spec(&candidate).is_err());
candidate = valid.clone();
candidate.retry.max_attempts = 0;
assert!(validate_spec(&candidate).is_err());
candidate = valid.clone();
candidate.retry.backoff_ms = 2;
candidate.retry.max_backoff_ms = 1;
assert!(validate_spec(&candidate).is_err());
candidate = valid.clone();
candidate.budget.child_processes = Some(0);
assert!(validate_spec(&candidate).is_err());
candidate = valid.clone();
candidate.budget.memory_bytes = Some(0);
assert!(validate_spec(&candidate).is_err());
candidate = valid.clone();
candidate.budget.cpu_percent = Some(0.0);
assert!(validate_spec(&candidate).is_err());
candidate = valid.clone();
candidate.budget.cpu_time_ms = Some(0);
assert!(validate_spec(&candidate).is_err());
candidate = valid.clone();
candidate.budget.memory_bytes = Some(1024);
candidate.backend = ExecutionBackendSpec::Isolated {
isolation: crate::model::IsolatedExecutionSpec {
image: "local".to_string(),
workspace: crate::model::IsolatedWorkspaceMode::None,
guest_cwd: None,
memory_mb: None,
cpu_count: None,
pids_max: None,
network: crate::model::IsolatedNetworkPolicy::None,
exports: Vec::new(),
},
};
assert!(validate_spec(&candidate).is_err());
candidate = valid.clone();
candidate.idempotency_key = Some(String::new());
assert!(validate_spec(&candidate).is_err());
candidate.idempotency_key = Some("x".repeat(257));
assert!(validate_spec(&candidate).is_err());
candidate = valid.clone();
candidate.program = ExecutionProgram::Command {
argv: Vec::new(),
shell: false,
};
assert!(validate_spec(&candidate).is_err());
candidate.program = ExecutionProgram::Command {
argv: vec!["echo one".to_string(), "extra".to_string()],
shell: true,
};
assert!(validate_spec(&candidate).is_err());
candidate.program = ExecutionProgram::Javascript {
source: "return 1;".to_string(),
input: serde_json::Value::Null,
};
assert!(validate_spec(&candidate).is_err());
let command = CommandUnit {
id: "first".to_string(),
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
};
candidate = valid.clone();
candidate.program = ExecutionProgram::Commands {
commands: vec![command.clone()],
topology: CommandTopology::Parallel {
max_parallel: 1,
fail_fast: false,
},
};
validate_spec(&candidate).unwrap();
candidate.retry.max_attempts = 2;
assert!(validate_spec(&candidate).is_err());
candidate = valid.clone();
candidate.program = ExecutionProgram::Commands {
commands: Vec::new(),
topology: CommandTopology::Parallel {
max_parallel: 1,
fail_fast: false,
},
};
assert!(validate_spec(&candidate).is_err());
for invalid in [
CommandUnit {
id: String::new(),
..command.clone()
},
CommandUnit {
argv: Vec::new(),
..command.clone()
},
CommandUnit {
argv: vec!["echo one".to_string(), "extra".to_string()],
shell: true,
..command.clone()
},
] {
candidate.program = ExecutionProgram::Commands {
commands: vec![invalid],
topology: CommandTopology::Parallel {
max_parallel: 1,
fail_fast: false,
},
};
assert!(validate_spec(&candidate).is_err());
}
candidate.program = ExecutionProgram::Commands {
commands: vec![command.clone(), command.clone()],
topology: CommandTopology::Parallel {
max_parallel: 1,
fail_fast: false,
},
};
assert!(validate_spec(&candidate).is_err());
for max_parallel in [0, 257] {
candidate.program = ExecutionProgram::Commands {
commands: vec![command.clone()],
topology: CommandTopology::Parallel {
max_parallel,
fail_fast: false,
},
};
assert!(validate_spec(&candidate).is_err());
}
candidate.program = ExecutionProgram::Commands {
commands: vec![command],
topology: CommandTopology::Pipeline { pipefail: true },
};
candidate.io = IoMode::Pty;
assert!(validate_spec(&candidate).is_err());
let poll = ExecutionProgram::Poll {
target: PollTarget::Command {
argv: vec!["/usr/bin/true".to_string()],
shell: false,
},
schedule: PollSchedule::FixedDelay { interval_ms: 1 },
condition: Some(PollCondition::ExitCode { codes: vec![0] }),
max_iterations: Some(1),
duration_ms: None,
iteration_timeout_ms: None,
};
candidate = valid.clone();
candidate.program = poll.clone();
validate_spec(&candidate).unwrap();
candidate.retry.max_attempts = 2;
assert!(validate_spec(&candidate).is_err());
candidate = valid.clone();
candidate.io = IoMode::Pty;
candidate.program = poll.clone();
assert!(validate_spec(&candidate).is_err());
let command_target = || PollTarget::Command {
argv: vec!["/usr/bin/true".to_string()],
shell: false,
};
let fixed_schedule = || PollSchedule::FixedDelay { interval_ms: 1 };
let exit_condition = || Some(PollCondition::ExitCode { codes: vec![0] });
let make_poll =
|target, schedule, condition, max_iterations, duration_ms, iteration_timeout_ms| {
ExecutionProgram::Poll {
target,
schedule,
condition,
max_iterations,
duration_ms,
iteration_timeout_ms,
}
};
for program in [
make_poll(command_target(), fixed_schedule(), None, None, None, None),
make_poll(
command_target(),
fixed_schedule(),
exit_condition(),
Some(0),
None,
None,
),
make_poll(
command_target(),
fixed_schedule(),
exit_condition(),
Some(1),
Some(0),
None,
),
make_poll(
command_target(),
fixed_schedule(),
exit_condition(),
Some(1),
None,
Some(0),
),
make_poll(
PollTarget::Command {
argv: Vec::new(),
shell: false,
},
fixed_schedule(),
exit_condition(),
Some(1),
None,
None,
),
make_poll(
PollTarget::Command {
argv: vec!["echo one".to_string(), "extra".to_string()],
shell: true,
},
fixed_schedule(),
exit_condition(),
Some(1),
None,
None,
),
make_poll(
PollTarget::ApocCall {
method: " ".to_string(),
arguments: serde_json::json!({}),
},
fixed_schedule(),
exit_condition(),
Some(1),
None,
None,
),
make_poll(
PollTarget::ApocCall {
method: "version".to_string(),
arguments: serde_json::Value::Null,
},
fixed_schedule(),
exit_condition(),
Some(1),
None,
None,
),
make_poll(
command_target(),
PollSchedule::FixedDelay { interval_ms: 0 },
exit_condition(),
Some(1),
None,
None,
),
make_poll(
command_target(),
PollSchedule::FixedRate { interval_ms: 0 },
exit_condition(),
Some(1),
None,
None,
),
make_poll(
command_target(),
PollSchedule::Backoff {
initial_interval_ms: 0,
multiplier: 2.0,
max_interval_ms: 10,
},
exit_condition(),
Some(1),
None,
None,
),
make_poll(
command_target(),
PollSchedule::Backoff {
initial_interval_ms: 1,
multiplier: f64::NAN,
max_interval_ms: 10,
},
exit_condition(),
Some(1),
None,
None,
),
make_poll(
command_target(),
PollSchedule::Backoff {
initial_interval_ms: 10,
multiplier: 2.0,
max_interval_ms: 9,
},
exit_condition(),
Some(1),
None,
None,
),
make_poll(
command_target(),
fixed_schedule(),
Some(PollCondition::ExitCode { codes: Vec::new() }),
Some(1),
None,
None,
),
make_poll(
command_target(),
fixed_schedule(),
Some(PollCondition::StdoutRegex {
pattern: "[".to_string(),
}),
Some(1),
None,
None,
),
make_poll(
command_target(),
fixed_schedule(),
Some(PollCondition::Javascript {
expression: " ".to_string(),
}),
Some(1),
None,
None,
),
] {
candidate = valid.clone();
candidate.program = program;
assert!(validate_spec(&candidate).is_err());
}
}
#[test]
fn identifies_the_first_exceeded_resource_budget_from_one_family_sample() {
let sample = TelemetrySample {
sequence: 0,
sampled_at: 1,
wall_time_ms: 100,
process_count: 3,
cpu_percent: 175.0,
cpu_time_ms: 800,
memory_bytes: 96 * 1024 * 1024,
virtual_memory_bytes: 0,
disk_read_bytes: 0,
disk_written_bytes: 0,
processes: Vec::new(),
};
let budget = ExecutionBudget {
memory_bytes: Some(64 * 1024 * 1024),
cpu_percent: Some(150.0),
cpu_time_ms: Some(500),
child_processes: Some(2),
..ExecutionBudget::default()
};
let violation = resource_budget_violation(&budget, &sample).unwrap();
assert_eq!(violation["error_code"], "EXECUTION_CHILD_PROCESS_BUDGET");
assert_eq!(violation["limit"], 2);
assert_eq!(violation["observed"], 3);
}
#[test]
fn reports_each_resource_budget_with_its_measured_unit() {
let sample = TelemetrySample {
sequence: 0,
sampled_at: 1,
wall_time_ms: 100,
process_count: 1,
cpu_percent: 125.0,
cpu_time_ms: 250,
memory_bytes: 32 * 1024 * 1024,
virtual_memory_bytes: 0,
disk_read_bytes: 0,
disk_written_bytes: 0,
processes: Vec::new(),
};
for (budget, code, limit, observed) in [
(
ExecutionBudget {
memory_bytes: Some(16 * 1024 * 1024),
..ExecutionBudget::default()
},
"EXECUTION_MEMORY_BUDGET",
serde_json::json!(16 * 1024 * 1024),
serde_json::json!(32 * 1024 * 1024),
),
(
ExecutionBudget {
cpu_percent: Some(100.0),
..ExecutionBudget::default()
},
"EXECUTION_CPU_PERCENT_BUDGET",
serde_json::json!(100.0),
serde_json::json!(125.0),
),
(
ExecutionBudget {
cpu_time_ms: Some(100),
..ExecutionBudget::default()
},
"EXECUTION_CPU_TIME_BUDGET",
serde_json::json!(100),
serde_json::json!(250),
),
] {
let violation = resource_budget_violation(&budget, &sample).unwrap();
assert_eq!(violation["error_code"], code);
assert_eq!(violation["limit"], limit);
assert_eq!(violation["observed"], observed);
}
}
#[test]
fn covers_execution_summary_log_budget_and_status_helpers() {
let failed = CommandOutcome::launch_failure("command".to_string(), "launch".to_string());
assert!(failed.failed());
assert_eq!(failed.status, "failed");
assert!(failed.execution_id.is_none());
assert_eq!(failed.stderr_tail.as_deref(), Some("launch"));
let passed = CommandOutcome {
id: "command".to_string(),
execution_id: Some("execution".to_string()),
status: "completed".to_string(),
exit_code: Some(0),
stderr_tail: None,
assertion_failed: false,
};
assert!(!passed.failed());
assert_eq!(
parse_execution_cursor("42:execution").unwrap(),
(42, "execution".to_string())
);
for invalid in ["", "42", "nope:id", "42:"] {
assert!(parse_execution_cursor(invalid).is_err());
}
assert_eq!(bounded_text("short", 10), ("short".to_string(), false));
let (bounded, truncated) = bounded_text("aébc", 3);
assert_eq!(bounded, "aé");
assert!(truncated);
for (name, status) in [
("starting", ExecutionStatus::Queued),
("running", ExecutionStatus::Running),
("completed", ExecutionStatus::Completed),
("stopped", ExecutionStatus::Canceled),
("timed_out", ExecutionStatus::Canceled),
(
"context_record_failed: fixture",
ExecutionStatus::Interrupted,
),
("failed", ExecutionStatus::Failed),
] {
assert_eq!(execution_status(name), status);
assert_eq!(
status_name(status),
match status {
ExecutionStatus::Queued => "queued",
ExecutionStatus::Running => "running",
ExecutionStatus::Stalled => "stalled",
ExecutionStatus::Interrupted => "interrupted",
ExecutionStatus::Completed => "completed",
ExecutionStatus::Failed => "failed",
ExecutionStatus::Canceled => "canceled",
}
);
}
assert!(!terminal(ExecutionStatus::Queued));
assert!(!terminal(ExecutionStatus::Stalled));
assert!(terminal(ExecutionStatus::Completed));
assert!(terminal(ExecutionStatus::Failed));
assert!(terminal(ExecutionStatus::Canceled));
assert_eq!(lease_id("execution"), "execution/execution");
let mut execution = Execution {
id: "execution".to_string(),
parent_id: None,
session_id: None,
spec: spec(
vec!["/usr/bin/true".to_string()],
ExecutionBudget::default(),
),
status: ExecutionStatus::Running,
passes: Vec::new(),
attempts: Vec::new(),
created_at: 1,
updated_at: 1,
last_progress_at: 1,
result: None,
};
assert!(!cancellation_requested(&execution));
execution.result = Some(serde_json::json!({"cancel_requested": true}));
assert!(cancellation_requested(&execution));
execution.result = None;
execution.status = ExecutionStatus::Canceled;
assert!(cancellation_requested(&execution));
let directory = tempdir().unwrap();
let stdout = directory.path().join("stdout.log");
let stderr = directory.path().join("stderr.log");
std::fs::write(&stdout, "abcdefghij").unwrap();
std::fs::write(&stderr, "xy").unwrap();
let run = crate::model::ManagedRun {
id: "run".to_string(),
spec: RunSpec {
name: "run".to_string(),
purpose: None,
labels: BTreeMap::new(),
requested_by: None,
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
resolved_env: BTreeMap::new(),
suppressed_env: Vec::new(),
shell: false,
io: IoMode::Pipes,
lifetime: RunLifetime::Persistent,
restart: RestartPolicy::Never,
timeout_ms: None,
replicas: 1,
max_attempts: 1,
retry_backoff_ms: 1,
retry_max_backoff_ms: 1,
},
status: "completed".to_string(),
processes: Vec::new(),
process_history: Vec::new(),
created_at: 1,
updated_at: 2,
exit_code: Some(0),
stdout_path: Some(stdout.to_string_lossy().into_owned()),
stderr_path: Some(stderr.to_string_lossy().into_owned()),
};
assert!(enforce_log_budget(&run, 6).unwrap());
assert!(
file_size(run.stdout_path.as_deref()).unwrap()
+ file_size(run.stderr_path.as_deref()).unwrap()
<= 6
);
assert!(!enforce_log_budget(&run, 100).unwrap());
assert_eq!(file_size(None).unwrap(), 0);
assert_eq!(file_size(Some("/definitely/missing/apoc-log")).unwrap(), 0);
truncate_tail(None, 1).unwrap();
truncate_tail(Some("/definitely/missing/apoc-log"), 1).unwrap();
let (tail, truncated) = read_tail_path(&stdout, 2).unwrap();
assert!(tail.len() <= 2);
assert!(truncated);
assert_eq!(
read_tail_path(&directory.path().join("missing"), 2).unwrap(),
(String::new(), false)
);
assert!(read_tail_path(directory.path(), 2).is_err());
}
#[tokio::test]
async fn reads_and_removes_command_group_logs_without_a_managed_run() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut group_spec = spec(Vec::new(), ExecutionBudget::default());
group_spec.program = ExecutionProgram::Commands {
commands: vec![CommandUnit {
id: "child".to_string(),
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
}],
topology: CommandTopology::Parallel {
max_parallel: 1,
fail_fast: false,
},
};
let now = now_ms();
let mut group = Execution {
id: "group-logs".to_string(),
parent_id: None,
session_id: None,
spec: group_spec,
status: ExecutionStatus::Running,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
store.put_execution_record(&group).unwrap();
put_execution_lease(&store, &group, false);
let run_directory = run_directory(&store, &group.id);
std::fs::create_dir_all(&run_directory).unwrap();
std::fs::write(
run_directory.join("stdout.log"),
"zero\nmatch one\nmatch two\nomega\n",
)
.unwrap();
std::fs::write(run_directory.join("stderr.log"), "match err\nlast\n").unwrap();
let tails = manager.logs(&group.id, 10).unwrap();
assert!(tails.stdout_truncated);
assert!(tails.stderr_truncated);
assert!(tails.stdout.ends_with("two\nomega\n"));
assert!(tails.stderr.ends_with("err\nlast\n"));
let matches = manager
.log_slice(
&group.id,
&crate::run::LogSliceRequest {
pattern: Some(regex::Regex::new("match").unwrap()),
context: 0,
max_matches: 1,
start: 1,
end: usize::MAX,
},
)
.unwrap();
assert_eq!(matches.stdout_lines, 4);
assert_eq!(matches.stderr_lines, 2);
assert_eq!(matches.total_matches, Some(3));
assert!(matches.truncated);
assert_eq!(matches.lines.len(), 2);
assert!(matches.lines.iter().all(|line| line.text.contains("match")));
let range = manager
.log_slice(
&group.id,
&crate::run::LogSliceRequest {
pattern: None,
context: 0,
max_matches: 0,
start: 2,
end: 3,
},
)
.unwrap();
assert_eq!(range.total_matches, None);
assert!(!range.truncated);
assert_eq!(range.stdout_lines, 4);
assert_eq!(range.stderr_lines, 2);
assert_eq!(
range
.lines
.iter()
.map(|line| (line.stream.as_str(), line.number, line.text.as_str()))
.collect::<Vec<_>>(),
vec![
("stdout", 2, "match one"),
("stdout", 3, "match two"),
("stderr", 2, "last"),
]
);
let non_group = Execution {
id: "no-managed-run".to_string(),
status: ExecutionStatus::Completed,
spec: spec(
vec!["/usr/bin/true".to_string()],
ExecutionBudget::default(),
),
..group.clone()
};
store.put_execution_record(&non_group).unwrap();
assert!(
manager
.logs(&non_group.id, 10)
.unwrap_err()
.contains("has no process logs")
);
assert!(
manager
.log_slice(
&non_group.id,
&crate::run::LogSliceRequest {
pattern: None,
context: 0,
max_matches: 0,
start: 1,
end: 1,
},
)
.unwrap_err()
.contains("has no process logs")
);
assert!(
manager
.remove(&group.id)
.await
.unwrap_err()
.contains("is active")
);
group.status = ExecutionStatus::Completed;
group.updated_at = now_ms();
store.put_execution_record(&group).unwrap();
assert!(manager.remove(&group.id).await.unwrap());
assert!(store.get_execution_record(&group.id).unwrap().is_none());
assert!(
store
.get_lease(&lease_id(&group.id))
.unwrap()
.unwrap()
.released
);
}
#[test]
fn synchronizes_stalled_and_terminal_runs_without_losing_history_or_errors() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let identity = crate::model::ProcessIdentity {
boot_id: "fixture-boot".to_string(),
pid: 42,
ppid: Some(1),
pgid: Some(42),
started_at: 10,
started_at_ns: 10_500_000_000,
name: "fixture".to_string(),
executable: Some("/fixture".to_string()),
};
let execution = Execution {
id: "sync-run".to_string(),
parent_id: None,
session_id: None,
spec: spec(
vec!["/usr/bin/true".to_string()],
ExecutionBudget::default(),
),
status: ExecutionStatus::Stalled,
passes: Vec::new(),
attempts: Vec::new(),
created_at: 1,
updated_at: 1,
last_progress_at: 1,
result: None,
};
store.put_execution_record(&execution).unwrap();
put_execution_lease(&store, &execution, false);
let mut run = crate::model::ManagedRun {
id: execution.id.clone(),
spec: RunSpec {
name: "sync-run".to_string(),
purpose: Some("exercise durable synchronization".to_string()),
labels: BTreeMap::new(),
requested_by: None,
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
resolved_env: BTreeMap::new(),
suppressed_env: Vec::new(),
shell: false,
io: IoMode::Pipes,
lifetime: RunLifetime::Persistent,
restart: RestartPolicy::Never,
timeout_ms: None,
replicas: 1,
max_attempts: 1,
retry_backoff_ms: 1,
retry_max_backoff_ms: 1,
},
status: "running".to_string(),
processes: vec![identity.clone()],
process_history: vec![identity.clone()],
created_at: 1,
updated_at: 2,
exit_code: None,
stdout_path: None,
stderr_path: None,
};
let stalled = manager.sync_run(execution, &run, None).unwrap();
assert_eq!(stalled.status, ExecutionStatus::Stalled);
assert_eq!(stalled.attempts.len(), 1);
assert_eq!(stalled.attempts[0].processes, vec![identity.clone()]);
assert_eq!(stalled.attempts[0].started_at, 10_500);
assert_eq!(stalled.attempts[0].completed_at, None);
assert_eq!(stalled.attempts[0].exit_code, None);
let renewed = store.get_lease(&lease_id(&stalled.id)).unwrap().unwrap();
assert_eq!(
renewed.resources,
vec![format!("process/{}", process_context_key(&identity))]
);
assert!(!renewed.released);
let mut stalled = stalled;
stalled.result = Some(serde_json::json!({ "error_code": "FIXTURE_ERROR" }));
run.status = "failed".to_string();
run.processes.clear();
run.updated_at = 3;
run.exit_code = Some(9);
let terminal = manager.sync_run(stalled, &run, None).unwrap();
assert_eq!(terminal.status, ExecutionStatus::Failed);
assert_eq!(terminal.attempts.len(), 1);
assert_eq!(terminal.attempts[0].completed_at, Some(3));
assert_eq!(terminal.attempts[0].exit_code, Some(9));
assert_eq!(
terminal.result.as_ref().unwrap()["error_code"],
"FIXTURE_ERROR"
);
assert_eq!(terminal.result.as_ref().unwrap()["output_truncated"], false);
assert!(
store
.get_lease(&lease_id(&terminal.id))
.unwrap()
.unwrap()
.released
);
}
#[test]
fn terminal_sync_preserves_a_durable_progress_timeout_timestamp() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let execution = Execution {
id: "stale-progress-timeout-sync".to_string(),
parent_id: None,
session_id: None,
spec: spec(
vec!["/bin/sleep".to_string(), "30".to_string()],
ExecutionBudget::default(),
),
status: ExecutionStatus::Running,
passes: Vec::new(),
attempts: Vec::new(),
created_at: 1,
updated_at: 2,
last_progress_at: 1,
result: Some(serde_json::json!({
"error_code": "EXECUTION_PROGRESS_TIMEOUT",
"last_progress_at": 1,
"stalled_at": 2
})),
};
store.put_execution_record(&execution).unwrap();
put_execution_lease(&store, &execution, false);
let run = crate::model::ManagedRun {
id: execution.id.clone(),
spec: run_spec(
&store,
&execution.spec,
&ProcessController::capture(true, None),
)
.unwrap(),
status: "stopped".to_string(),
processes: Vec::new(),
process_history: Vec::new(),
created_at: 1,
updated_at: 3,
exit_code: None,
stdout_path: None,
stderr_path: None,
};
let terminal = manager.sync_run(execution, &run, None).unwrap();
assert_eq!(terminal.status, ExecutionStatus::Canceled);
assert_eq!(terminal.updated_at, 3);
assert_eq!(terminal.last_progress_at, 1);
assert_eq!(terminal.result.as_ref().unwrap()["last_progress_at"], 1);
}
#[test]
fn observation_reads_do_not_rewind_output_progress_on_a_live_run() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let execution = Execution {
id: "live-output-progress".to_string(),
parent_id: None,
session_id: None,
spec: spec(
vec!["/bin/sleep".to_string(), "30".to_string()],
ExecutionBudget {
progress_timeout_ms: Some(250),
cancel_on_stall: true,
..ExecutionBudget::default()
},
),
status: ExecutionStatus::Running,
passes: Vec::new(),
attempts: Vec::new(),
created_at: 1,
updated_at: 50,
last_progress_at: 50,
result: None,
};
store.put_execution_record(&execution).unwrap();
put_execution_lease(&store, &execution, false);
store
.put_run(&crate::model::ManagedRun {
id: execution.id.clone(),
spec: run_spec(
&store,
&execution.spec,
&ProcessController::capture(true, None),
)
.unwrap(),
status: "running".to_string(),
processes: Vec::new(),
process_history: Vec::new(),
created_at: 1,
updated_at: 1,
exit_code: None,
stdout_path: None,
stderr_path: None,
})
.unwrap();
let lease_renewed_at = store
.get_lease(&lease_id(&execution.id))
.unwrap()
.unwrap()
.renewed_at;
let observed = manager.get(&execution.id).unwrap();
let stored = store.get_execution_record(&execution.id).unwrap().unwrap();
let lease = store.get_lease(&lease_id(&execution.id)).unwrap().unwrap();
assert_eq!(observed.status, ExecutionStatus::Running);
assert_eq!(observed.last_progress_at, 50);
assert_eq!(stored.last_progress_at, 50);
assert_eq!(stored.updated_at, 50);
assert_eq!(lease.renewed_at, lease_renewed_at);
assert!(!lease.released);
}
#[test]
fn wall_timeout_keeps_progress_timeout_timestamps() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let execution = Execution {
id: "stalled-then-timed-out".to_string(),
parent_id: None,
session_id: None,
spec: spec(
vec!["/bin/sleep".to_string(), "30".to_string()],
ExecutionBudget {
wall_time_ms: Some(100),
progress_timeout_ms: Some(25),
..ExecutionBudget::default()
},
),
status: ExecutionStatus::Stalled,
passes: Vec::new(),
attempts: Vec::new(),
created_at: 1,
updated_at: 2,
last_progress_at: 1,
result: Some(serde_json::json!({
"error_code": "EXECUTION_PROGRESS_TIMEOUT",
"last_progress_at": 1,
"stalled_at": 2
})),
};
store.put_execution_record(&execution).unwrap();
put_execution_lease(&store, &execution, false);
let mut managed_spec = run_spec(
&store,
&execution.spec,
&ProcessController::capture(true, None),
)
.unwrap();
managed_spec.timeout_ms = Some(100);
let run = crate::model::ManagedRun {
id: execution.id.clone(),
spec: managed_spec,
status: "timed_out".to_string(),
processes: Vec::new(),
process_history: Vec::new(),
created_at: 1,
updated_at: 3,
exit_code: None,
stdout_path: None,
stderr_path: None,
};
let terminal = manager.sync_run(execution, &run, None).unwrap();
assert_eq!(terminal.status, ExecutionStatus::Canceled);
assert_eq!(terminal.last_progress_at, 1);
assert_eq!(
terminal.result.as_ref().unwrap()["error_code"],
"EXECUTION_TIMEOUT"
);
assert_eq!(terminal.result.as_ref().unwrap()["last_progress_at"], 1);
assert_eq!(terminal.result.as_ref().unwrap()["stalled_at"], 2);
assert_eq!(terminal.result.as_ref().unwrap()["timeout_ms"], 100);
}
#[test]
fn a_met_nonzero_child_is_reported_completed() {
let mut execution = Execution {
id: "accepted-nonzero".to_string(),
parent_id: None,
session_id: None,
spec: spec(
vec!["/usr/bin/false".to_string()],
ExecutionBudget::default(),
),
status: ExecutionStatus::Failed,
passes: Vec::new(),
attempts: Vec::new(),
created_at: 1,
updated_at: 2,
last_progress_at: 1,
result: Some(serde_json::json!({"exit_code": 1})),
};
execution.spec.expect = Some(PollCondition::ExitCode { codes: vec![1] });
execution.record_expectation(true, None);
assert_eq!(reported_child_status(&execution), "completed");
assert!(!CommandOutcome::from_execution("child".to_string(), &execution, None).failed());
assert!(
!CommandOutcome::from_execution("child".to_string(), &execution, None).assertion_failed
);
}
#[tokio::test]
async fn bounds_execution_pages_and_skips_inapplicable_recovery_records() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let long = "x".repeat(10_000);
for index in 0..30_u64 {
let mut execution_spec =
spec(vec![format!("/fixture/{long}")], ExecutionBudget::default());
execution_spec.name = format!("{index}-{long}");
execution_spec.workspace.cwd = Some(format!("/fixture/{index}/{long}"));
store
.put_execution_record(&Execution {
id: format!("page-{index:02}"),
parent_id: None,
session_id: None,
spec: execution_spec,
status: ExecutionStatus::Completed,
passes: Vec::new(),
attempts: Vec::new(),
created_at: index + 1,
updated_at: index + 1,
last_progress_at: index + 1,
result: Some(serde_json::json!({ "error": long })),
})
.unwrap();
}
for limit in [0, 101] {
assert!(manager.list_page(limit, None, None, None, None).is_err());
}
assert!(
manager
.list_page(1, Some("invalid"), None, None, None)
.is_err()
);
let page = manager.list_page(100, None, None, None, None).unwrap();
assert!(!page.executions.is_empty());
assert!(page.executions.len() < 30);
assert!(page.next_cursor.is_some());
assert!(serde_json::to_vec(&page).unwrap().len() <= 128 * 1024);
let next = manager
.list_page(100, page.next_cursor.as_deref(), None, None, None)
.unwrap();
assert!(!next.executions.is_empty());
assert!(
manager
.list_page(
5,
None,
Some(ExecutionStatus::Completed),
None,
Some("missing-parent")
)
.unwrap()
.executions
.is_empty()
);
assert_eq!(manager.list().unwrap().len(), 30);
assert!(manager.get("missing").unwrap_err().contains("not found"));
assert!(
manager
.tree("missing", &ProcessController::capture(true, None))
.unwrap_err()
.contains("not found")
);
let mut poll_spec = spec(Vec::new(), ExecutionBudget::default());
poll_spec.program = ExecutionProgram::Poll {
target: PollTarget::Command {
argv: vec!["/usr/bin/true".to_string()],
shell: false,
},
schedule: PollSchedule::FixedDelay { interval_ms: 1 },
condition: None,
max_iterations: Some(1),
duration_ms: None,
iteration_timeout_ms: None,
};
let poll = Execution {
id: "recover-poll".to_string(),
parent_id: None,
session_id: None,
spec: poll_spec,
status: ExecutionStatus::Running,
passes: Vec::new(),
attempts: Vec::new(),
created_at: 100,
updated_at: 100,
last_progress_at: 100,
result: None,
};
store.put_execution_record(&poll).unwrap();
let managed = Execution {
id: "recover-managed".to_string(),
status: ExecutionStatus::Running,
created_at: 101,
updated_at: 101,
last_progress_at: 101,
spec: spec(
vec!["/usr/bin/true".to_string()],
ExecutionBudget::default(),
),
..poll.clone()
};
store.put_execution_record(&managed).unwrap();
store
.put_run(&crate::model::ManagedRun {
id: managed.id.clone(),
spec: RunSpec {
name: "recover-managed".to_string(),
purpose: None,
labels: BTreeMap::new(),
requested_by: None,
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
resolved_env: BTreeMap::new(),
suppressed_env: Vec::new(),
shell: false,
io: IoMode::Pipes,
lifetime: RunLifetime::Persistent,
restart: RestartPolicy::Never,
timeout_ms: None,
replicas: 1,
max_attempts: 1,
retry_backoff_ms: 1,
retry_max_backoff_ms: 1,
},
status: "running".to_string(),
processes: Vec::new(),
process_history: Vec::new(),
created_at: 101,
updated_at: 101,
exit_code: None,
stdout_path: None,
stderr_path: None,
})
.unwrap();
manager.recover().unwrap();
assert_eq!(
manager.get(&poll.id).unwrap().status,
ExecutionStatus::Running
);
assert_eq!(
manager.get(&managed.id).unwrap().status,
ExecutionStatus::Running
);
}
#[test]
fn execution_summary_uses_the_shared_terminal_result_error_code() {
let execution = Execution {
id: "progress-timeout-summary".to_string(),
parent_id: None,
session_id: None,
spec: spec(vec!["/bin/sleep".to_string()], ExecutionBudget::default()),
status: ExecutionStatus::Canceled,
passes: Vec::new(),
attempts: Vec::new(),
created_at: 1,
updated_at: 2,
last_progress_at: 1,
result: Some(serde_json::json!({
"error_code": "EXECUTION_PROGRESS_TIMEOUT"
})),
};
assert_eq!(
execution_summary(&execution).error_code.as_deref(),
Some("EXECUTION_PROGRESS_TIMEOUT")
);
}
#[test]
fn a_filtered_list_includes_a_terminal_run_before_background_sync() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let controller = ProcessController::capture(true, None);
let execution = Execution {
id: "terminal-run-before-sync".to_string(),
parent_id: None,
session_id: None,
spec: spec(
vec!["/bin/sleep".to_string(), "30".to_string()],
ExecutionBudget {
wall_time_ms: Some(100),
..ExecutionBudget::default()
},
),
status: ExecutionStatus::Running,
passes: Vec::new(),
attempts: Vec::new(),
created_at: 1,
updated_at: 1,
last_progress_at: 1,
result: None,
};
let managed_spec = run_spec(&store, &execution.spec, &controller).unwrap();
store.put_execution_record(&execution).unwrap();
store
.put_run(&crate::model::ManagedRun {
id: execution.id.clone(),
spec: managed_spec,
status: "timed_out".to_string(),
processes: Vec::new(),
process_history: Vec::new(),
created_at: 1,
updated_at: 2,
exit_code: Some(128),
stdout_path: None,
stderr_path: None,
})
.unwrap();
let page = manager
.list_page(
10,
None,
Some(ExecutionStatus::Canceled),
Some("EXECUTION_TIMEOUT"),
None,
)
.unwrap();
assert_eq!(page.executions.len(), 1, "{page:#?}");
assert_eq!(page.executions[0].id, execution.id);
assert_eq!(
page.executions[0].error_code.as_deref(),
Some("EXECUTION_TIMEOUT")
);
}
#[tokio::test]
async fn runs_one_command_as_a_durable_execution_and_releases_its_lease() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let execution = manager
.start(
ExecutionSpec {
schema_version: SCHEMA_VERSION,
idempotency_key: None,
name: "echo".to_string(),
purpose: Some("verify the execution runtime".to_string()),
labels: BTreeMap::new(),
program: ExecutionProgram::Command {
argv: vec!["/bin/echo".to_string(), "apocalypse".to_string()],
shell: false,
},
backend: crate::model::ExecutionBackendSpec::default(),
workspace: WorkspaceContext {
cwd: Some(directory.path().to_string_lossy().into_owned()),
..WorkspaceContext::default()
},
env: BTreeMap::new(),
secrets: BTreeMap::new(),
profile_selection: None,
io: IoMode::Pipes,
owner_kind: LeaseOwnerKind::Persistent,
owner_id: None,
expires_at: None,
retry: RetryPolicy::default(),
budget: ExecutionBudget::default(),
telemetry: crate::model::TelemetryPolicy::default(),
expect: None,
},
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(2)))
.await
.unwrap();
assert_eq!(finished.status, ExecutionStatus::Completed);
assert_eq!(finished.attempts.len(), 1);
assert!(
store
.get_lease(&lease_id(&execution.id))
.unwrap()
.unwrap()
.released
);
assert!(
manager
.logs(&execution.id, 1024)
.unwrap()
.stdout
.contains("apocalypse")
);
}
#[tokio::test]
async fn returns_failed_execution_when_post_acceptance_spawn_fails() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let missing = directory
.path()
.join("missing")
.join("apoc-definitely-nonexistent-executable");
let execution = manager
.start(
spec(
vec![missing.to_string_lossy().into_owned()],
ExecutionBudget::default(),
),
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
assert_eq!(execution.status, ExecutionStatus::Failed);
assert_eq!(
execution.result.as_ref().unwrap()["error_code"],
"EXECUTION_START_FAILED"
);
assert_eq!(store.list_execution_records().unwrap().len(), 1);
assert!(
store
.get_lease(&lease_id(&execution.id))
.unwrap()
.unwrap()
.released
);
assert_eq!(
store
.causal_events(0, 100)
.unwrap()
.iter()
.filter(|event| event.kind == "execution.failed")
.count(),
1
);
}
#[tokio::test]
async fn returns_failed_execution_when_post_acceptance_secret_resolution_fails() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(vec!["/usr/bin/env".to_string()], ExecutionBudget::default());
execution_spec.secrets.insert(
"TEST_APOC_SECRET".to_string(),
SecretRef {
provider: "env".to_string(),
reference: "APOC_DEFINITELY_MISSING_SECRET_FOR_EXECUTION_TEST".to_string(),
version: None,
},
);
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
assert_eq!(execution.status, ExecutionStatus::Failed);
assert_eq!(
execution.result.as_ref().unwrap()["error_code"],
"EXECUTION_START_FAILED"
);
assert!(
execution.result.as_ref().unwrap()["error"]
.as_str()
.unwrap()
.contains("SECRET_NOT_FOUND")
);
assert!(
store
.get_lease(&lease_id(&execution.id))
.unwrap()
.unwrap()
.released
);
}
#[test]
fn recovery_fails_an_accepted_command_without_a_durable_run_by_backend() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let now = now_ms();
let isolated = ExecutionBackendSpec::Isolated {
isolation: crate::model::IsolatedExecutionSpec {
image: "local".to_string(),
workspace: crate::model::IsolatedWorkspaceMode::None,
guest_cwd: None,
memory_mb: None,
cpu_count: None,
pids_max: None,
network: crate::model::IsolatedNetworkPolicy::None,
exports: Vec::new(),
},
};
let cases = [
(
"accepted-native-without-run",
ExecutionBackendSpec::Native,
"EXECUTION_RECOVERY_MISSING_RUN",
),
(
"accepted-isolated-without-run",
isolated,
"EXECUTION_BACKEND_LOST",
),
];
for (id, backend, _) in &cases {
let mut execution_spec = spec(
vec!["/usr/bin/true".to_string()],
ExecutionBudget::default(),
);
execution_spec.backend = backend.clone();
let execution = Execution {
id: (*id).to_string(),
parent_id: None,
session_id: None,
spec: execution_spec,
status: ExecutionStatus::Queued,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
store.put_execution_record(&execution).unwrap();
store
.put_lease(&Lease {
id: lease_id(&execution.id),
owner_kind: LeaseOwnerKind::Execution,
owner_id: execution.id.clone(),
resources: Vec::new(),
created_at: now,
renewed_at: now,
expires_at: None,
released: false,
})
.unwrap();
}
manager.recover().unwrap();
manager.recover().unwrap();
for (id, _, error_code) in cases {
let failed = manager.get(id).unwrap();
assert_eq!(failed.status, ExecutionStatus::Failed);
assert_eq!(failed.result.unwrap()["error_code"], error_code);
assert!(store.get_lease(&lease_id(id)).unwrap().unwrap().released);
}
assert_eq!(
store
.causal_events(0, 100)
.unwrap()
.iter()
.filter(|event| event.kind == "execution.failed")
.count(),
2
);
}
#[test]
fn recovery_removes_unredacted_provider_logs_when_secret_resolution_is_unavailable() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(vec!["/usr/bin/env".to_string()], ExecutionBudget::default());
execution_spec.backend = ExecutionBackendSpec::Isolated {
isolation: crate::model::IsolatedExecutionSpec {
image: "local".to_string(),
workspace: crate::model::IsolatedWorkspaceMode::None,
guest_cwd: None,
memory_mb: None,
cpu_count: None,
pids_max: None,
network: crate::model::IsolatedNetworkPolicy::None,
exports: Vec::new(),
},
};
execution_spec.secrets.insert(
"TOKEN".to_string(),
SecretRef {
provider: "env".to_string(),
reference: "APOC_MISSING_RECOVERY_SECRET".to_string(),
version: None,
},
);
let now = now_ms();
let execution = Execution {
id: "provider-secret-recovery".to_string(),
parent_id: None,
session_id: None,
spec: execution_spec,
status: ExecutionStatus::Running,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
store.put_execution_record(&execution).unwrap();
store
.put_lease(&Lease {
id: lease_id(&execution.id),
owner_kind: LeaseOwnerKind::Execution,
owner_id: execution.id.clone(),
resources: Vec::new(),
created_at: now,
renewed_at: now,
expires_at: None,
released: false,
})
.unwrap();
let logs = run_directory(&store, &execution.id);
std::fs::create_dir_all(&logs).unwrap();
std::fs::write(logs.join("stdout.log"), "unredacted-secret").unwrap();
std::fs::write(logs.join("stderr.log"), "unredacted-secret").unwrap();
require_redaction(&logs).unwrap();
let pending = manager.logs(&execution.id, 1024).unwrap_err();
assert!(pending.contains("unavailable until secret redaction completes"));
manager.fail_missing_run(execution).unwrap();
assert!(!logs.join("stdout.log").exists());
assert!(!logs.join("stderr.log").exists());
assert!(!logs.join("redaction.required").exists());
let recovered = manager.get("provider-secret-recovery").unwrap();
assert_eq!(recovered.status, ExecutionStatus::Failed);
assert_eq!(
recovered.result.unwrap()["error_code"],
"EXECUTION_LOG_REDACTION_FAILED"
);
assert!(
store
.get_lease(&lease_id("provider-secret-recovery"))
.unwrap()
.unwrap()
.released
);
}
#[test]
fn recovery_terminalizes_an_execution_whose_managed_run_already_finished() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let now = now_ms();
let execution = Execution {
id: "finished-run-running-execution".to_string(),
parent_id: None,
session_id: None,
spec: spec(
vec!["/usr/bin/true".to_string()],
ExecutionBudget::default(),
),
status: ExecutionStatus::Running,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now.saturating_sub(10),
updated_at: now.saturating_sub(5),
last_progress_at: now.saturating_sub(5),
result: None,
};
store.put_execution_record(&execution).unwrap();
put_execution_lease(&store, &execution, false);
store
.put_run(&crate::model::ManagedRun {
id: execution.id.clone(),
spec: RunSpec {
name: "finished".to_string(),
purpose: None,
labels: BTreeMap::new(),
requested_by: None,
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
resolved_env: BTreeMap::new(),
suppressed_env: Vec::new(),
shell: false,
io: IoMode::Pipes,
lifetime: RunLifetime::Persistent,
restart: RestartPolicy::Never,
timeout_ms: None,
replicas: 1,
max_attempts: 1,
retry_backoff_ms: 1,
retry_max_backoff_ms: 1,
},
status: "completed".to_string(),
processes: Vec::new(),
process_history: Vec::new(),
created_at: now.saturating_sub(10),
updated_at: now,
exit_code: Some(0),
stdout_path: None,
stderr_path: None,
})
.unwrap();
manager.recover().unwrap();
let recovered = store.get_execution_record(&execution.id).unwrap().unwrap();
assert_eq!(recovered.status, ExecutionStatus::Completed);
assert_eq!(recovered.result.as_ref().unwrap()["exit_code"], 0);
assert!(
store
.get_lease(&lease_id(&execution.id))
.unwrap()
.unwrap()
.released
);
assert_eq!(
store
.causal_events(0, 100)
.unwrap()
.iter()
.filter(|event| event.kind == "execution.completed")
.count(),
1
);
}
#[tokio::test]
async fn recovery_reinstalls_the_terminal_monitor_for_a_live_managed_run() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let runs = RunManager::new(store.clone()).unwrap();
let manager = ExecutionManager::new(store.clone(), runs.clone());
let mut child = std::process::Command::new("/bin/sleep")
.arg("0.2")
.spawn()
.unwrap();
let identity = await_process_identity(child.id()).await;
let now = now_ms();
let execution = Execution {
id: "recovered-live-run".to_string(),
parent_id: None,
session_id: None,
spec: spec(
vec!["/bin/sleep".to_string(), "0.2".to_string()],
ExecutionBudget::default(),
),
status: ExecutionStatus::Running,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
store.put_execution_record(&execution).unwrap();
put_execution_lease(&store, &execution, false);
store
.put_run(&crate::model::ManagedRun {
id: execution.id.clone(),
spec: RunSpec {
name: "recovered-live-run".to_string(),
purpose: None,
labels: BTreeMap::new(),
requested_by: None,
argv: vec!["/bin/sleep".to_string(), "0.2".to_string()],
cwd: None,
env: BTreeMap::new(),
resolved_env: BTreeMap::new(),
suppressed_env: Vec::new(),
shell: false,
io: IoMode::Pipes,
lifetime: RunLifetime::Persistent,
restart: RestartPolicy::Never,
timeout_ms: None,
replicas: 1,
max_attempts: 1,
retry_backoff_ms: 1,
retry_max_backoff_ms: 1,
},
status: "running".to_string(),
processes: vec![identity.clone()],
process_history: vec![identity],
created_at: now,
updated_at: now,
exit_code: None,
stdout_path: None,
stderr_path: None,
})
.unwrap();
runs.recover().await.unwrap();
manager.recover().unwrap();
let reaper = tokio::task::spawn_blocking(move || child.wait().unwrap());
let recovered = tokio::time::timeout(Duration::from_secs(3), async {
loop {
let current = store.get_execution_record(&execution.id).unwrap().unwrap();
if terminal(current.status) {
break current;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("recovered execution never terminalized");
reaper.await.unwrap();
let recovered_run = store.get_run(&execution.id).unwrap().unwrap();
assert_eq!(recovered_run.status, "interrupted");
assert_eq!(recovered_run.exit_code, None);
assert_eq!(recovered.status, ExecutionStatus::Failed);
assert_eq!(
crate::model::ExecutionVerdict::of_execution(&recovered),
crate::model::ExecutionVerdict::Error
);
assert_eq!(
recovered.result.as_ref().unwrap()["exit_code"],
serde_json::Value::Null
);
assert!(
store
.get_lease(&lease_id(&execution.id))
.unwrap()
.unwrap()
.released
);
assert_eq!(
store
.causal_events(0, 100)
.unwrap()
.iter()
.filter(|event| event.kind == "execution.failed")
.count(),
1
);
}
#[test]
fn cleanup_persistence_failure_is_truthful_on_the_parent() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let now = now_ms();
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![CommandUnit {
id: "child".to_string(),
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
}],
topology: CommandTopology::Pipeline { pipefail: true },
};
let parent = Execution {
id: "cleanup-parent".to_string(),
parent_id: None,
session_id: None,
spec: execution_spec,
status: ExecutionStatus::Running,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
store.put_execution_record(&parent).unwrap();
store
.put_lease(&Lease {
id: lease_id(&parent.id),
owner_kind: LeaseOwnerKind::Execution,
owner_id: parent.id.clone(),
resources: Vec::new(),
created_at: now,
renewed_at: now,
expires_at: None,
released: false,
})
.unwrap();
manager
.fail_command_cleanup(
parent.clone(),
"child cleanup could not be persisted".to_string(),
&ProcessController::capture(true, Some(parent.id.clone())),
false,
)
.unwrap();
let failed = manager.get(&parent.id).unwrap();
assert_eq!(failed.status, ExecutionStatus::Failed);
assert_eq!(failed.result.as_ref().unwrap()["children_complete"], false);
assert_eq!(
failed.result.as_ref().unwrap()["error_code"],
"EXECUTION_CLEANUP_FAILED"
);
assert!(
store
.get_lease(&lease_id(&parent.id))
.unwrap()
.unwrap()
.released
);
}
#[tokio::test]
async fn artifact_failure_terminalizes_a_fresh_command_group() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
reject_artifact_writes(&store);
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![CommandUnit {
id: "child".to_string(),
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
}],
topology: CommandTopology::Parallel {
max_parallel: 1,
fail_fast: false,
},
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(2)))
.await
.unwrap();
let children = manager
.list()
.unwrap()
.into_iter()
.filter(|child| child.parent_id.as_deref() == Some(&execution.id))
.collect::<Vec<_>>();
assert_eq!(finished.status, ExecutionStatus::Failed);
assert_eq!(
finished.result.as_ref().unwrap()["error_code"],
"EXECUTION_CLEANUP_FAILED"
);
assert_eq!(finished.result.as_ref().unwrap()["children_complete"], true);
assert!(
finished.result.as_ref().unwrap()["error"]
.as_str()
.unwrap()
.contains("forced artifact failure")
);
assert_eq!(children.len(), 1);
assert_eq!(children[0].status, ExecutionStatus::Completed);
assert_child_leases_released(&store, &children);
assert!(
store
.get_lease(&lease_id(&execution.id))
.unwrap()
.unwrap()
.released
);
assert_eq!(
store
.causal_events_for_subject(&format!("execution/{}", execution.id), 100)
.unwrap()
.iter()
.filter(|event| event.kind == "execution.failed")
.count(),
1
);
}
#[tokio::test]
async fn command_groups_recover_child_verdicts_without_relaunching() {
let batch = CommandTopology::Parallel {
max_parallel: 1,
fail_fast: true,
};
let pipeline = |pipefail| CommandTopology::Pipeline { pipefail };
for (topology, exits, judgments, passes) in [
(pipeline(true), [0, 0], [false, true], false),
(batch, [1, 1], [true, true], true),
(pipeline(true), [1, 1], [true, true], true),
(pipeline(false), [1, 1], [true, true], true),
(pipeline(true), [0, 0], [true, false], false),
(pipeline(false), [0, 0], [false, true], false),
(pipeline(false), [0, 0], [true, false], false),
] {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager =
ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let commands = (0..2)
.map(|index| CommandUnit {
id: format!("stage-{index}"),
argv: vec!["/apoc-fixture-must-not-launch".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: Some(PollCondition::ExitCode {
codes: vec![if judgments[index] { exits[index] } else { 1 }],
}),
})
.collect::<Vec<_>>();
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: commands.clone(),
topology,
};
let now = now_ms();
let parent = Execution {
id: "recovered-verdict-parent".to_string(),
parent_id: None,
session_id: None,
spec: execution_spec,
status: ExecutionStatus::Running,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
store.put_execution_record(&parent).unwrap();
put_execution_lease(&store, &parent, false);
let mut children = Vec::new();
for (index, command) in commands.iter().enumerate() {
let mut child = Execution {
id: format!("recovered-verdict-child-{index}"),
parent_id: Some(parent.id.clone()),
session_id: None,
spec: child_spec(&parent, command),
status: if exits[index] == 0 {
ExecutionStatus::Completed
} else {
ExecutionStatus::Failed
},
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: Some(serde_json::json!({"exit_code": exits[index]})),
};
child.record_expectation(judgments[index], None);
store.put_execution_record(&child).unwrap();
put_execution_lease(&store, &child, true);
children.push(child);
}
manager.recover().unwrap();
let finished = manager
.wait(&parent.id, Some(Duration::from_secs(2)))
.await
.unwrap();
assert_eq!(
finished.status,
if passes {
ExecutionStatus::Completed
} else {
ExecutionStatus::Failed
},
"exits={exits:?}, judgments={judgments:?}: {finished:?}"
);
let failed = judgments.iter().filter(|met| !**met).count();
assert_eq!(
finished.result.as_ref().unwrap()["counts"],
serde_json::json!({
"total": 2, "completed": 2 - failed, "failed": failed, "canceled": 0, "skipped": 0
})
);
manager.recover().unwrap();
assert_eq!(store.list_execution_records().unwrap().len(), 3);
assert_eq!(
serde_json::to_value(manager.get(&parent.id).unwrap()).unwrap(),
serde_json::to_value(&finished).unwrap()
);
for child in &children {
assert_eq!(
serde_json::to_value(manager.get(&child.id).unwrap()).unwrap(),
serde_json::to_value(child).unwrap()
);
assert!(store.get_run(&child.id).unwrap().is_none());
assert!(
store
.get_lease(&lease_id(&child.id))
.unwrap()
.unwrap()
.released
);
}
assert!(
store
.get_lease(&lease_id(&parent.id))
.unwrap()
.unwrap()
.released
);
assert_eq!(
store
.causal_events_for_subject(&format!("execution/{}", parent.id), 100)
.unwrap()
.iter()
.filter(|event| matches!(
event.kind.as_str(),
"execution.completed" | "execution.failed"
))
.count(),
1
);
let artifact = finished.result.as_ref().unwrap()["children_artifact"]
.as_str()
.unwrap();
let outcomes = store.get(&parent.id, artifact).await.unwrap().unwrap();
for (index, child) in children.iter().enumerate() {
assert_eq!(outcomes[index]["execution_id"], child.id);
assert_eq!(outcomes[index]["exit_code"], exits[index]);
assert_eq!(
outcomes[index]["status"],
if judgments[index] {
"completed"
} else {
"failed"
}
);
}
}
}
#[tokio::test]
async fn artifact_failure_terminalizes_a_recovered_command_group() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let command = CommandUnit {
id: "child".to_string(),
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
};
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![command.clone()],
topology: CommandTopology::Parallel {
max_parallel: 1,
fail_fast: false,
},
};
let now = now_ms();
let parent = Execution {
id: "recovered-artifact-parent".to_string(),
parent_id: None,
session_id: None,
spec: execution_spec,
status: ExecutionStatus::Running,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
let child = Execution {
id: "recovered-artifact-child".to_string(),
parent_id: Some(parent.id.clone()),
session_id: None,
spec: child_spec(&parent, &command),
status: ExecutionStatus::Completed,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: Some(serde_json::json!({ "exit_code": 0 })),
};
store.put_execution_record(&parent).unwrap();
store.put_execution_record(&child).unwrap();
put_execution_lease(&store, &parent, false);
put_execution_lease(&store, &child, true);
reject_artifact_writes(&store);
manager.recover().unwrap();
let finished = manager
.wait(&parent.id, Some(Duration::from_secs(2)))
.await
.unwrap();
assert_eq!(finished.status, ExecutionStatus::Failed);
assert_eq!(
finished.result.as_ref().unwrap()["error_code"],
"EXECUTION_CLEANUP_FAILED"
);
assert_eq!(finished.result.as_ref().unwrap()["children_complete"], true);
assert!(
finished.result.as_ref().unwrap()["error"]
.as_str()
.unwrap()
.contains("forced artifact failure")
);
assert_eq!(
manager.get(&child.id).unwrap().status,
ExecutionStatus::Completed
);
assert!(
store
.get_lease(&lease_id(&parent.id))
.unwrap()
.unwrap()
.released
);
assert!(
store
.get_lease(&lease_id(&child.id))
.unwrap()
.unwrap()
.released
);
assert_eq!(
store
.causal_events_for_subject(&format!("execution/{}", parent.id), 100)
.unwrap()
.iter()
.filter(|event| event.kind == "execution.failed")
.count(),
1
);
}
#[tokio::test]
async fn recovery_terminalizes_every_missing_pipeline_child() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let commands = vec![
CommandUnit {
id: "first".to_string(),
argv: vec!["/usr/bin/printf".to_string(), "never-run".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
},
CommandUnit {
id: "second".to_string(),
argv: vec!["/usr/bin/cat".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
},
];
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands,
topology: CommandTopology::Pipeline { pipefail: true },
};
let now = now_ms();
let parent = Execution {
id: "missing-pipeline-children-parent".to_string(),
parent_id: None,
session_id: None,
spec: execution_spec,
status: ExecutionStatus::Running,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
store.put_execution_record(&parent).unwrap();
put_execution_lease(&store, &parent, false);
manager.recover().unwrap();
let finished = manager
.wait(&parent.id, Some(Duration::from_secs(2)))
.await
.unwrap();
let children = manager
.list()
.unwrap()
.into_iter()
.filter(|child| child.parent_id.as_deref() == Some(&parent.id))
.collect::<Vec<_>>();
assert_eq!(finished.status, ExecutionStatus::Failed);
assert_eq!(children.len(), 2, "child states: {children:?}");
assert!(
children
.iter()
.all(|child| child.status == ExecutionStatus::Failed)
);
assert_child_leases_released(&store, &children);
assert!(
finished.result.as_ref().unwrap()["failures"]
.as_array()
.unwrap()
.iter()
.all(|failure| !failure["execution_id"].is_null())
);
}
#[tokio::test]
async fn terminal_event_failure_does_not_rewrite_command_group_result() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let now = now_ms();
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: Vec::new(),
topology: CommandTopology::Parallel {
max_parallel: 1,
fail_fast: false,
},
};
let parent = Execution {
id: "terminal-event-parent".to_string(),
parent_id: None,
session_id: None,
spec: execution_spec,
status: ExecutionStatus::Running,
passes: Vec::new(),
attempts: Vec::new(),
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
store.put_execution_record(&parent).unwrap();
put_execution_lease(&store, &parent, false);
rusqlite::Connection::open(store.path())
.unwrap()
.execute_batch(
"CREATE TRIGGER reject_parent_terminal_event
BEFORE INSERT ON causal_events
WHEN NEW.subject = 'execution/terminal-event-parent'
AND NEW.kind = 'execution.completed'
BEGIN
SELECT RAISE(FAIL, 'forced terminal event failure');
END;",
)
.unwrap();
let error = manager
.finish_or_fail_commands(
parent.clone(),
Vec::new(),
false,
&ProcessController::capture(true, Some(parent.id.clone())),
)
.await
.unwrap_err();
let finished = manager.get(&parent.id).unwrap();
assert!(error.contains("forced terminal event failure"));
assert_eq!(finished.status, ExecutionStatus::Completed);
assert_eq!(finished.result.as_ref().unwrap()["counts"]["total"], 0);
assert!(
store
.get_lease(&lease_id(&parent.id))
.unwrap()
.unwrap()
.released
);
assert!(
store
.causal_events_for_subject(&format!("execution/{}", parent.id), 100)
.unwrap()
.is_empty()
);
}
#[tokio::test]
async fn cancels_an_execution_after_its_progress_deadline() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let execution = manager
.start(
spec(
vec!["/bin/sleep".to_string(), "30".to_string()],
ExecutionBudget {
progress_timeout_ms: Some(25),
cancel_on_stall: true,
..ExecutionBudget::default()
},
),
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(2)))
.await
.unwrap();
assert_eq!(finished.status, ExecutionStatus::Canceled);
assert!(
store
.causal_events(0, 100)
.unwrap()
.iter()
.any(|event| event.kind == "execution.stalled")
);
assert!(
finished.last_progress_at < finished.updated_at,
"terminal cancellation must preserve when progress actually stopped: {finished:#?}"
);
assert_eq!(
finished.result.as_ref().unwrap()["last_progress_at"],
finished.last_progress_at
);
assert!(
finished.result.as_ref().unwrap()["stalled_at"]
.as_u64()
.unwrap()
> finished.last_progress_at
);
assert!(
finished.result.as_ref().unwrap()["stalled_at"]
.as_u64()
.unwrap()
<= finished.updated_at
);
}
#[tokio::test]
async fn output_activity_keeps_an_execution_alive_past_its_progress_deadline() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let execution = manager
.start(
spec(
vec![
"/bin/zsh".to_string(),
"-c".to_string(),
"for n in 1 2 3 4 5 6 7 8 9 10 11 12; do printf 'progress-%s\\n' \"$n\"; sleep 0.05; done"
.to_string(),
],
ExecutionBudget {
progress_timeout_ms: Some(250),
cancel_on_stall: true,
..ExecutionBudget::default()
},
),
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(2)))
.await
.unwrap();
let logs = manager.logs(&execution.id, 1024).unwrap();
assert_eq!(finished.status, ExecutionStatus::Completed, "{finished:#?}");
assert!(
finished.updated_at.saturating_sub(finished.created_at) >= 500,
"fixture did not run beyond its progress deadline: {finished:#?}"
);
assert!(logs.stdout.contains("progress-12"), "{logs:#?}");
}
#[tokio::test]
async fn observation_wait_and_get_keep_an_outputting_run_alive() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let execution = manager
.start(
spec(
vec![
"/bin/zsh".to_string(),
"-c".to_string(),
"for n in 1 2 3 4 5 6 7 8 9 10 11 12; do printf 'progress-%s\\n' \"$n\"; sleep 0.05; done"
.to_string(),
],
ExecutionBudget {
progress_timeout_ms: Some(250),
cancel_on_stall: true,
..ExecutionBudget::default()
},
),
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let deadline = tokio::time::Instant::now() + Duration::from_secs(2);
let mut last_progress_at = execution.last_progress_at;
while tokio::time::Instant::now() < deadline {
let observed = manager
.wait(&execution.id, Some(Duration::from_millis(10)))
.await
.unwrap();
let got = manager.get(&execution.id).unwrap();
assert_eq!(observed.status, got.status, "{observed:#?} {got:#?}");
assert!(
got.last_progress_at >= last_progress_at,
"observation rewound progress from {last_progress_at} to {}: {got:#?}",
got.last_progress_at
);
last_progress_at = got.last_progress_at;
if terminal(got.status) {
assert_eq!(got.status, ExecutionStatus::Completed, "{got:#?}");
return;
}
}
panic!("outputting run did not complete under interleaved wait/get");
}
#[tokio::test]
async fn bounds_retained_output_at_terminal_state() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let execution = manager
.start(
spec(
vec![
"/usr/bin/printf".to_string(),
"abcdefghijklmnopqrstuvwxyz".to_string(),
],
ExecutionBudget {
artifact_bytes: Some(10),
..ExecutionBudget::default()
},
),
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
manager
.wait(&execution.id, Some(Duration::from_secs(2)))
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(50)).await;
let finished = manager.get(&execution.id).unwrap();
let logs = manager.logs(&execution.id, 1024).unwrap();
assert!(logs.stdout.len() + logs.stderr.len() <= 10);
assert_eq!(finished.result.unwrap()["output_truncated"], true);
}
#[tokio::test]
async fn cancels_an_execution_that_exceeds_its_child_process_budget() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let execution = manager
.start(
spec(
vec![
"/bin/zsh".to_string(),
"-c".to_string(),
"sleep 30 & wait".to_string(),
],
ExecutionBudget {
child_processes: Some(1),
..ExecutionBudget::default()
},
),
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
manager
.wait(&execution.id, Some(Duration::from_secs(2)))
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(25)).await;
let finished = manager.get(&execution.id).unwrap();
assert_eq!(finished.status, ExecutionStatus::Canceled);
assert_eq!(
finished.result.unwrap()["error_code"],
"EXECUTION_CHILD_PROCESS_BUDGET"
);
}
#[tokio::test]
async fn cancels_an_execution_that_exceeds_its_cpu_time_budget() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store).unwrap());
let execution = manager
.start(
spec(
vec![
"/bin/zsh".to_string(),
"-c".to_string(),
"while :; do :; done".to_string(),
],
ExecutionBudget {
wall_time_ms: Some(5_000),
cpu_time_ms: Some(1),
..ExecutionBudget::default()
},
),
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(3)))
.await
.unwrap();
assert_eq!(finished.status, ExecutionStatus::Canceled);
assert_eq!(
finished.result.unwrap()["error_code"],
"EXECUTION_CPU_TIME_BUDGET"
);
}
#[tokio::test]
async fn applies_finite_retries_and_records_every_attempt_identity() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(
vec![
"/bin/zsh".to_string(),
"-c".to_string(),
"exit 7".to_string(),
],
ExecutionBudget::default(),
);
execution_spec.retry = RetryPolicy {
max_attempts: 2,
backoff_ms: 1,
max_backoff_ms: 1,
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(2)))
.await
.unwrap();
assert_eq!(finished.status, ExecutionStatus::Failed);
assert_eq!(finished.attempts.len(), 2);
assert_ne!(
finished.attempts[0].processes[0],
finished.attempts[1].processes[0]
);
}
#[tokio::test]
async fn cancels_an_execution_during_retry_backoff() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(
vec!["/usr/bin/false".to_string()],
ExecutionBudget::default(),
);
execution_spec.retry = RetryPolicy {
max_attempts: 2,
backoff_ms: 500,
max_backoff_ms: 500,
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(100)).await;
let canceled = manager
.cancel(
&execution.id,
Duration::from_millis(250),
"test/retry-backoff-cancel",
&ProcessController::capture(true, None),
)
.await
.unwrap();
assert_eq!(canceled.status, ExecutionStatus::Canceled);
tokio::time::sleep(Duration::from_millis(600)).await;
let finished = manager.get(&execution.id).unwrap();
assert_eq!(finished.status, ExecutionStatus::Canceled);
assert_eq!(finished.attempts.len(), 1);
}
#[tokio::test]
async fn canceling_parallel_parent_terminalizes_every_child_as_canceled() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: (0..3)
.map(|index| CommandUnit {
id: format!("sleep-{index}"),
argv: vec!["/bin/sleep".to_string(), "30".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
})
.collect(),
topology: CommandTopology::Parallel {
max_parallel: 3,
fail_fast: false,
},
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let started = tokio::time::Instant::now();
loop {
let children = manager
.list()
.unwrap()
.into_iter()
.filter(|child| child.parent_id.as_deref() == Some(&execution.id))
.collect::<Vec<_>>();
if children.len() == 3
&& children
.iter()
.all(|child| child.status == ExecutionStatus::Running)
{
break;
}
assert!(started.elapsed() < Duration::from_secs(2));
tokio::time::sleep(Duration::from_millis(5)).await;
}
let canceled = manager
.cancel(
&execution.id,
Duration::from_secs(2),
"test/parallel-parent-cancel",
&ProcessController::capture(true, None),
)
.await
.unwrap();
assert_eq!(canceled.status, ExecutionStatus::Canceled);
let children = manager
.list()
.unwrap()
.into_iter()
.filter(|child| child.parent_id.as_deref() == Some(&execution.id))
.collect::<Vec<_>>();
let statuses = children
.iter()
.map(|child| (child.spec.name.clone(), child.status))
.collect::<Vec<_>>();
assert_eq!(children.len(), 3, "child states: {statuses:?}");
assert!(
children
.iter()
.all(|child| child.status == ExecutionStatus::Canceled),
"child states: {statuses:?}"
);
assert!(children.iter().all(|child| {
store
.get_run(&child.id)
.unwrap()
.is_some_and(|run| run.status == "stopped" && run.processes.is_empty())
}));
assert_child_leases_released(&store, &children);
assert!(
store
.get_lease(&lease_id(&execution.id))
.unwrap()
.unwrap()
.released
);
}
#[tokio::test]
async fn timed_out_group_cancel_still_finishes_durably() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![CommandUnit {
id: "sleep".to_string(),
argv: vec!["/bin/sleep".to_string(), "30".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
}],
topology: CommandTopology::Parallel {
max_parallel: 1,
fail_fast: false,
},
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let started = tokio::time::Instant::now();
loop {
let child_running = manager.list().unwrap().into_iter().any(|child| {
child.parent_id.as_deref() == Some(&execution.id)
&& child.status == ExecutionStatus::Running
});
if child_running {
break;
}
assert!(started.elapsed() < Duration::from_secs(2));
tokio::time::sleep(Duration::from_millis(5)).await;
}
let error = manager
.cancel(
&execution.id,
Duration::ZERO,
"test/parallel-parent-timeout-cancel",
&ProcessController::capture(true, None),
)
.await
.unwrap_err();
assert!(error.contains("waiting for child cleanup"));
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(2)))
.await
.unwrap();
assert_eq!(finished.status, ExecutionStatus::Canceled);
let child = manager
.list()
.unwrap()
.into_iter()
.find(|child| child.parent_id.as_deref() == Some(&execution.id))
.unwrap();
assert_eq!(child.status, ExecutionStatus::Canceled);
assert!(
store
.get_lease(&lease_id(&execution.id))
.unwrap()
.unwrap()
.released
);
assert!(
store
.get_lease(&lease_id(&child.id))
.unwrap()
.unwrap()
.released
);
}
#[tokio::test]
async fn runs_a_bounded_parallel_batch_as_child_executions() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![
CommandUnit {
id: "first".to_string(),
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
},
CommandUnit {
id: "second".to_string(),
argv: vec![
"/bin/zsh".to_string(),
"-c".to_string(),
"exit 7".to_string(),
],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
},
],
topology: CommandTopology::Parallel {
max_parallel: 2,
fail_fast: false,
},
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(5)))
.await
.unwrap();
assert_eq!(finished.status, ExecutionStatus::Failed);
assert_eq!(finished.result.as_ref().unwrap()["counts"]["total"], 2);
assert_eq!(finished.result.as_ref().unwrap()["counts"]["failed"], 1);
let children = manager
.list()
.unwrap()
.into_iter()
.filter(|child| child.parent_id.as_deref() == Some(&execution.id))
.collect::<Vec<_>>();
assert_eq!(children.len(), 2);
assert!(children.iter().all(|child| terminal(child.status)));
assert_child_leases_released(&store, &children);
}
#[tokio::test]
async fn execution_tree_recursively_includes_owned_child_processes() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![CommandUnit {
id: "sleeper".to_string(),
argv: vec!["/bin/sleep".to_string(), "2".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
}],
topology: CommandTopology::Parallel {
max_parallel: 1,
fail_fast: false,
},
};
let controller = ProcessController::capture(true, None);
let parent = manager
.start(execution_spec, controller.clone(), None, None)
.await
.unwrap();
let tree = loop {
let tree = manager.tree(&parent.id, &controller).unwrap();
if tree.state == ExecutionTreeState::Active {
break tree;
}
tokio::time::sleep(Duration::from_millis(10)).await;
};
assert_eq!(tree.execution_id, parent.id);
assert_eq!(tree.roots.len(), 1);
assert_ne!(tree.roots[0].execution_id, parent.id);
assert_eq!(tree.roots[0].identity.name, "sleep");
assert_eq!(tree.processes[0].identity.name, "sleep");
manager
.cancel(
&parent.id,
Duration::from_secs(1),
"test/execution-tree-parent-cancel",
&ProcessController::capture(true, Some(parent.id.clone())),
)
.await
.unwrap();
}
#[test]
fn execution_tree_reports_a_mismatched_unfinished_identity_as_inconsistent() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut identity = crate::process::identity_for_pid(std::process::id()).unwrap();
identity.started_at_ns = identity.started_at_ns.saturating_add(1);
let now = now_ms();
let execution = Execution {
id: "mismatched-tree-root".to_string(),
parent_id: None,
session_id: None,
spec: spec(
vec!["/usr/bin/true".to_string()],
ExecutionBudget::default(),
),
status: ExecutionStatus::Running,
passes: Vec::new(),
attempts: vec![ExecutionAttempt {
id: "mismatched-attempt".to_string(),
number: 1,
processes: vec![identity],
started_at: now,
completed_at: None,
exit_code: None,
error_code: None,
stdout_artifact: None,
stderr_artifact: None,
}],
created_at: now,
updated_at: now,
last_progress_at: now,
result: None,
};
store.put_execution_record(&execution).unwrap();
let tree = manager
.tree(
&execution.id,
&ProcessController::capture(true, Some(execution.id.clone())),
)
.unwrap();
assert_eq!(tree.state, ExecutionTreeState::Inconsistent);
assert_eq!(tree.roots.len(), 1);
assert!(!tree.roots[0].live);
assert!(tree.processes.is_empty());
}
#[tokio::test]
async fn records_parallel_launch_failures_as_terminal_children() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![
CommandUnit {
id: "valid".to_string(),
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
},
CommandUnit {
id: "missing".to_string(),
argv: vec!["/definitely/missing-apoc-executable".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
},
],
topology: CommandTopology::Parallel {
max_parallel: 2,
fail_fast: false,
},
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(5)))
.await
.unwrap();
let children = manager
.list()
.unwrap()
.into_iter()
.filter(|child| child.parent_id.as_deref() == Some(&execution.id))
.collect::<Vec<_>>();
assert_eq!(finished.status, ExecutionStatus::Failed);
assert_eq!(finished.result.as_ref().unwrap()["counts"]["failed"], 1);
assert_eq!(children.len(), 2);
assert!(children.iter().all(|child| terminal(child.status)));
assert_child_leases_released(&store, &children);
let missing = children
.iter()
.find(|child| child.spec.name == "missing")
.unwrap();
assert_eq!(missing.status, ExecutionStatus::Failed);
assert_eq!(
missing.result.as_ref().unwrap()["error_code"],
"EXECUTION_START_FAILED"
);
}
#[tokio::test]
async fn connects_a_linear_pipeline_with_native_pipes_and_pipefail() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![
CommandUnit {
id: "produce".to_string(),
argv: vec!["/usr/bin/printf".to_string(), "apocalypse".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
},
CommandUnit {
id: "transform".to_string(),
argv: vec![
"/usr/bin/tr".to_string(),
"[:lower:]".to_string(),
"[:upper:]".to_string(),
],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
},
],
topology: CommandTopology::Pipeline { pipefail: true },
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(5)))
.await
.unwrap_or_else(|error| {
let parent = manager.get(&execution.id);
let children = manager.list().map(|executions| {
executions
.into_iter()
.filter(|child| child.parent_id.as_deref() == Some(&execution.id))
.collect::<Vec<_>>()
});
let leases = store.list_leases(false);
let logs = manager.logs(&execution.id, 4096);
panic!(
"{error}; parent={parent:?}; children={children:?}; leases={leases:?}; logs={logs:?}"
);
});
assert_eq!(finished.status, ExecutionStatus::Completed);
assert_eq!(
manager.logs(&execution.id, 1024).unwrap().stdout,
"APOCALYPSE"
);
let children = manager
.list()
.unwrap()
.into_iter()
.filter(|child| child.parent_id.as_deref() == Some(&execution.id))
.collect::<Vec<_>>();
assert_eq!(children.len(), 2);
assert!(
children
.iter()
.all(|child| child.status == ExecutionStatus::Completed)
);
}
#[tokio::test]
async fn records_pipeline_launch_failures_as_terminal_children() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![
CommandUnit {
id: "started".to_string(),
argv: vec!["/bin/sleep".to_string(), "30".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
},
CommandUnit {
id: "missing".to_string(),
argv: vec!["/definitely/missing-apoc-pipeline".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
},
],
topology: CommandTopology::Pipeline { pipefail: true },
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(5)))
.await
.unwrap();
let children = manager
.list()
.unwrap()
.into_iter()
.filter(|child| child.parent_id.as_deref() == Some(&execution.id))
.collect::<Vec<_>>();
assert_eq!(finished.status, ExecutionStatus::Failed);
assert_eq!(children.len(), 2);
assert!(children.iter().all(|child| terminal(child.status)));
assert!(children.iter().all(|child| {
store
.get_lease(&lease_id(&child.id))
.unwrap()
.is_some_and(|lease| lease.released)
}));
assert!(
finished.result.as_ref().unwrap()["failures"]
.as_array()
.unwrap()
.iter()
.all(|failure| !failure["execution_id"].is_null())
);
}
#[tokio::test]
async fn fail_fast_records_unlaunched_commands_as_skipped_children() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![
CommandUnit {
id: "fail".to_string(),
argv: vec!["/usr/bin/false".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
},
CommandUnit {
id: "never".to_string(),
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
},
],
topology: CommandTopology::Parallel {
max_parallel: 1,
fail_fast: true,
},
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(5)))
.await
.unwrap();
assert_eq!(finished.result.as_ref().unwrap()["counts"]["failed"], 1);
assert_eq!(finished.result.as_ref().unwrap()["counts"]["skipped"], 1);
let children = manager
.list()
.unwrap()
.into_iter()
.filter(|child| child.parent_id.as_deref() == Some(&execution.id))
.collect::<Vec<_>>();
assert_eq!(children.len(), 2);
assert_child_leases_released(&store, &children);
}
fn assert_child_leases_released(store: &Store, children: &[Execution]) {
assert!(children.iter().all(|child| {
store
.get_lease(&lease_id(&child.id))
.unwrap()
.is_some_and(|lease| lease.released)
}));
}
#[tokio::test]
async fn pipeline_pipefail_fails_the_parent_when_an_earlier_stage_fails() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let manager = ExecutionManager::new(store.clone(), RunManager::new(store).unwrap());
let mut execution_spec = spec(Vec::new(), ExecutionBudget::default());
execution_spec.program = ExecutionProgram::Commands {
commands: vec![
CommandUnit {
id: "fail".to_string(),
argv: vec!["/usr/bin/false".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
},
CommandUnit {
id: "succeed".to_string(),
argv: vec!["/usr/bin/true".to_string()],
cwd: None,
env: BTreeMap::new(),
secrets: BTreeMap::new(),
shell: false,
timeout_ms: None,
expect: None,
},
],
topology: CommandTopology::Pipeline { pipefail: true },
};
let execution = manager
.start(
execution_spec,
ProcessController::capture(true, None),
None,
None,
)
.await
.unwrap();
let finished = manager
.wait(&execution.id, Some(Duration::from_secs(5)))
.await
.unwrap();
assert_eq!(finished.status, ExecutionStatus::Failed);
assert_eq!(finished.result.as_ref().unwrap()["counts"]["failed"], 1);
}
}