use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use incurs::command::RequestContext;
use incurs_codemode::{Connector, IncurConnector, ToolContext};
use regex::Regex;
use rquickjs::{Context, Runtime};
use serde_json::Value;
use tokio_util::sync::CancellationToken;
use crate::cli::build_agent_cli_with_store;
use crate::execution::ExecutionManager;
use crate::model::{
CausalEvent, Execution, ExecutionBudget, ExecutionProgram, ExecutionSpec, ExecutionStatus,
IoMode, LeaseOwnerKind, PollCondition, PollIteration, PollIterationPage, PollIterationStatus,
PollIterationSummary, PollSchedule, PollTarget, ProcessController, RetryPolicy, SCHEMA_VERSION,
WorkspaceContext, now_ms,
};
use crate::store::Store;
const OBSERVATION_INLINE_BYTES: usize = 32 * 1024;
const PREDICATE_TIMEOUT: Duration = Duration::from_millis(100);
const PREDICATE_MEMORY_BYTES: usize = 16 * 1024 * 1024;
const PREDICATE_STACK_BYTES: usize = 512 * 1024;
const WORKSPACE_CWD_HEADER: &str = "x-apoc-workspace-cwd";
#[derive(Clone)]
pub(crate) struct PollManager {
store: Arc<Store>,
executions: ExecutionManager,
connector: Arc<dyn Connector>,
active: Arc<Mutex<HashMap<String, CancellationToken>>>,
}
enum TargetResult {
Observed(Value),
Canceled,
Fatal(&'static str, String),
}
struct PollProgram {
target: PollTarget,
schedule: PollSchedule,
condition: Option<PollCondition>,
max_iterations: Option<u64>,
duration_ms: Option<u64>,
iteration_timeout_ms: Option<u64>,
}
impl PollManager {
pub fn new(store: Arc<Store>, executions: ExecutionManager) -> Self {
let catalog = build_agent_cli_with_store(Some(store.clone())).tool_catalog();
let connector: Arc<dyn Connector> = Arc::new(IncurConnector::new(catalog));
let active = executions.poll_cancellations();
Self {
store,
executions,
connector,
active,
}
}
pub async fn start(
&self,
spec: ExecutionSpec,
controller: ProcessController,
parent_id: Option<String>,
session_id: Option<String>,
) -> Result<(Execution, bool), String> {
self.validate_apoc_target(&spec).await?;
let (execution, replayed) = self
.executions
.start_with_replay(spec, controller.clone(), parent_id, session_id)
.await?;
if !replayed && !terminal(execution.status) {
self.spawn(execution.clone(), controller);
}
Ok((execution, replayed))
}
pub fn recover(&self) -> Result<(), String> {
for execution in self.store.recoverable_execution_records()? {
if !terminal(execution.status)
&& matches!(execution.spec.program, ExecutionProgram::Poll { .. })
{
let controller = ProcessController::capture(false, Some(execution.id.clone()));
self.spawn(execution, controller);
}
}
Ok(())
}
pub fn notify_cancel(&self, id: &str) -> Result<(), String> {
self.executions.notify_poll_cancellations(id)
}
pub fn iterations(
&self,
poll_id: &str,
limit: usize,
cursor: Option<&str>,
) -> Result<PollIterationPage, String> {
self.require_poll(poll_id)?;
if !(1..=100).contains(&limit) {
return Err("limit must be between 1 and 100".to_string());
}
let before = cursor
.map(|value| {
value
.parse::<u64>()
.map_err(|_| "Invalid poll iteration cursor".to_string())
})
.transpose()?;
let mut values = self
.store
.list_poll_iterations(poll_id, before, limit + 1)?;
let has_more = values.len() > limit;
values.truncate(limit);
let next_cursor = has_more
.then(|| values.last().map(|value| value.number.to_string()))
.flatten();
Ok(PollIterationPage {
iterations: values.iter().map(iteration_summary).collect(),
next_cursor,
})
}
pub fn iteration(&self, poll_id: &str, id: &str) -> Result<PollIteration, String> {
self.require_poll(poll_id)?;
let mut iteration = self
.store
.get_poll_iteration(poll_id, id)?
.ok_or_else(|| format!("Poll iteration \"{id}\" not found"))?;
if iteration.observation.is_none()
&& let Some(payload_id) = iteration.payload_id.as_deref()
{
iteration.observation = self.store.causal_payload(payload_id)?;
}
Ok(iteration)
}
fn require_poll(&self, id: &str) -> Result<Execution, String> {
let execution = self
.store
.get_execution_record(id)?
.ok_or_else(|| format!("Execution \"{id}\" not found"))?;
if !matches!(execution.spec.program, ExecutionProgram::Poll { .. }) {
return Err(format!("Execution \"{id}\" is not a poll"));
}
Ok(execution)
}
async fn validate_apoc_target(&self, spec: &ExecutionSpec) -> Result<(), String> {
let ExecutionProgram::Poll {
target: PollTarget::ApocCall { method, arguments },
..
} = &spec.program
else {
return Ok(());
};
let method = normalize_method(method);
let description = self.connector.describe().await?;
let tool = description
.tools
.iter()
.find(|tool| tool.name == method)
.ok_or_else(|| format!("Unknown apoc poll method \"{method}\""))?;
let mut arguments = arguments.clone();
enrich_apoc_arguments(
&tool.input_schema,
&mut arguments,
spec.purpose.as_deref(),
"poll/preflight",
);
let validator = jsonschema::draft202012::new(&tool.input_schema)
.map_err(|error| format!("Invalid schema for apoc method \"{method}\": {error}"))?;
validator
.validate(&arguments)
.map_err(|error| format!("Invalid arguments for apoc method \"{method}\": {error}"))
}
fn spawn(&self, execution: Execution, controller: ProcessController) {
let token = CancellationToken::new();
if let Ok(mut active) = self.active.lock() {
if active.contains_key(&execution.id) {
return;
}
active.insert(execution.id.clone(), token.clone());
}
let manager = self.clone();
tokio::spawn(async move {
let id = execution.id.clone();
if let Err(error) = manager
.supervise(execution, controller.clone(), token)
.await
{
let _ =
manager.finish_failed(&id, "POLL_SUPERVISOR_FAILED", error, &controller, None);
}
if let Ok(mut active) = manager.active.lock() {
active.remove(&id);
}
});
}
async fn supervise(
&self,
parent: Execution,
controller: ProcessController,
cancel: CancellationToken,
) -> Result<(), String> {
loop {
let current = self.require_poll(&parent.id)?;
if terminal(current.status) {
return Ok(());
}
if cancellation_requested(¤t) || cancel.is_cancelled() {
self.finish_canceled(¤t, &controller)?;
return Ok(());
}
let program = poll_program(¤t)?;
if deadline_reached(¤t, program.duration_ms) {
self.finish_bound(
¤t,
program.condition.as_ref(),
"duration",
&controller,
)?;
return Ok(());
}
let latest = self
.store
.list_poll_iterations(¤t.id, None, 1)?
.into_iter()
.next();
let mut iteration = match latest {
Some(iteration) => iteration,
None => self.accept_iteration(¤t, 1, &controller)?,
};
match iteration.status {
PollIterationStatus::Accepted => {
iteration = self
.run_target(¤t, iteration, &program, &controller, &cancel)
.await?;
}
PollIterationStatus::Running => match &program.target {
PollTarget::Command { .. } if iteration.execution_id.is_some() => {
iteration = self
.resume_command(
¤t,
iteration,
program.duration_ms,
&controller,
&cancel,
)
.await?;
}
_ => {
self.finish_failed(
¤t.id,
"POLL_ITERATION_OUTCOME_UNKNOWN",
"The daemon restarted after dispatch but before recording the typed result"
.to_string(),
&controller,
Some(iteration.id),
)?;
return Ok(());
}
},
PollIterationStatus::Canceled => {
if deadline_reached(¤t, program.duration_ms) {
self.finish_bound(
¤t,
program.condition.as_ref(),
"duration",
&controller,
)?;
} else {
self.finish_canceled(¤t, &controller)?;
}
return Ok(());
}
PollIterationStatus::Failed => {
self.finish_failed(
¤t.id,
iteration
.error_code
.as_deref()
.unwrap_or("POLL_TARGET_FAILED"),
iteration
.error
.clone()
.unwrap_or_else(|| "Poll target failed".to_string()),
&controller,
Some(iteration.id),
)?;
return Ok(());
}
PollIterationStatus::Observed => {}
}
if iteration.status != PollIterationStatus::Observed {
continue;
}
if !iteration.evaluated {
let observation = load_observation(&self.store, &iteration)?;
match program
.condition
.as_ref()
.map(|condition| evaluate_condition(condition, &observation))
.transpose()
{
Ok(matched) => {
iteration.matched = matched.unwrap_or(false);
iteration.evaluated = true;
self.store.put_poll_iteration(&iteration)?;
self.event(
¤t,
&controller,
"poll.iteration_evaluated",
serde_json::json!({
"iteration_id": iteration.id,
"number": iteration.number,
"matched": iteration.matched
}),
)?;
}
Err(error) => {
self.finish_failed(
¤t.id,
"POLL_PREDICATE_FAILED",
error,
&controller,
Some(iteration.id),
)?;
return Ok(());
}
}
}
if iteration.matched {
self.finish_completed(
¤t,
"matched",
true,
Some(iteration.id),
iteration.number,
&controller,
)?;
return Ok(());
}
if program
.max_iterations
.is_some_and(|limit| iteration.number >= limit)
{
self.finish_bound(
¤t,
program.condition.as_ref(),
"max_iterations",
&controller,
)?;
return Ok(());
}
if deadline_reached(¤t, program.duration_ms) {
self.finish_bound(
¤t,
program.condition.as_ref(),
"duration",
&controller,
)?;
return Ok(());
}
if !self
.wait_schedule(
¤t,
&program.schedule,
iteration.number,
program.duration_ms,
&cancel,
)
.await?
{
continue;
}
self.accept_iteration(¤t, iteration.number + 1, &controller)?;
}
}
fn accept_iteration(
&self,
parent: &Execution,
number: u64,
controller: &ProcessController,
) -> Result<PollIteration, String> {
let iteration = PollIteration {
id: uuid::Uuid::now_v7().to_string(),
poll_id: parent.id.clone(),
number,
status: PollIterationStatus::Accepted,
started_at: now_ms(),
completed_at: None,
execution_id: None,
payload_id: None,
observation: None,
matched: false,
evaluated: false,
error_code: None,
error: None,
};
self.store.put_poll_iteration(&iteration)?;
self.event(
parent,
controller,
"poll.iteration_accepted",
serde_json::json!({ "iteration_id": iteration.id, "number": number }),
)?;
Ok(iteration)
}
async fn run_target(
&self,
parent: &Execution,
mut iteration: PollIteration,
program: &PollProgram,
controller: &ProcessController,
cancel: &CancellationToken,
) -> Result<PollIteration, String> {
iteration.status = PollIterationStatus::Running;
self.store.put_poll_iteration(&iteration)?;
let result = match &program.target {
PollTarget::Command { argv, shell } => {
self.run_command(
parent,
&mut iteration,
argv,
*shell,
program.iteration_timeout_ms,
program.duration_ms,
controller,
cancel,
)
.await
}
PollTarget::ApocCall { method, arguments } => {
self.run_apoc_call(
parent,
&iteration,
method,
arguments,
program.iteration_timeout_ms,
program.duration_ms,
cancel,
)
.await
}
};
self.record_target_result(parent, iteration, result, controller)
}
#[allow(clippy::too_many_arguments)]
async fn run_command(
&self,
parent: &Execution,
iteration: &mut PollIteration,
argv: &[String],
shell: bool,
iteration_timeout_ms: Option<u64>,
duration_ms: Option<u64>,
controller: &ProcessController,
cancel: &CancellationToken,
) -> TargetResult {
let spec = poll_child_spec(parent, iteration.number, argv, shell, iteration_timeout_ms);
let child = match self
.executions
.start_with_replay(
spec,
ProcessController::capture(false, Some(parent.id.clone())),
Some(parent.id.clone()),
parent.session_id.clone(),
)
.await
{
Ok((child, _)) => child,
Err(error) => return TargetResult::Fatal("POLL_TARGET_LAUNCH_FAILED", error),
};
iteration.execution_id = Some(child.id.clone());
if let Err(error) = self.store.put_poll_iteration(iteration) {
let _ = self
.executions
.cancel_unclaimed(&child.id, Duration::from_secs(5), controller)
.await;
return TargetResult::Fatal("POLL_ITERATION_PERSIST_FAILED", error);
}
if child
.result
.as_ref()
.and_then(|value| value.get("error_code"))
.and_then(Value::as_str)
== Some("EXECUTION_START_FAILED")
{
return TargetResult::Fatal(
"POLL_TARGET_LAUNCH_FAILED",
child
.result
.as_ref()
.and_then(|value| value.get("error"))
.and_then(Value::as_str)
.unwrap_or("Poll command could not be launched")
.to_string(),
);
}
self.await_command(parent, &child.id, duration_ms, controller, cancel)
.await
}
async fn resume_command(
&self,
parent: &Execution,
iteration: PollIteration,
duration_ms: Option<u64>,
controller: &ProcessController,
cancel: &CancellationToken,
) -> Result<PollIteration, String> {
let child = iteration
.execution_id
.as_deref()
.ok_or_else(|| "Running command iteration has no execution ID".to_string())?;
let result = self
.await_command(parent, child, duration_ms, controller, cancel)
.await;
self.record_target_result(parent, iteration, result, controller)
}
async fn await_command(
&self,
parent: &Execution,
child_id: &str,
duration_ms: Option<u64>,
controller: &ProcessController,
cancel: &CancellationToken,
) -> TargetResult {
let remaining = remaining_duration(parent, duration_ms);
let wait = self.executions.wait(child_id, None);
tokio::pin!(wait);
let child = if let Some(remaining) = remaining {
tokio::select! {
result = &mut wait => result,
_ = cancel.cancelled() => {
let _ = self
.executions
.cancel_unclaimed(child_id, Duration::from_secs(5), controller)
.await;
return TargetResult::Canceled;
}
_ = tokio::time::sleep(remaining) => {
let _ = self
.executions
.cancel_unclaimed(child_id, Duration::from_secs(5), controller)
.await;
return TargetResult::Canceled;
}
}
} else {
tokio::select! {
result = &mut wait => result,
_ = cancel.cancelled() => {
let _ = self
.executions
.cancel_unclaimed(child_id, Duration::from_secs(5), controller)
.await;
return TargetResult::Canceled;
}
}
};
let child = match child {
Ok(child) => child,
Err(error) => return TargetResult::Fatal("POLL_TARGET_FAILED", error),
};
let logs = match self.executions.logs(child_id, OBSERVATION_INLINE_BYTES) {
Ok(logs) => logs,
Err(error) => return TargetResult::Fatal("POLL_TARGET_LOGS_FAILED", error),
};
let exit_code = child
.result
.as_ref()
.and_then(|value| value.get("exit_code"))
.and_then(Value::as_i64)
.and_then(|value| i32::try_from(value).ok());
let parsed = serde_json::from_str::<Value>(&logs.stdout).ok();
TargetResult::Observed(serde_json::json!({
"kind": "command",
"execution_id": child_id,
"status": status_name(child.status),
"exit_code": exit_code,
"stdout": logs.stdout,
"stderr": logs.stderr,
"stdout_truncated": logs.stdout_truncated,
"stderr_truncated": logs.stderr_truncated,
"json": parsed
}))
}
#[allow(clippy::too_many_arguments)]
async fn run_apoc_call(
&self,
parent: &Execution,
iteration: &PollIteration,
method: &str,
arguments: &Value,
iteration_timeout_ms: Option<u64>,
duration_ms: Option<u64>,
cancel: &CancellationToken,
) -> TargetResult {
let method = normalize_method(method);
let mut arguments = arguments.clone();
let description = match self.connector.describe().await {
Ok(description) => description,
Err(error) => return TargetResult::Fatal("POLL_TARGET_SCHEMA_FAILED", error),
};
let schema = description
.tools
.iter()
.find(|tool| tool.name == method)
.map(|tool| &tool.input_schema);
if let Some(schema) = schema {
enrich_apoc_arguments(
schema,
&mut arguments,
parent.spec.purpose.as_deref(),
&format!("poll/{}/{}", parent.id, iteration.number),
);
}
let call_cancel = CancellationToken::new();
let mut headers = HashMap::new();
if let Some(cwd) = parent.spec.workspace.cwd.as_deref() {
headers.insert(WORKSPACE_CWD_HEADER.to_string(), cwd.to_string());
}
let context = ToolContext {
execution_id: parent.id.clone(),
control: incurs::tool::ToolCallControl {
cancellation: call_cancel.clone(),
events: None,
},
request: Some(RequestContext {
headers,
method: "poll".to_string(),
path: "run".to_string(),
}),
};
let call = crate::process::with_execution_context(
parent.id.clone(),
parent.spec.workspace.cwd.clone(),
Some(crate::model::Verbosity::Trace.as_str().to_string()),
parent.spec.profile_selection.clone(),
crate::hook::execution_hook_depth(&parent.spec),
0,
self.connector.execute(&method, arguments, &context),
);
tokio::pin!(call);
let result = tokio::select! {
result = &mut call => result,
_ = cancel.cancelled() => {
call_cancel.cancel();
return TargetResult::Canceled;
}
_ = optional_sleep(iteration_timeout_ms.map(Duration::from_millis)) => {
call_cancel.cancel();
return TargetResult::Observed(serde_json::json!({
"kind": "apoc_call",
"method": method,
"error": {
"code": "POLL_ITERATION_TIMEOUT",
"message": "Typed apoc iteration exceeded iteration_timeout_ms"
}
}));
}
_ = optional_sleep(remaining_duration(parent, duration_ms)) => {
call_cancel.cancel();
return TargetResult::Canceled;
}
};
match result {
Ok(result) => TargetResult::Observed(serde_json::json!({
"kind": "apoc_call",
"method": method,
"result": result
})),
Err(error) => TargetResult::Observed(serde_json::json!({
"kind": "apoc_call",
"method": method,
"error": error
})),
}
}
fn record_target_result(
&self,
parent: &Execution,
mut iteration: PollIteration,
result: TargetResult,
controller: &ProcessController,
) -> Result<PollIteration, String> {
iteration.completed_at = Some(now_ms());
match result {
TargetResult::Observed(observation) => {
iteration.status = PollIterationStatus::Observed;
if serde_json::to_vec(&observation)
.map_err(|error| error.to_string())?
.len()
> OBSERVATION_INLINE_BYTES
{
let payload = self.store.put_causal_payload(
"poll.observation",
Some(&parent.id),
&observation,
)?;
iteration.payload_id = Some(payload.id);
iteration.observation = None;
} else {
iteration.observation = Some(observation);
}
}
TargetResult::Canceled => {
iteration.status = PollIterationStatus::Canceled;
iteration.error_code = Some("POLL_ITERATION_CANCELED".to_string());
}
TargetResult::Fatal(code, error) => {
iteration.status = PollIterationStatus::Failed;
iteration.error_code = Some(code.to_string());
iteration.error = Some(error);
}
}
self.store.put_poll_iteration(&iteration)?;
self.event(
parent,
controller,
match iteration.status {
PollIterationStatus::Observed => "poll.iteration_observed",
PollIterationStatus::Canceled => "poll.iteration_canceled",
PollIterationStatus::Failed => "poll.iteration_failed",
_ => "poll.iteration_updated",
},
serde_json::json!({
"iteration_id": iteration.id,
"number": iteration.number,
"status": iteration.status,
"execution_id": iteration.execution_id,
"payload_id": iteration.payload_id,
"error_code": iteration.error_code
}),
)?;
Ok(iteration)
}
async fn wait_schedule(
&self,
parent: &Execution,
schedule: &PollSchedule,
completed: u64,
duration_ms: Option<u64>,
cancel: &CancellationToken,
) -> Result<bool, String> {
let delay_ms = match schedule {
PollSchedule::FixedDelay { interval_ms } => *interval_ms,
PollSchedule::Backoff {
initial_interval_ms,
multiplier,
max_interval_ms,
} => ((*initial_interval_ms as f64)
* multiplier.powf(completed.saturating_sub(1) as f64))
.min(*max_interval_ms as f64) as u64,
PollSchedule::FixedRate { interval_ms } => {
let elapsed = now_ms().saturating_sub(parent.created_at);
let next_tick = elapsed / interval_ms + 1;
next_tick
.saturating_mul(*interval_ms)
.saturating_sub(elapsed)
}
};
let delay = Duration::from_millis(delay_ms);
let remaining = remaining_duration(parent, duration_ms);
let wait = remaining.map_or(delay, |remaining| delay.min(remaining));
tokio::select! {
_ = tokio::time::sleep(wait) => {}
_ = cancel.cancelled() => return Ok(false),
}
let current = self.require_poll(&parent.id)?;
Ok(!cancellation_requested(¤t) && !deadline_reached(¤t, duration_ms))
}
fn finish_bound(
&self,
parent: &Execution,
condition: Option<&PollCondition>,
reason: &str,
controller: &ProcessController,
) -> Result<(), String> {
let count = self
.store
.list_poll_iterations(&parent.id, None, 1)?
.first()
.map_or(0, |value| value.number);
if condition.is_some() {
self.finish_failed(
&parent.id,
"POLL_CONDITION_NOT_MET",
format!("Poll condition was not met before {reason}"),
controller,
None,
)
} else {
self.finish_completed(parent, reason, false, None, count, controller)
}
}
fn finish_completed(
&self,
parent: &Execution,
stop_reason: &str,
matched: bool,
matched_iteration_id: Option<String>,
count: u64,
controller: &ProcessController,
) -> Result<(), String> {
let mut parent = self.require_poll(&parent.id)?;
if terminal(parent.status) {
return Ok(());
}
parent.status = ExecutionStatus::Completed;
parent.updated_at = now_ms();
parent.last_progress_at = parent.updated_at;
parent.result = Some(serde_json::json!({
"matched": matched,
"stop_reason": stop_reason,
"iterations": count,
"elapsed_ms": parent.updated_at.saturating_sub(parent.created_at),
"last_iteration_id": self.store.list_poll_iterations(&parent.id, None, 1)?
.first().map(|value| value.id.clone()),
"matched_iteration_id": matched_iteration_id
}));
self.store.put_execution_record(&parent)?;
self.store
.release_lease(&format!("execution/{}", parent.id))?;
self.event(
&parent,
controller,
"execution.completed",
parent.result.clone().unwrap_or_default(),
)?;
Ok(())
}
fn finish_failed(
&self,
id: &str,
error_code: &str,
error: String,
controller: &ProcessController,
last_iteration_id: Option<String>,
) -> Result<(), String> {
let mut parent = self.require_poll(id)?;
if terminal(parent.status) {
return Ok(());
}
let count = self
.store
.list_poll_iterations(id, None, 1)?
.first()
.map_or(0, |value| value.number);
parent.status = ExecutionStatus::Failed;
parent.updated_at = now_ms();
parent.last_progress_at = parent.updated_at;
parent.result = Some(serde_json::json!({
"matched": false,
"stop_reason": "failed",
"iterations": count,
"elapsed_ms": parent.updated_at.saturating_sub(parent.created_at),
"last_iteration_id": last_iteration_id,
"error_code": error_code,
"error": error
}));
self.store.put_execution_record(&parent)?;
self.store
.release_lease(&format!("execution/{}", parent.id))?;
self.event(
&parent,
controller,
"execution.failed",
parent.result.clone().unwrap_or_default(),
)?;
Ok(())
}
fn finish_canceled(
&self,
parent: &Execution,
controller: &ProcessController,
) -> Result<(), String> {
let mut parent = self.require_poll(&parent.id)?;
if terminal(parent.status) {
return Ok(());
}
parent.status = ExecutionStatus::Canceled;
parent.updated_at = now_ms();
parent.last_progress_at = parent.updated_at;
let count = self
.store
.list_poll_iterations(&parent.id, None, 1)?
.first()
.map_or(0, |value| value.number);
parent.result = Some(serde_json::json!({
"matched": false,
"stop_reason": "canceled",
"iterations": count,
"elapsed_ms": parent.updated_at.saturating_sub(parent.created_at),
"last_iteration_id": self.store.list_poll_iterations(&parent.id, None, 1)?
.first().map(|value| value.id.clone())
}));
self.store.put_execution_record(&parent)?;
self.store
.release_lease(&format!("execution/{}", parent.id))?;
self.event(
&parent,
controller,
"execution.canceled",
parent.result.clone().unwrap_or_default(),
)?;
Ok(())
}
fn event(
&self,
execution: &Execution,
controller: &ProcessController,
kind: &str,
details: Value,
) -> Result<CausalEvent, String> {
self.store.append_causal_event(CausalEvent {
sequence: 0,
id: uuid::Uuid::now_v7().to_string(),
at: now_ms(),
kind: kind.to_string(),
subject: format!("execution/{}", execution.id),
actor: controller.actor.clone(),
correlation_id: execution.id.clone(),
causation_id: None,
execution_id: Some(execution.id.clone()),
session_id: execution.session_id.clone(),
payload_id: None,
details,
})
}
}
fn poll_program(execution: &Execution) -> Result<PollProgram, String> {
let ExecutionProgram::Poll {
target,
schedule,
condition,
max_iterations,
duration_ms,
iteration_timeout_ms,
} = &execution.spec.program
else {
return Err(format!("Execution \"{}\" is not a poll", execution.id));
};
Ok(PollProgram {
target: target.clone(),
schedule: schedule.clone(),
condition: condition.clone(),
max_iterations: *max_iterations,
duration_ms: *duration_ms,
iteration_timeout_ms: *iteration_timeout_ms,
})
}
fn poll_child_spec(
parent: &Execution,
number: u64,
argv: &[String],
shell: bool,
timeout_ms: Option<u64>,
) -> ExecutionSpec {
let mut labels = parent.spec.labels.clone();
labels.insert("apoc.poll_iteration".to_string(), number.to_string());
ExecutionSpec {
schema_version: SCHEMA_VERSION,
idempotency_key: Some(format!("poll/{}/{}", parent.id, number)),
name: format!("{} iteration {number}", parent.spec.name),
purpose: parent.spec.purpose.clone(),
labels,
program: ExecutionProgram::Command {
argv: argv.to_vec(),
shell,
},
backend: parent.spec.backend.clone(),
workspace: WorkspaceContext {
cwd: parent.spec.workspace.cwd.clone(),
..parent.spec.workspace.clone()
},
env: parent.spec.env.clone(),
secrets: parent.spec.secrets.clone(),
profile_selection: parent.spec.profile_selection.clone(),
io: IoMode::Pipes,
owner_kind: LeaseOwnerKind::Execution,
owner_id: Some(parent.id.clone()),
expires_at: parent.spec.expires_at,
retry: RetryPolicy::default(),
budget: ExecutionBudget {
wall_time_ms: timeout_ms,
artifact_bytes: parent.spec.budget.artifact_bytes,
..ExecutionBudget::default()
},
telemetry: crate::model::TelemetryPolicy::default(),
expect: None,
}
}
pub(crate) fn command_observation(
exit_code: Option<i32>,
status: &str,
logs: &crate::run::RunLogs,
) -> Value {
serde_json::json!({
"kind": "command",
"status": status,
"exit_code": exit_code,
"stdout": logs.stdout,
"stderr": logs.stderr,
"stdout_truncated": logs.stdout_truncated,
"stderr_truncated": logs.stderr_truncated,
"json": serde_json::from_str::<Value>(&logs.stdout).ok()
})
}
pub(crate) fn judge(condition: &PollCondition, observation: &Value) -> Result<bool, String> {
evaluate_condition(condition, observation)
}
fn evaluate_condition(condition: &PollCondition, observation: &Value) -> Result<bool, String> {
match condition {
PollCondition::ExitCode { codes } => Ok(observation
.get("exit_code")
.and_then(Value::as_i64)
.and_then(|value| i32::try_from(value).ok())
.is_some_and(|value| codes.contains(&value))),
PollCondition::StdoutRegex { pattern } => Regex::new(pattern)
.map_err(|error| error.to_string())
.map(|regex| {
observation
.get("stdout")
.and_then(Value::as_str)
.is_some_and(|value| regex.is_match(value))
}),
PollCondition::StderrRegex { pattern } => Regex::new(pattern)
.map_err(|error| error.to_string())
.map(|regex| {
observation
.get("stderr")
.and_then(Value::as_str)
.is_some_and(|value| regex.is_match(value))
}),
PollCondition::JsonEquals { pointer, value } => {
let root = if observation.get("kind").and_then(Value::as_str) == Some("command") {
observation.get("json").unwrap_or(&Value::Null)
} else {
observation.get("result").unwrap_or(&Value::Null)
};
Ok(root.pointer(pointer) == Some(value))
}
PollCondition::Javascript { expression } => {
evaluate_javascript(expression.clone(), observation.clone())
}
}
}
fn evaluate_javascript(expression: String, observation: Value) -> Result<bool, String> {
let handle = std::thread::Builder::new()
.name("apoc-poll-predicate".to_string())
.spawn(move || {
let runtime = Runtime::new().map_err(|error| error.to_string())?;
runtime.set_memory_limit(PREDICATE_MEMORY_BYTES);
runtime.set_max_stack_size(PREDICATE_STACK_BYTES);
let started = Instant::now();
runtime.set_interrupt_handler(Some(Box::new(move || {
started.elapsed() >= PREDICATE_TIMEOUT
})));
let context = Context::full(&runtime).map_err(|error| error.to_string())?;
let observation = serde_json::to_string(&observation)
.and_then(|value| serde_json::to_string(&value))
.map_err(|error| error.to_string())?;
let source = format!(
"const observation = JSON.parse({observation});\n\
const __result = ({expression});\n\
if (typeof __result !== 'boolean') throw new TypeError('predicate must return boolean');\n\
__result;"
);
context
.with(|context| context.eval::<bool, _>(source))
.map_err(|error| error.to_string())
})
.map_err(|error| error.to_string())?;
handle
.join()
.map_err(|_| "Poll predicate evaluator panicked".to_string())?
}
fn load_observation(store: &Store, iteration: &PollIteration) -> Result<Value, String> {
if let Some(observation) = iteration.observation.clone() {
return Ok(observation);
}
let payload_id = iteration
.payload_id
.as_deref()
.ok_or_else(|| "Observed poll iteration has no observation payload".to_string())?;
store
.causal_payload(payload_id)?
.ok_or_else(|| format!("Poll observation payload \"{payload_id}\" is unavailable"))
}
fn normalize_method(method: &str) -> String {
method.trim().replace('.', "_")
}
fn schema_has_property(schema: &Value, name: &str) -> bool {
schema
.get("properties")
.and_then(Value::as_object)
.is_some_and(|properties| properties.contains_key(name))
}
fn enrich_apoc_arguments(
schema: &Value,
arguments: &mut Value,
purpose: Option<&str>,
idempotency_key: &str,
) {
let Some(values) = arguments.as_object_mut() else {
return;
};
if schema_has_property(schema, "purpose")
&& !values.contains_key("purpose")
&& let Some(purpose) = purpose
{
values.insert("purpose".to_string(), Value::String(purpose.to_string()));
}
if schema_has_property(schema, "idempotency_key") && !values.contains_key("idempotency_key") {
values.insert(
"idempotency_key".to_string(),
Value::String(idempotency_key.to_string()),
);
}
}
fn iteration_summary(iteration: &PollIteration) -> PollIterationSummary {
PollIterationSummary {
id: iteration.id.clone(),
poll_id: iteration.poll_id.clone(),
number: iteration.number,
status: iteration.status,
started_at: iteration.started_at,
completed_at: iteration.completed_at,
execution_id: iteration.execution_id.clone(),
payload_id: iteration.payload_id.clone(),
matched: iteration.matched,
error_code: iteration.error_code.clone(),
}
}
fn remaining_duration(parent: &Execution, duration_ms: Option<u64>) -> Option<Duration> {
duration_ms.map(|duration| {
Duration::from_millis(
parent
.created_at
.saturating_add(duration)
.saturating_sub(now_ms()),
)
})
}
async fn optional_sleep(duration: Option<Duration>) {
match duration {
Some(duration) => tokio::time::sleep(duration).await,
None => std::future::pending::<()>().await,
}
}
fn deadline_reached(parent: &Execution, duration_ms: Option<u64>) -> bool {
duration_ms.is_some_and(|duration| now_ms() >= parent.created_at.saturating_add(duration))
}
fn cancellation_requested(execution: &Execution) -> bool {
execution
.result
.as_ref()
.and_then(|result| result.get("cancel_requested"))
.and_then(Value::as_bool)
.unwrap_or(false)
}
fn status_name(status: ExecutionStatus) -> &'static str {
match status {
ExecutionStatus::Queued => "queued",
ExecutionStatus::Running => "running",
ExecutionStatus::Stalled => "stalled",
ExecutionStatus::Interrupted => "interrupted",
ExecutionStatus::Completed => "completed",
ExecutionStatus::Failed => "failed",
ExecutionStatus::Canceled => "canceled",
}
}
fn terminal(status: ExecutionStatus) -> bool {
matches!(
status,
ExecutionStatus::Completed | ExecutionStatus::Failed | ExecutionStatus::Canceled
)
}
#[cfg(test)]
mod tests {
use std::collections::BTreeMap;
use tempfile::tempdir;
use super::*;
use crate::model::Lease;
use crate::run::RunManager;
#[test]
fn javascript_predicate_is_boolean_and_bounded() {
assert!(
evaluate_javascript(
"observation.ready === true".to_string(),
serde_json::json!({"ready": true})
)
.unwrap()
);
assert!(evaluate_javascript("42".to_string(), Value::Null).is_err());
assert!(evaluate_javascript("(()=>{while(true){}})()".to_string(), Value::Null).is_err());
}
#[test]
fn common_conditions_match_normalized_observations() {
let observation = serde_json::json!({
"kind": "command",
"exit_code": 0,
"stdout": "ready\n",
"stderr": "",
"json": {"state": "ready"}
});
assert!(
evaluate_condition(&PollCondition::ExitCode { codes: vec![0] }, &observation).unwrap()
);
assert!(
evaluate_condition(
&PollCondition::StdoutRegex {
pattern: "ready".to_string()
},
&observation
)
.unwrap()
);
assert!(
evaluate_condition(
&PollCondition::JsonEquals {
pointer: "/state".to_string(),
value: Value::String("ready".to_string())
},
&observation
)
.unwrap()
);
}
#[tokio::test]
async fn covers_poll_program_observation_argument_and_timing_helpers() {
let command_target = PollTarget::Command {
argv: vec!["/usr/bin/true".to_string()],
shell: false,
};
let program = ExecutionProgram::Poll {
target: command_target.clone(),
schedule: PollSchedule::FixedDelay { interval_ms: 10 },
condition: Some(PollCondition::ExitCode { codes: vec![0] }),
max_iterations: Some(3),
duration_ms: Some(1_000),
iteration_timeout_ms: Some(500),
};
let mut parent = fixture_execution("poll", program, ExecutionStatus::Running, now_ms());
let resolved = poll_program(&parent).unwrap();
assert!(matches!(resolved.target, PollTarget::Command { .. }));
assert!(matches!(
resolved.schedule,
PollSchedule::FixedDelay { interval_ms: 10 }
));
assert_eq!(resolved.max_iterations, Some(3));
assert_eq!(resolved.duration_ms, Some(1_000));
assert_eq!(resolved.iteration_timeout_ms, Some(500));
let non_poll = fixture_execution(
"command",
ExecutionProgram::Command {
argv: vec!["/usr/bin/true".to_string()],
shell: false,
},
ExecutionStatus::Running,
now_ms(),
);
assert!(poll_program(&non_poll).is_err());
let child = poll_child_spec(
&parent,
7,
&["/usr/bin/printf".to_string(), "ready".to_string()],
false,
Some(250),
);
assert_eq!(child.idempotency_key.as_deref(), Some("poll/poll/7"));
assert_eq!(child.labels["apoc.poll_iteration"], "7");
assert_eq!(child.owner_id.as_deref(), Some("poll"));
assert_eq!(child.budget.wall_time_ms, Some(250));
assert_eq!(child.io, IoMode::Pipes);
let logs = crate::run::RunLogs {
id: "run".to_string(),
stdout: "{\"state\":\"ready\"}".to_string(),
stderr: "warning".to_string(),
stdout_truncated: true,
stderr_truncated: false,
tail_bytes: 1024,
};
let observation = command_observation(Some(0), "completed", &logs);
assert_eq!(observation["json"]["state"], "ready");
assert_eq!(observation["stdout_truncated"], true);
let invalid_json = command_observation(
Some(1),
"failed",
&crate::run::RunLogs {
stdout: "not-json".to_string(),
..logs.clone()
},
);
assert!(invalid_json["json"].is_null());
assert!(
!evaluate_condition(&PollCondition::ExitCode { codes: vec![1] }, &observation).unwrap()
);
assert!(
evaluate_condition(
&PollCondition::StderrRegex {
pattern: "warn".to_string()
},
&observation
)
.unwrap()
);
assert!(
evaluate_condition(
&PollCondition::StdoutRegex {
pattern: "[".to_string()
},
&observation
)
.is_err()
);
let typed = serde_json::json!({
"kind": "apoc_call",
"result": {"state": "ready"}
});
assert!(
evaluate_condition(
&PollCondition::JsonEquals {
pointer: "/state".to_string(),
value: Value::String("ready".to_string())
},
&typed
)
.unwrap()
);
assert!(
!evaluate_condition(
&PollCondition::JsonEquals {
pointer: "/missing".to_string(),
value: Value::Null
},
&typed
)
.unwrap()
);
assert_eq!(normalize_method(" execution.get "), "execution_get");
let schema = serde_json::json!({
"type": "object",
"properties": {
"purpose": {"type": "string"},
"idempotency_key": {"type": "string"}
}
});
assert!(schema_has_property(&schema, "purpose"));
assert!(!schema_has_property(&schema, "missing"));
let mut arguments = serde_json::json!({});
enrich_apoc_arguments(&schema, &mut arguments, Some("Poll purpose"), "poll/key");
assert_eq!(arguments["purpose"], "Poll purpose");
assert_eq!(arguments["idempotency_key"], "poll/key");
let mut preserved = serde_json::json!({
"purpose": "Explicit",
"idempotency_key": "explicit-key"
});
enrich_apoc_arguments(&schema, &mut preserved, Some("Ignored"), "ignored");
assert_eq!(preserved["purpose"], "Explicit");
assert_eq!(preserved["idempotency_key"], "explicit-key");
let mut scalar = Value::Null;
enrich_apoc_arguments(&schema, &mut scalar, Some("Ignored"), "ignored");
assert!(scalar.is_null());
let directory = tempdir().unwrap();
let store = Store::open(directory.path().join("state.sqlite3")).unwrap();
let mut iteration = fixture_iteration();
iteration.observation = Some(serde_json::json!({"inline": true}));
assert_eq!(
load_observation(&store, &iteration).unwrap()["inline"],
true
);
iteration.observation = None;
assert!(load_observation(&store, &iteration).is_err());
let payload = store
.put_causal_payload(
"poll.observation",
Some("poll"),
&serde_json::json!({"retained": true}),
)
.unwrap();
iteration.payload_id = Some(payload.id.clone());
assert_eq!(
load_observation(&store, &iteration).unwrap()["retained"],
true
);
store.prune_causal_payloads_to_size(0).unwrap();
assert!(load_observation(&store, &iteration).is_err());
iteration.status = PollIterationStatus::Observed;
iteration.completed_at = Some(20);
iteration.matched = true;
let summary = iteration_summary(&iteration);
assert_eq!(summary.number, 1);
assert!(summary.matched);
assert_eq!(summary.payload_id, Some(payload.id));
assert!(remaining_duration(&parent, Some(1_000)).is_some());
assert!(remaining_duration(&parent, None).is_none());
parent.created_at = 1;
assert_eq!(remaining_duration(&parent, Some(1)), Some(Duration::ZERO));
assert!(deadline_reached(&parent, Some(1)));
assert!(!deadline_reached(&parent, None));
assert!(!cancellation_requested(&parent));
parent.result = Some(serde_json::json!({"cancel_requested": true}));
assert!(cancellation_requested(&parent));
optional_sleep(Some(Duration::ZERO)).await;
for (status, name) in [
(ExecutionStatus::Queued, "queued"),
(ExecutionStatus::Running, "running"),
(ExecutionStatus::Stalled, "stalled"),
(ExecutionStatus::Interrupted, "interrupted"),
(ExecutionStatus::Completed, "completed"),
(ExecutionStatus::Failed, "failed"),
(ExecutionStatus::Canceled, "canceled"),
] {
assert_eq!(status_name(status), name);
}
assert!(!terminal(ExecutionStatus::Running));
assert!(terminal(ExecutionStatus::Completed));
assert!(terminal(ExecutionStatus::Failed));
assert!(terminal(ExecutionStatus::Canceled));
}
#[tokio::test]
async fn persists_target_results_and_recovers_unknown_running_iteration() {
let directory = tempdir().unwrap();
let store = Arc::new(Store::open(directory.path().join("state.sqlite3")).unwrap());
let executions =
ExecutionManager::new(store.clone(), RunManager::new(store.clone()).unwrap());
let manager = PollManager::new(store.clone(), executions);
let controller = ProcessController::capture(true, None);
let program = ExecutionProgram::Poll {
target: PollTarget::ApocCall {
method: "execution_get".to_string(),
arguments: serde_json::json!({"id": "missing"}),
},
schedule: PollSchedule::FixedDelay { interval_ms: 10 },
condition: None,
max_iterations: Some(2),
duration_ms: None,
iteration_timeout_ms: Some(50),
};
let parent = fixture_execution(
"poll-results",
program.clone(),
ExecutionStatus::Running,
now_ms(),
);
store.put_execution_record(&parent).unwrap();
let mut inline = fixture_iteration();
inline.id = "inline".to_string();
inline.poll_id = parent.id.clone();
inline.execution_id = None;
let inline = manager
.record_target_result(
&parent,
inline,
TargetResult::Observed(serde_json::json!({"ready": true})),
&controller,
)
.unwrap();
assert_eq!(inline.status, PollIterationStatus::Observed);
assert_eq!(inline.observation.unwrap()["ready"], true);
assert!(inline.payload_id.is_none());
let mut retained = fixture_iteration();
retained.id = "retained".to_string();
retained.poll_id = parent.id.clone();
retained.number = 2;
retained.execution_id = None;
let retained = manager
.record_target_result(
&parent,
retained,
TargetResult::Observed(serde_json::json!({
"output": "x".repeat(OBSERVATION_INLINE_BYTES + 1)
})),
&controller,
)
.unwrap();
assert!(retained.observation.is_none());
assert!(retained.payload_id.is_some());
assert_eq!(
manager
.iteration(&parent.id, &retained.id)
.unwrap()
.observation
.unwrap()["output"]
.as_str()
.unwrap()
.len(),
OBSERVATION_INLINE_BYTES + 1
);
let mut canceled = fixture_iteration();
canceled.id = "canceled".to_string();
canceled.poll_id = parent.id.clone();
canceled.number = 3;
canceled.execution_id = None;
let canceled = manager
.record_target_result(&parent, canceled, TargetResult::Canceled, &controller)
.unwrap();
assert_eq!(canceled.status, PollIterationStatus::Canceled);
assert_eq!(
canceled.error_code.as_deref(),
Some("POLL_ITERATION_CANCELED")
);
let mut failed = fixture_iteration();
failed.id = "failed".to_string();
failed.poll_id = parent.id.clone();
failed.number = 4;
failed.execution_id = None;
let failed = manager
.record_target_result(
&parent,
failed,
TargetResult::Fatal("POLL_TARGET_FAILED", "target failed".to_string()),
&controller,
)
.unwrap();
assert_eq!(failed.status, PollIterationStatus::Failed);
assert_eq!(failed.error_code.as_deref(), Some("POLL_TARGET_FAILED"));
assert_eq!(failed.error.as_deref(), Some("target failed"));
let recovered = fixture_execution(
"poll-recovered",
program,
ExecutionStatus::Running,
now_ms(),
);
store.put_execution_record(&recovered).unwrap();
put_execution_lease(&store, &recovered);
let mut running = fixture_iteration();
running.id = "running".to_string();
running.poll_id = recovered.id.clone();
running.execution_id = None;
store.put_poll_iteration(&running).unwrap();
manager
.supervise(
recovered.clone(),
ProcessController::capture(true, Some(recovered.id.clone())),
CancellationToken::new(),
)
.await
.unwrap();
let recovered = store.get_execution_record(&recovered.id).unwrap().unwrap();
assert_eq!(recovered.status, ExecutionStatus::Failed);
assert_eq!(
recovered.result.as_ref().unwrap()["error_code"],
"POLL_ITERATION_OUTCOME_UNKNOWN"
);
assert!(
store
.get_lease(&format!("execution/{}", recovered.id))
.unwrap()
.unwrap()
.released
);
manager
.finish_failed(
&recovered.id,
"IGNORED",
"already terminal".to_string(),
&controller,
None,
)
.unwrap();
}
fn put_execution_lease(store: &Store, execution: &Execution) {
let now = now_ms();
store
.put_lease(&Lease {
id: format!("execution/{}", execution.id),
owner_kind: LeaseOwnerKind::Execution,
owner_id: execution.id.clone(),
resources: Vec::new(),
created_at: now,
renewed_at: now,
expires_at: None,
released: false,
})
.unwrap();
}
fn fixture_execution(
id: &str,
program: ExecutionProgram,
status: ExecutionStatus,
created_at: u64,
) -> Execution {
Execution {
id: id.to_string(),
parent_id: None,
session_id: None,
spec: ExecutionSpec {
schema_version: SCHEMA_VERSION,
idempotency_key: None,
name: id.to_string(),
purpose: Some("Poll fixture".to_string()),
labels: BTreeMap::from([("fixture".to_string(), "true".to_string())]),
program,
backend: crate::model::ExecutionBackendSpec::default(),
workspace: WorkspaceContext::default(),
env: BTreeMap::new(),
secrets: BTreeMap::new(),
profile_selection: None,
io: IoMode::Pipes,
owner_kind: LeaseOwnerKind::Persistent,
owner_id: None,
expires_at: None,
retry: RetryPolicy::default(),
budget: ExecutionBudget::default(),
telemetry: crate::model::TelemetryPolicy::default(),
expect: None,
},
status,
passes: Vec::new(),
attempts: Vec::new(),
created_at,
updated_at: created_at,
last_progress_at: created_at,
result: None,
}
}
fn fixture_iteration() -> PollIteration {
PollIteration {
id: "iteration".to_string(),
poll_id: "poll".to_string(),
number: 1,
status: PollIterationStatus::Running,
started_at: 10,
completed_at: None,
execution_id: Some("child".to_string()),
payload_id: None,
observation: None,
matched: false,
evaluated: true,
error_code: None,
error: None,
}
}
}