use std::fmt;
use std::fmt::Write as _;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use bytes::Bytes;
use zeph_db::{DbPool, sql};
use crate::backend::execution_lock::ExecutionLock;
use crate::backend::{BackendCapabilities, ExecutionBackend, ExecutionSummary, RedactedEntry};
use crate::cipher::{EntryKindTag, PayloadAad, PayloadCipher, ensure_payload_within_limit};
use crate::config::RetentionPolicy;
use crate::error::DurableError;
use crate::ids::{
ExecutionId, ExecutionKind, IdempotencyKey, JournalSeq, PromiseId, StepId, TimerId,
};
use crate::journal::{EntryKind, ExecutionStatus, Journal, JournalEntry};
use crate::promise::PromiseRecord;
use crate::retention::{CheckpointSnapshot, FoldedStep, decode_checkpoint, encode_checkpoint};
use crate::waiters::NotifyRegistry;
use tracing::Instrument as _;
const SEAL_OVERHEAD_SLACK: u64 = 128;
type ExecutionRow = (String, String, String, i64, i64, Option<i64>, i64);
type RedactedRow = (
i64,
i64,
String,
Option<Vec<u8>>,
Option<String>,
Option<i64>,
i64,
);
fn idem_key_prefix(bytes: &[u8]) -> String {
bytes.iter().take(8).fold(String::new(), |mut acc, b| {
let _ = write!(acc, "{b:02x}");
acc
})
}
pub struct LocalBackend {
pool: DbPool,
cipher: Option<Arc<dyn PayloadCipher>>,
hmac_key: Option<[u8; 32]>,
max_payload_bytes: u64,
promise_waiters: NotifyRegistry,
timer_waiters: NotifyRegistry,
lock_dir: Option<PathBuf>,
orphan_sweep_warned: std::sync::atomic::AtomicBool,
}
impl fmt::Debug for LocalBackend {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("LocalBackend")
.field("cipher", &self.cipher.as_ref().map(|_| "<cipher>"))
.field("hmac_key", &self.hmac_key.as_ref().map(|_| "<redacted>"))
.field("max_payload_bytes", &self.max_payload_bytes)
.finish_non_exhaustive()
}
}
impl LocalBackend {
#[must_use]
pub fn new(pool: DbPool, max_payload_bytes: u64) -> Self {
Self {
pool,
cipher: None,
hmac_key: None,
max_payload_bytes,
promise_waiters: NotifyRegistry::default(),
timer_waiters: NotifyRegistry::default(),
lock_dir: None,
orphan_sweep_warned: std::sync::atomic::AtomicBool::new(false),
}
}
pub async fn open(path: &str, max_payload_bytes: u64) -> Result<Self, DurableError> {
let pool = zeph_db::DbConfig {
url: path.to_string(),
pool_size: 5,
}
.connect()
.await
.map_err(|e| DurableError::storage("open", e))?;
let mut backend = Self::new(pool, max_payload_bytes);
backend.lock_dir = lock_dir_for_path(path);
Ok(backend)
}
#[must_use]
pub fn with_cipher(mut self, cipher: Arc<dyn PayloadCipher>) -> Self {
self.cipher = Some(cipher);
self
}
#[must_use]
pub fn with_hmac_key(mut self, key: [u8; 32]) -> Self {
self.hmac_key = Some(key);
self
}
#[must_use]
pub fn pool(&self) -> &DbPool {
&self.pool
}
pub async fn init(&self) -> Result<(), DurableError> {
zeph_db::run_migrations(&self.pool)
.await
.map_err(|e| DurableError::storage("init", e))?;
Ok(())
}
pub async fn list_executions(
&self,
status: Option<&str>,
kind: Option<&str>,
limit: i64,
) -> Result<Vec<ExecutionSummary>, DurableError> {
let span = tracing::info_span!(
"durable.backend.list",
status = status.unwrap_or("*"),
kind = kind.unwrap_or("*"),
count = tracing::field::Empty,
);
async move {
let rows: Vec<ExecutionRow> =
zeph_db::query_as(sql!(
"SELECT
e.execution_id,
e.kind,
e.status,
e.created_at,
e.updated_at,
e.finalized_at,
(SELECT COUNT(*) FROM durable_journal j WHERE j.execution_id = e.execution_id)
FROM durable_executions e
WHERE e.status = COALESCE(?, e.status)
AND e.kind = COALESCE(?, e.kind)
ORDER BY e.created_at DESC
LIMIT ?"
))
.bind(status)
.bind(kind)
.bind(limit)
.fetch_all(&self.pool)
.await
.map_err(|e| DurableError::storage("list", e))?;
tracing::Span::current().record("count", rows.len());
rows.into_iter()
.map(|(id, kind, status, created, updated, finalized, steps)| {
Ok(ExecutionSummary {
execution_id: parse_execution_id(&id)?,
kind,
status: ExecutionStatus::from_tag(&status).ok_or(DurableError::Decode {
context: "execution status is not a recognized CHECK-constrained value",
})?,
created_at_ms: created,
updated_at_ms: updated,
finalized_at_ms: finalized,
step_count: steps.max(0).cast_unsigned(),
})
})
.collect()
}
.instrument(span)
.await
}
pub async fn read_execution_redacted(
&self,
id: ExecutionId,
) -> Result<Vec<RedactedEntry>, DurableError> {
let exec = id.as_uuid().to_string();
let rows: Vec<RedactedRow> = zeph_db::query_as(sql!(
"SELECT seq, step_id, entry_kind, idem_key, effect_class, LENGTH(payload), created_at
FROM durable_journal WHERE execution_id = ? ORDER BY seq"
))
.bind(&exec)
.fetch_all(&self.pool)
.await
.map_err(|e| DurableError::storage("read_redacted", e))?;
Ok(rows
.into_iter()
.map(
|(seq, step, entry_kind, idem, effect_class, payload_len, created)| RedactedEntry {
seq,
step_id: StepId::new(u32::try_from(step).unwrap_or(0)),
entry_kind,
effect_class,
idem_key_prefix: idem.as_deref().map(idem_key_prefix),
payload_len: payload_len.unwrap_or(0).max(0).cast_unsigned(),
created_at_ms: created,
},
)
.collect())
}
pub async fn count_prunable(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
let cutoffs = crate::retention::PruneCutoffs::from_policy(policy, now_unix_millis());
let (count,): (i64,) = zeph_db::query_as(sql!(
"SELECT COUNT(*) FROM durable_executions
WHERE finalized_at IS NOT NULL
AND ( (status = 'completed' AND finalized_at <= ?)
OR (status IN ('failed', 'aborted') AND finalized_at <= ?) )"
))
.bind(cutoffs.completed_before_ms)
.bind(cutoffs.failed_before_ms)
.fetch_one(&self.pool)
.await
.map_err(|e| DurableError::storage("count_prunable", e))?;
Ok(count.max(0).cast_unsigned())
}
pub async fn count_orphans(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
if policy.stale_running_after_secs == 0 {
return Ok(0);
}
let Some(lock_dir) = self.lock_dir.clone() else {
return Ok(0);
};
let cutoff_ms = orphan_cutoff_ms(policy, now_unix_millis());
let candidates: Vec<(String,)> = zeph_db::query_as(sql!(
"SELECT execution_id FROM durable_executions WHERE status = 'running' AND updated_at <= ?"
))
.bind(cutoff_ms)
.fetch_all(&self.pool)
.await
.map_err(|e| DurableError::storage("count_orphans", e))?;
let mut count = 0u64;
for (exec_str,) in &candidates {
let Ok(execution_id) = parse_execution_id(exec_str) else {
continue;
};
if ExecutionLock::acquire(&lock_dir, execution_id).is_ok() {
count += 1;
}
}
Ok(count)
}
pub async fn open_execution(
&self,
id: ExecutionId,
kind: ExecutionKind,
) -> Result<bool, DurableError> {
let span = tracing::info_span!(
"durable.backend.open",
execution_id = %id.as_uuid(),
kind = kind.as_str(),
is_resume = tracing::field::Empty,
);
async move {
let exec = id.as_uuid().to_string();
let reopened = zeph_db::query(sql!(
"UPDATE durable_executions SET status = 'running', updated_at = ?, finalized_at = NULL
WHERE execution_id = ? AND status IN ('completed', 'failed', 'aborted')"
))
.bind(now_unix_millis())
.bind(&exec)
.execute(&self.pool)
.await
.map_err(|e| DurableError::storage("open", e))?;
if reopened.rows_affected() > 0 {
tracing::Span::current().record("is_resume", true);
return Ok(true);
}
let existing: Option<(String,)> = zeph_db::query_as(sql!(
"SELECT status FROM durable_executions WHERE execution_id = ?"
))
.bind(&exec)
.fetch_optional(&self.pool)
.await
.map_err(|e| DurableError::storage("open", e))?;
if existing.is_some() {
tracing::Span::current().record("is_resume", true);
return Ok(true);
}
let now = now_unix_millis();
zeph_db::query(sql!(
"INSERT INTO durable_executions
(execution_id, kind, status, created_at, updated_at, finalized_at)
VALUES (?, ?, 'running', ?, ?, NULL)"
))
.bind(&exec)
.bind(kind.as_str())
.bind(now)
.bind(now)
.execute(&self.pool)
.await
.map_err(|e| DurableError::storage("open", e))?;
tracing::Span::current().record("is_resume", false);
Ok(false)
}
.instrument(span)
.await
}
pub async fn open_execution_exclusive(
&self,
id: ExecutionId,
kind: ExecutionKind,
) -> Result<(bool, Option<ExecutionLock>), DurableError> {
let lock = self
.lock_dir
.as_deref()
.map(|dir| ExecutionLock::acquire(dir, id))
.transpose()?;
let is_resume = self.open_execution(id, kind).await?;
Ok((is_resume, lock))
}
pub(crate) async fn append_batch(&self, entries: &[JournalEntry]) -> Result<(), DurableError> {
if entries.is_empty() {
return Ok(());
}
let mut rows = Vec::with_capacity(entries.len());
for entry in entries {
rows.push(self.prepare_row(entry)?);
}
let insert = sql!(
"INSERT INTO durable_journal
(execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
);
let mut tx = zeph_db::begin_write(&self.pool)
.await
.map_err(|e| DurableError::storage("append_batch", e))?;
for row in rows {
zeph_db::query(insert)
.bind(row.execution_id)
.bind(row.step_id)
.bind(row.entry_kind)
.bind(row.idem_key)
.bind(row.effect_class)
.bind(row.payload)
.bind(row.payload_version)
.bind(row.hmac)
.bind(row.created_at)
.execute(&mut *tx)
.await
.map_err(|e| DurableError::storage("append_batch", e))?;
}
tx.commit()
.await
.map_err(|e| DurableError::storage("append_batch", e))?;
Ok(())
}
pub(crate) async fn lookup_committed_result(
&self,
id: ExecutionId,
idem_key: IdempotencyKey,
) -> Result<Option<JournalEntry>, DurableError> {
let span = tracing::info_span!(
"durable.journal.lookup_idem",
execution_id = %id.as_uuid(),
found = tracing::field::Empty,
);
async move {
let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
"SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
FROM durable_journal
WHERE execution_id = ? AND idem_key = ? AND entry_kind = 'step_result'
ORDER BY seq LIMIT 1"
))
.bind(id.as_uuid().to_string())
.bind(idem_key.as_bytes().to_vec())
.fetch_all(&self.pool)
.await
.map_err(|e| DurableError::storage("lookup_idem", e))?;
let entry = self.rows_to_entries(id, rows).await?.into_iter().next();
tracing::Span::current().record("found", entry.is_some());
Ok(entry)
}
.instrument(span)
.await
}
pub(crate) async fn max_seq(&self) -> Result<Option<JournalSeq>, DurableError> {
let max: Option<i64> = zeph_db::query_scalar(sql!("SELECT MAX(seq) FROM durable_journal"))
.fetch_one(&self.pool)
.await
.map_err(|e| DurableError::storage("max_seq", e))?;
Ok(max.map(JournalSeq::new))
}
pub(crate) fn promise_waiters(&self) -> &NotifyRegistry {
&self.promise_waiters
}
pub(crate) fn timer_waiters(&self) -> &NotifyRegistry {
&self.timer_waiters
}
pub(crate) async fn insert_promise(
&self,
id: PromiseId,
execution_id: ExecutionId,
resolver_token_hash: [u8; 32],
created_at_ms: i64,
) -> Result<(), DurableError> {
let span = tracing::info_span!("durable.promise.create", promise_id = %id.as_uuid());
async move {
zeph_db::query(sql!(
"INSERT INTO durable_promises
(promise_id, execution_id, resolver_token_hash, resolved, payload, created_at, resolved_at)
VALUES (?, ?, ?, 0, NULL, ?, NULL)"
))
.bind(id.as_uuid().to_string())
.bind(execution_id.as_uuid().to_string())
.bind(resolver_token_hash.to_vec())
.bind(created_at_ms)
.execute(&self.pool)
.await
.map_err(|e| DurableError::storage("insert_promise", e))?;
Ok(())
}
.instrument(span)
.await
}
pub(crate) async fn promise_state(
&self,
id: PromiseId,
) -> Result<Option<PromiseRecord>, DurableError> {
let row: Option<PromiseRowRead> = zeph_db::query_as(sql!(
"SELECT execution_id, resolver_token_hash, resolved, payload
FROM durable_promises WHERE promise_id = ?"
))
.bind(id.as_uuid().to_string())
.fetch_optional(&self.pool)
.await
.map_err(|e| DurableError::storage("promise_state", e))?;
let Some((exec, hash, resolved, payload)) = row else {
return Ok(None);
};
Ok(Some(PromiseRecord {
execution_id: parse_execution_id(&exec)?,
resolver_token_hash: slice_to_array32(&hash, "promise resolver_token_hash")?,
resolved: resolved != 0,
payload,
}))
}
pub(crate) async fn resolve_promise(
&self,
id: PromiseId,
execution_id: ExecutionId,
value_plaintext: &[u8],
resolved_at_ms: i64,
) -> Result<bool, DurableError> {
let span = tracing::info_span!("durable.promise.resolve", promise_id = %id.as_uuid());
async move {
ensure_payload_within_limit(value_plaintext.len(), self.max_payload_bytes)?;
let aad = promise_payload_aad(execution_id, id);
let sealed = self.seal_payload(value_plaintext, &aad)?;
let affected = zeph_db::query(sql!(
"UPDATE durable_promises SET resolved = 1, payload = ?, resolved_at = ?
WHERE promise_id = ? AND resolved = 0"
))
.bind(sealed)
.bind(resolved_at_ms)
.bind(id.as_uuid().to_string())
.execute(&self.pool)
.await
.map_err(|e| DurableError::storage("resolve_promise", e))?
.rows_affected();
if affected > 0 {
self.promise_waiters.wake(id.as_uuid());
}
Ok(affected > 0)
}
.instrument(span)
.await
}
pub(crate) async fn claim_promise_notification(
&self,
id: PromiseId,
notified_at_ms: i64,
) -> Result<bool, DurableError> {
let span = tracing::info_span!("durable.promise.claim_notify", promise_id = %id.as_uuid());
async move {
let affected = zeph_db::query(sql!(
"UPDATE durable_promises SET notified_at = ?
WHERE promise_id = ? AND notified_at IS NULL"
))
.bind(notified_at_ms)
.bind(id.as_uuid().to_string())
.execute(&self.pool)
.await
.map_err(|e| DurableError::storage("claim_promise_notification", e))?
.rows_affected();
Ok(affected > 0)
}
.instrument(span)
.await
}
pub(crate) fn open_promise_payload(
&self,
id: PromiseId,
execution_id: ExecutionId,
sealed: &[u8],
) -> Result<Bytes, DurableError> {
ensure_payload_within_limit(
sealed.len(),
self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
)?;
let aad = promise_payload_aad(execution_id, id);
self.open_payload(sealed, &aad)
}
pub(crate) async fn arm_timer(
&self,
id: TimerId,
execution_id: ExecutionId,
due_at_ms: i64,
created_at_ms: i64,
) -> Result<(), DurableError> {
let span = tracing::info_span!("durable.timer.arm", timer_id = %id.as_uuid(), due_at_ms);
async move {
zeph_db::query(sql!(
"INSERT INTO durable_timers (timer_id, execution_id, due_at, fired, created_at)
VALUES (?, ?, ?, 0, ?)"
))
.bind(id.as_uuid().to_string())
.bind(execution_id.as_uuid().to_string())
.bind(due_at_ms)
.bind(created_at_ms)
.execute(&self.pool)
.await
.map_err(|e| DurableError::storage("arm_timer", e))?;
Ok(())
}
.instrument(span)
.await
}
pub(crate) async fn timer_state(
&self,
id: TimerId,
) -> Result<Option<(i64, bool)>, DurableError> {
let row: Option<(i64, i64)> = zeph_db::query_as(sql!(
"SELECT due_at, fired FROM durable_timers WHERE timer_id = ?"
))
.bind(id.as_uuid().to_string())
.fetch_optional(&self.pool)
.await
.map_err(|e| DurableError::storage("timer_state", e))?;
Ok(row.map(|(due_at, fired)| (due_at, fired != 0)))
}
pub(crate) async fn due_timers(&self, now_ms: i64) -> Result<Vec<TimerId>, DurableError> {
let rows: Vec<(String,)> = zeph_db::query_as(sql!(
"SELECT timer_id FROM durable_timers WHERE fired = 0 AND due_at <= ? ORDER BY due_at"
))
.bind(now_ms)
.fetch_all(&self.pool)
.await
.map_err(|e| DurableError::storage("due_timers", e))?;
rows.into_iter().map(|(id,)| parse_timer_id(&id)).collect()
}
pub(crate) async fn mark_timer_fired(&self, id: TimerId) -> Result<bool, DurableError> {
let span = tracing::info_span!("durable.timer.fire", timer_id = %id.as_uuid());
async move {
let affected = zeph_db::query(sql!(
"UPDATE durable_timers SET fired = 1 WHERE timer_id = ? AND fired = 0"
))
.bind(id.as_uuid().to_string())
.execute(&self.pool)
.await
.map_err(|e| DurableError::storage("mark_timer_fired", e))?
.rows_affected();
if affected > 0 {
self.timer_waiters.wake(id.as_uuid());
}
Ok(affected > 0)
}
.instrument(span)
.await
}
fn open_foldable_steps(
&self,
execution_id: ExecutionId,
rows: Vec<FoldableRowRead>,
) -> Result<Vec<FoldedStep>, DurableError> {
let mut folded = Vec::with_capacity(rows.len());
for (step_raw, idem, version, payload) in rows {
let step = u32::try_from(step_raw).map_err(|_| DurableError::Decode {
context: "checkpoint step_id out of u32 range",
})?;
let idem_bytes = idem.ok_or(DurableError::Decode {
context: "checkpoint step result missing idem_key",
})?;
let idem_key =
IdempotencyKey::from_bytes(slice_to_array32(&idem_bytes, "checkpoint idem_key")?);
let sealed = payload.ok_or(DurableError::Decode {
context: "checkpoint step result missing payload",
})?;
let aad = PayloadAad::new(
execution_id,
StepId::new(step),
EntryKindTag::StepResult,
Some(idem_key),
);
let plaintext = self.open_payload(&sealed, &aad)?;
let payload_version =
u8::try_from(version.unwrap_or(1)).map_err(|_| DurableError::Decode {
context: "checkpoint payload_version out of u8 range",
})?;
folded.push(FoldedStep {
step_id: step,
idem_key: *idem_key.as_bytes(),
payload_version,
payload: plaintext,
});
}
Ok(folded)
}
pub(crate) async fn checkpoint_fold(
&self,
execution_id: ExecutionId,
up_to_step: u32,
) -> Result<u64, DurableError> {
let span = tracing::info_span!(
"durable.journal.checkpoint",
execution_id = %execution_id.as_uuid(),
folded_count = tracing::field::Empty,
);
async move {
let exec = execution_id.as_uuid().to_string();
let rows: Vec<FoldableRowRead> = zeph_db::query_as(sql!(
"SELECT step_id, idem_key, payload_version, payload FROM durable_journal
WHERE execution_id = ? AND entry_kind = 'step_result'
AND effect_class = 'idempotent' AND step_id < ?
ORDER BY step_id"
))
.bind(&exec)
.bind(i64::from(up_to_step))
.fetch_all(&self.pool)
.await
.map_err(|e| DurableError::storage("checkpoint", e))?;
if rows.is_empty() {
return Ok(0);
}
let mut folded = self.open_foldable_steps(execution_id, rows)?;
let lens: Vec<usize> = folded.iter().map(|s| s.payload.len()).collect();
let take = crate::retention::fold_prefix_len(
&lens,
crate::retention::checkpoint_budget(self.max_payload_bytes),
);
if take == 0 {
return Ok(0);
}
folded.truncate(take);
let fold_end = folded.last().map_or(up_to_step, |s| s.step_id.saturating_add(1));
let snapshot = encode_checkpoint(&folded);
let snap_aad =
PayloadAad::new(execution_id, StepId::new(fold_end), EntryKindTag::Checkpoint, None);
let sealed_snapshot = self.seal_payload(&snapshot, &snap_aad)?;
let mut tx = zeph_db::begin_write(&self.pool)
.await
.map_err(|e| DurableError::storage("checkpoint", e))?;
zeph_db::query(sql!(
"INSERT INTO durable_journal
(execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
VALUES (?, ?, 'checkpoint', NULL, NULL, ?, ?, NULL, ?)"
))
.bind(&exec)
.bind(i64::from(fold_end))
.bind(sealed_snapshot)
.bind(i32::from(crate::step::PAYLOAD_VERSION))
.bind(now_unix_millis())
.execute(&mut *tx)
.await
.map_err(|e| DurableError::storage("checkpoint", e))?;
zeph_db::query(sql!(
"DELETE FROM durable_journal
WHERE execution_id = ? AND entry_kind = 'step_result'
AND effect_class = 'idempotent' AND step_id < ?"
))
.bind(&exec)
.bind(i64::from(fold_end))
.execute(&mut *tx)
.await
.map_err(|e| DurableError::storage("checkpoint", e))?;
tx.commit()
.await
.map_err(|e| DurableError::storage("checkpoint", e))?;
let count = folded.len() as u64;
tracing::Span::current().record("folded_count", count);
Ok(count)
}
.instrument(span)
.await
}
pub(crate) async fn read_checkpoints(
&self,
execution_id: ExecutionId,
) -> Result<Vec<JournalEntry>, DurableError> {
let rows: Vec<(i64, Option<Vec<u8>>)> = zeph_db::query_as(sql!(
"SELECT step_id, payload FROM durable_journal
WHERE execution_id = ? AND entry_kind = 'checkpoint' ORDER BY step_id"
))
.bind(execution_id.as_uuid().to_string())
.fetch_all(&self.pool)
.await
.map_err(|e| DurableError::storage("read_checkpoints", e))?;
if rows.is_empty() {
return Ok(Vec::new());
}
let mut folded: CheckpointSnapshot = Vec::new();
for (up_to, payload) in rows {
let up_to = u32::try_from(up_to).map_err(|_| DurableError::Decode {
context: "checkpoint up_to_step out of u32 range",
})?;
let sealed = payload.ok_or(DurableError::Decode {
context: "checkpoint entry missing snapshot payload",
})?;
ensure_payload_within_limit(
sealed.len(),
self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
)?;
let aad = PayloadAad::new(
execution_id,
StepId::new(up_to),
EntryKindTag::Checkpoint,
None,
);
let plaintext = self.open_payload(&sealed, &aad)?;
folded.extend(decode_checkpoint(&plaintext)?);
}
let kind = self.lookup_kind(execution_id).await?;
let entries = folded
.into_iter()
.map(|step| JournalEntry {
seq: None,
execution_id,
kind,
step_id: StepId::new(step.step_id),
entry: EntryKind::StepResult {
idempotency_key: IdempotencyKey::from_bytes(step.idem_key),
payload: step.payload,
effect: crate::EffectClass::Idempotent,
payload_version: step.payload_version,
},
created_at_ms: 0,
})
.collect();
Ok(entries)
}
async fn delete_prune_batch(
&self,
cutoffs: crate::retention::PruneCutoffs,
batch: u64,
) -> Result<u64, DurableError> {
let mut tx = zeph_db::begin_write(&self.pool)
.await
.map_err(|e| DurableError::storage("prune", e))?;
#[cfg(feature = "postgres")]
zeph_db::query(sql!(
"SELECT execution_id FROM durable_executions
WHERE finalized_at IS NOT NULL
AND ( (status = 'completed' AND finalized_at <= ?)
OR (status IN ('failed', 'aborted') AND finalized_at <= ?) )
ORDER BY finalized_at LIMIT ?
FOR UPDATE"
))
.bind(cutoffs.completed_before_ms)
.bind(cutoffs.failed_before_ms)
.bind(i64::try_from(batch).unwrap_or(i64::MAX))
.execute(&mut *tx)
.await
.map_err(|e| DurableError::storage("prune", e))?;
let ids: Vec<(String,)> = zeph_db::query_as(sql!(
"SELECT execution_id FROM durable_executions
WHERE finalized_at IS NOT NULL
AND ( (status = 'completed' AND finalized_at <= ?)
OR (status IN ('failed', 'aborted') AND finalized_at <= ?) )
ORDER BY finalized_at LIMIT ?"
))
.bind(cutoffs.completed_before_ms)
.bind(cutoffs.failed_before_ms)
.bind(i64::try_from(batch).unwrap_or(i64::MAX))
.fetch_all(&mut *tx)
.await
.map_err(|e| DurableError::storage("prune", e))?;
if ids.is_empty() {
tx.commit()
.await
.map_err(|e| DurableError::storage("prune", e))?;
return Ok(0);
}
let journal = sql!("DELETE FROM durable_journal WHERE execution_id = ?");
let promises = sql!("DELETE FROM durable_promises WHERE execution_id = ?");
let timers = sql!("DELETE FROM durable_timers WHERE execution_id = ?");
let executions = sql!(
"DELETE FROM durable_executions
WHERE execution_id = ?
AND finalized_at IS NOT NULL
AND ( (status = 'completed' AND finalized_at <= ?)
OR (status IN ('failed', 'aborted') AND finalized_at <= ?) )"
);
let mut removed = 0u64;
for (id,) in &ids {
for stmt in [journal, promises, timers] {
zeph_db::query(stmt)
.bind(id)
.execute(&mut *tx)
.await
.map_err(|e| DurableError::storage("prune", e))?;
}
let result = zeph_db::query(executions)
.bind(id)
.bind(cutoffs.completed_before_ms)
.bind(cutoffs.failed_before_ms)
.execute(&mut *tx)
.await
.map_err(|e| DurableError::storage("prune", e))?;
removed += result.rows_affected();
}
tx.commit()
.await
.map_err(|e| DurableError::storage("prune", e))?;
Ok(removed)
}
async fn sweep_orphan_batch(
&self,
lock_dir: &std::path::Path,
cutoff_ms: i64,
batch: u64,
cursor: Option<crate::retention::SweepCursor>,
) -> Result<crate::retention::SweepBatchOutcome, DurableError> {
let (after_updated_at, after_exec) = cursor.map_or((i64::MIN, String::new()), |c| {
(c.updated_at_ms, c.execution_id)
});
let candidates: Vec<(String, i64)> = zeph_db::query_as(sql!(
"SELECT execution_id, updated_at FROM durable_executions
WHERE status = 'running' AND updated_at <= ?
AND (updated_at > ? OR (updated_at = ? AND execution_id > ?))
ORDER BY updated_at, execution_id LIMIT ?"
))
.bind(cutoff_ms)
.bind(after_updated_at)
.bind(after_updated_at)
.bind(&after_exec)
.bind(i64::try_from(batch).unwrap_or(i64::MAX))
.fetch_all(&self.pool)
.await
.map_err(|e| DurableError::storage("sweep_orphans", e))?;
let scanned = u64::try_from(candidates.len()).unwrap_or(u64::MAX);
let next_cursor = candidates
.last()
.map(|(id, updated_at)| crate::retention::SweepCursor {
updated_at_ms: *updated_at,
execution_id: id.clone(),
});
let now = now_unix_millis();
let abort = sql!(
"UPDATE durable_executions SET status = 'aborted', finalized_at = ?, updated_at = ?
WHERE execution_id = ? AND status = 'running' AND finalized_at IS NULL"
);
let mut aborted = 0u64;
for (exec_str, _updated_at) in &candidates {
let Ok(execution_id) = parse_execution_id(exec_str) else {
continue;
};
match ExecutionLock::acquire(lock_dir, execution_id) {
Ok(_lock) => {
let result = zeph_db::query(abort)
.bind(now)
.bind(now)
.bind(exec_str)
.execute(&self.pool)
.await
.map_err(|e| DurableError::storage("sweep_orphans", e))?;
aborted += result.rows_affected();
}
Err(DurableError::ExecutionLocked { .. }) => {
}
Err(e) => return Err(e),
}
}
Ok(crate::retention::SweepBatchOutcome {
scanned,
aborted,
next_cursor,
})
}
fn seal_payload(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, DurableError> {
match &self.cipher {
Some(cipher) => Ok(cipher.seal(plaintext, aad)?),
None => Ok(plaintext.to_vec()),
}
}
fn open_payload(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Bytes, DurableError> {
match &self.cipher {
Some(cipher) => Ok(Bytes::from(cipher.open(sealed, aad)?)),
None => Ok(Bytes::copy_from_slice(sealed)),
}
}
fn control_hmac(
&self,
entry: &JournalEntry,
idem_key: Option<&IdempotencyKey>,
) -> Option<Vec<u8>> {
self.compute_control_hmac(
entry.execution_id,
entry.step_id,
entry.entry.tag(),
idem_key,
)
.map(|h| h.to_vec())
}
fn compute_control_hmac(
&self,
execution_id: ExecutionId,
step_id: StepId,
tag: &'static str,
idem_key: Option<&IdempotencyKey>,
) -> Option<[u8; 32]> {
let key = self.hmac_key.as_ref()?;
let mut input = Vec::with_capacity(16 + 4 + 16 + 32);
input.extend_from_slice(execution_id.as_bytes());
input.extend_from_slice(&step_id.value().to_le_bytes());
input.extend_from_slice(tag.as_bytes());
if let Some(k) = idem_key {
input.extend_from_slice(k.as_bytes());
}
Some(*blake3::keyed_hash(key, &input).as_bytes())
}
fn verify_control_hmac(
&self,
execution_id: ExecutionId,
step_id: StepId,
tag: &'static str,
idem_key: Option<&IdempotencyKey>,
stored: Option<[u8; 32]>,
) -> Result<(), DurableError> {
let Some(expected) = self.compute_control_hmac(execution_id, step_id, tag, idem_key) else {
return if stored.is_some() {
Err(DurableError::ControlIntegrity)
} else {
Ok(())
};
};
match stored {
Some(stored) if blake3::Hash::from(expected) == blake3::Hash::from(stored) => Ok(()),
_ => Err(DurableError::ControlIntegrity),
}
}
fn prepare_row(&self, entry: &JournalEntry) -> Result<JournalRow, DurableError> {
let execution_id = entry.execution_id.as_uuid().to_string();
let step_id = i64::from(entry.step_id.value());
let created_at = entry.created_at_ms;
let entry_kind = entry.entry.tag();
match &entry.entry {
EntryKind::StepResult {
idempotency_key,
payload,
effect,
payload_version,
} => {
ensure_payload_within_limit(payload.len(), self.max_payload_bytes)?;
let aad = PayloadAad::new(
entry.execution_id,
entry.step_id,
EntryKindTag::StepResult,
Some(*idempotency_key),
);
let sealed = self.seal_payload(payload.as_ref(), &aad)?;
Ok(JournalRow {
execution_id,
step_id,
entry_kind,
idem_key: Some(idempotency_key.as_bytes().to_vec()),
effect_class: Some(effect.as_str()),
payload: Some(sealed),
payload_version: Some(i32::from(*payload_version)),
hmac: None,
created_at,
})
}
EntryKind::EffectIntent {
idempotency_key,
effect,
hmac: _,
} => {
let hmac = self.control_hmac(entry, Some(idempotency_key));
Ok(JournalRow {
execution_id,
step_id,
entry_kind,
idem_key: Some(idempotency_key.as_bytes().to_vec()),
effect_class: Some(effect.as_str()),
payload: None,
payload_version: None,
hmac,
created_at,
})
}
EntryKind::PromiseCreated { .. }
| EntryKind::PromiseResolved { .. }
| EntryKind::TimerArmed { .. }
| EntryKind::TimerFired { .. }
| EntryKind::Checkpoint { .. } => {
Err(DurableError::UnsupportedEntryKind { kind: entry_kind })
}
}
}
async fn lookup_kind(&self, id: ExecutionId) -> Result<ExecutionKind, DurableError> {
let kind: Option<String> = zeph_db::query_scalar(sql!(
"SELECT kind FROM durable_executions WHERE execution_id = ?"
))
.bind(id.as_uuid().to_string())
.fetch_optional(&self.pool)
.await
.map_err(|e| DurableError::storage("read", e))?;
let kind = kind.ok_or(DurableError::Decode {
context: "journaled entries reference a missing execution row",
})?;
ExecutionKind::from_tag(&kind).ok_or(DurableError::Decode {
context: "execution kind is not reconstructible (custom kind read-back unsupported)",
})
}
fn row_to_entry(
&self,
id: ExecutionId,
kind: ExecutionKind,
row: JournalRowRead,
) -> Result<JournalEntry, DurableError> {
let (
seq,
step_id_raw,
entry_kind,
idem_key,
effect_class,
payload,
payload_version,
hmac,
created_at,
) = row;
let step_id =
StepId::new(
u32::try_from(step_id_raw).map_err(|_| DurableError::Decode {
context: "step_id out of u32 range",
})?,
);
let entry = match entry_kind.as_str() {
"step_result" => {
let idem_bytes = idem_key.ok_or(DurableError::Decode {
context: "step_result idem_key missing",
})?;
let idem_key = IdempotencyKey::from_bytes(slice_to_array32(
&idem_bytes,
"step_result idem_key",
)?);
let effect = effect_class
.as_deref()
.and_then(crate::EffectClass::from_tag)
.ok_or(DurableError::Decode {
context: "step_result effect_class missing or invalid",
})?;
let sealed = payload.ok_or(DurableError::Decode {
context: "step_result payload missing",
})?;
ensure_payload_within_limit(
sealed.len(),
self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
)?;
let aad = PayloadAad::new(id, step_id, EntryKindTag::StepResult, Some(idem_key));
let opened = self.open_payload(&sealed, &aad)?;
let version = u8::try_from(payload_version.unwrap_or(1)).map_err(|_| {
DurableError::Decode {
context: "payload_version out of u8 range",
}
})?;
EntryKind::StepResult {
idempotency_key: idem_key,
payload: opened,
effect,
payload_version: version,
}
}
"effect_intent" => {
let idem_bytes = idem_key.ok_or(DurableError::Decode {
context: "effect_intent idem_key missing",
})?;
let idem_key = IdempotencyKey::from_bytes(slice_to_array32(
&idem_bytes,
"effect_intent idem_key",
)?);
let effect = effect_class
.as_deref()
.and_then(crate::EffectClass::from_tag)
.ok_or(DurableError::Decode {
context: "effect_intent effect_class missing or invalid",
})?;
let hmac = hmac
.map(|bytes| slice_to_array32(&bytes, "effect_intent hmac"))
.transpose()?;
self.verify_control_hmac(
id,
step_id,
EntryKindTag::EffectIntent.as_str(),
Some(&idem_key),
hmac,
)?;
EntryKind::EffectIntent {
idempotency_key: idem_key,
effect,
hmac,
}
}
"checkpoint" => self.checkpoint_entry(id, step_id, payload)?,
other => {
return Err(DurableError::UnsupportedEntryKind {
kind: static_entry_tag(other),
});
}
};
Ok(JournalEntry {
seq: Some(JournalSeq::new(seq)),
execution_id: id,
kind,
step_id,
entry,
created_at_ms: created_at,
})
}
fn checkpoint_entry(
&self,
id: ExecutionId,
step_id: StepId,
payload: Option<Vec<u8>>,
) -> Result<EntryKind, DurableError> {
let sealed = payload.ok_or(DurableError::Decode {
context: "checkpoint entry missing snapshot payload",
})?;
ensure_payload_within_limit(
sealed.len(),
self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
)?;
let aad = PayloadAad::new(id, step_id, EntryKindTag::Checkpoint, None);
let snapshot = self.open_payload(&sealed, &aad)?;
Ok(EntryKind::Checkpoint {
up_to_step: step_id.value(),
snapshot,
})
}
async fn rows_to_entries(
&self,
id: ExecutionId,
rows: Vec<JournalRowRead>,
) -> Result<Vec<JournalEntry>, DurableError> {
if rows.is_empty() {
return Ok(Vec::new());
}
let kind = self.lookup_kind(id).await?;
let mut entries = Vec::with_capacity(rows.len());
for row in rows {
entries.push(self.row_to_entry(id, kind, row)?);
}
Ok(entries)
}
}
impl Journal for LocalBackend {
async fn append(&self, entry: JournalEntry) -> Result<JournalSeq, DurableError> {
let span = tracing::info_span!(
"durable.journal.append",
execution_id = %entry.execution_id.as_uuid(),
step_id = entry.step_id.value(),
entry_kind = entry.entry.tag(),
);
async move {
let row = self.prepare_row(&entry)?;
let (seq,): (i64,) = zeph_db::query_as(sql!(
"INSERT INTO durable_journal
(execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
RETURNING seq"
))
.bind(row.execution_id)
.bind(row.step_id)
.bind(row.entry_kind)
.bind(row.idem_key)
.bind(row.effect_class)
.bind(row.payload)
.bind(row.payload_version)
.bind(row.hmac)
.bind(row.created_at)
.fetch_one(&self.pool)
.await
.map_err(|e| DurableError::storage("append", e))?;
Ok(JournalSeq::new(seq))
}
.instrument(span)
.await
}
async fn read_execution(&self, id: ExecutionId) -> Result<Vec<JournalEntry>, DurableError> {
let span = tracing::info_span!(
"durable.journal.read",
execution_id = %id.as_uuid(),
step_count = tracing::field::Empty,
);
async move {
let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
"SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
FROM durable_journal WHERE execution_id = ? ORDER BY seq"
))
.bind(id.as_uuid().to_string())
.fetch_all(&self.pool)
.await
.map_err(|e| DurableError::storage("read", e))?;
let entries = self.rows_to_entries(id, rows).await?;
tracing::Span::current().record("step_count", entries.len());
Ok(entries)
}
.instrument(span)
.await
}
async fn read_execution_range(
&self,
id: ExecutionId,
from_step_id: u32,
limit: usize,
) -> Result<Vec<JournalEntry>, DurableError> {
let span = tracing::info_span!(
"durable.journal.read_segment",
execution_id = %id.as_uuid(),
from_step_id,
count = tracing::field::Empty,
);
async move {
let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
"SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
FROM durable_journal WHERE execution_id = ? AND step_id >= ? ORDER BY step_id, seq LIMIT ?"
))
.bind(id.as_uuid().to_string())
.bind(i64::from(from_step_id))
.bind(i64::try_from(limit).unwrap_or(i64::MAX))
.fetch_all(&self.pool)
.await
.map_err(|e| DurableError::storage("read_segment", e))?;
let entries = self.rows_to_entries(id, rows).await?;
tracing::Span::current().record("count", entries.len());
Ok(entries)
}
.instrument(span)
.await
}
async fn finalize(&self, id: ExecutionId, status: ExecutionStatus) -> Result<(), DurableError> {
let span = tracing::info_span!(
"durable.journal.finalize",
execution_id = %id.as_uuid(),
status = status.as_str(),
);
async move {
let now = now_unix_millis();
let finalized_at = (!status.is_running()).then_some(now);
let mut tx = zeph_db::begin_write(&self.pool)
.await
.map_err(|e| DurableError::storage("finalize", e))?;
zeph_db::query(sql!(
"UPDATE durable_executions SET status = ?, updated_at = ?, finalized_at = ?
WHERE execution_id = ? AND status = 'running'"
))
.bind(status.as_str())
.bind(now)
.bind(finalized_at)
.bind(id.as_uuid().to_string())
.execute(&mut *tx)
.await
.map_err(|e| DurableError::storage("finalize", e))?;
tx.commit()
.await
.map_err(|e| DurableError::storage("finalize", e))?;
Ok(())
}
.instrument(span)
.await
}
async fn prune(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
let now = now_unix_millis();
crate::retention::prune_in_batches(policy, now, |cutoffs, batch| {
self.delete_prune_batch(cutoffs, batch)
})
.await
}
async fn sweep_orphans(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
if policy.stale_running_after_secs == 0 {
return Ok(0);
}
let Some(lock_dir) = self.lock_dir.clone() else {
if !self
.orphan_sweep_warned
.swap(true, std::sync::atomic::Ordering::Relaxed)
{
tracing::warn!(
"durable: crash-orphan sweep requires an on-disk advisory-lock dir; orphan \
reclamation disabled for this backend (Postgres/:memory:/non-Unix)"
);
}
return Ok(0);
};
let cutoff_ms = orphan_cutoff_ms(policy, now_unix_millis());
crate::retention::sweep_orphans_in_batches(
policy.prune_batch_size,
cutoff_ms,
|cutoff, batch, cursor| self.sweep_orphan_batch(&lock_dir, cutoff, batch, cursor),
)
.await
}
}
impl crate::sealed::Sealed for LocalBackend {}
impl ExecutionBackend for LocalBackend {
fn capabilities(&self) -> BackendCapabilities {
BackendCapabilities {
parallel_steps: true,
cross_process: cfg!(feature = "postgres"),
max_payload: usize::try_from(self.max_payload_bytes).unwrap_or(usize::MAX),
}
}
async fn lookup_committed_result(
&self,
id: ExecutionId,
idem_key: IdempotencyKey,
) -> Result<Option<JournalEntry>, DurableError> {
LocalBackend::lookup_committed_result(self, id, idem_key).await
}
}
struct JournalRow {
execution_id: String,
step_id: i64,
entry_kind: &'static str,
idem_key: Option<Vec<u8>>,
effect_class: Option<&'static str>,
payload: Option<Vec<u8>>,
payload_version: Option<i32>,
hmac: Option<Vec<u8>>,
created_at: i64,
}
type JournalRowRead = (
i64,
i64,
String,
Option<Vec<u8>>,
Option<String>,
Option<Vec<u8>>,
Option<i32>,
Option<Vec<u8>>,
i64,
);
type PromiseRowRead = (String, Vec<u8>, i64, Option<Vec<u8>>);
type FoldableRowRead = (i64, Option<Vec<u8>>, Option<i32>, Option<Vec<u8>>);
#[cfg(feature = "sqlite")]
fn lock_dir_for_path(path: &str) -> Option<std::path::PathBuf> {
(path != ":memory:").then(|| std::path::PathBuf::from(format!("{path}.locks")))
}
#[cfg(not(feature = "sqlite"))]
fn lock_dir_for_path(_path: &str) -> Option<std::path::PathBuf> {
None
}
#[cfg(all(test, not(feature = "sqlite")))]
mod postgres_lock_dir_tests {
use super::lock_dir_for_path;
#[test]
fn postgres_url_never_derives_a_lock_dir() {
assert_eq!(
lock_dir_for_path("postgres://user:secret@host/db"),
None,
"a Postgres connection URL (which may embed credentials) must never be used to mint \
an on-disk lock directory name"
);
assert_eq!(lock_dir_for_path(":memory:"), None);
}
}
pub(crate) fn now_unix_millis() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX))
}
fn orphan_cutoff_ms(policy: &RetentionPolicy, now_ms: i64) -> i64 {
let threshold =
i64::try_from(policy.stale_running_after_secs.saturating_mul(1000)).unwrap_or(i64::MAX);
now_ms.saturating_sub(threshold)
}
fn slice_to_array32(bytes: &[u8], field: &'static str) -> Result<[u8; 32], DurableError> {
<[u8; 32]>::try_from(bytes).map_err(|_| DurableError::Decode { context: field })
}
fn parse_execution_id(text: &str) -> Result<ExecutionId, DurableError> {
uuid::Uuid::parse_str(text)
.map(ExecutionId::from_uuid)
.map_err(|_| DurableError::Decode {
context: "execution_id is not a valid UUID",
})
}
fn parse_timer_id(text: &str) -> Result<TimerId, DurableError> {
uuid::Uuid::parse_str(text)
.map(TimerId::from_uuid)
.map_err(|_| DurableError::Decode {
context: "timer_id is not a valid UUID",
})
}
fn promise_payload_aad(execution_id: ExecutionId, promise_id: PromiseId) -> PayloadAad {
let binding = IdempotencyKey::derive(
execution_id,
StepId::new(0),
promise_id.as_uuid().as_bytes(),
);
PayloadAad::new(
execution_id,
StepId::new(0),
EntryKindTag::PromiseResolved,
Some(binding),
)
}
fn static_entry_tag(tag: &str) -> &'static str {
match tag {
"promise_created" => "promise_created",
"promise_resolved" => "promise_resolved",
"timer_armed" => "timer_armed",
"timer_fired" => "timer_fired",
"checkpoint" => "checkpoint",
_ => "unknown",
}
}
#[cfg(all(test, feature = "sqlite"))]
mod tests {
use std::assert_matches;
use super::*;
use crate::cipher::CipherError;
use crate::effect::EffectClass;
struct XorCipher;
const XOR_MASK: u8 = 0x5A;
impl PayloadCipher for XorCipher {
fn seal(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
let tag = blake3::hash(&aad.canonical_bytes());
let mut out = tag.as_bytes()[..8].to_vec();
out.extend(plaintext.iter().map(|b| b ^ XOR_MASK));
Ok(out)
}
fn open(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
if sealed.len() < 8 {
return Err(CipherError::Malformed {
context: "sealed blob shorter than the aad tag",
});
}
let expected = blake3::hash(&aad.canonical_bytes());
if sealed[..8] != expected.as_bytes()[..8] {
return Err(CipherError::Authentication);
}
Ok(sealed[8..].iter().map(|b| b ^ XOR_MASK).collect())
}
}
async fn mem_backend(max_payload_bytes: u64) -> LocalBackend {
let backend = LocalBackend::open(":memory:", max_payload_bytes)
.await
.expect("open in-memory backend");
backend.init().await.expect("apply migrations");
backend
}
fn step_result(exec: ExecutionId, step: u32, payload: &[u8]) -> JournalEntry {
let step_id = StepId::new(step);
JournalEntry {
seq: None,
execution_id: exec,
kind: ExecutionKind::AgentTurn,
step_id,
entry: EntryKind::StepResult {
idempotency_key: IdempotencyKey::derive(exec, step_id, b"tool:read"),
payload: Bytes::copy_from_slice(payload),
effect: EffectClass::Idempotent,
payload_version: 1,
},
created_at_ms: 100,
}
}
fn effect_intent(exec: ExecutionId, step: u32) -> JournalEntry {
let step_id = StepId::new(step);
JournalEntry {
seq: None,
execution_id: exec,
kind: ExecutionKind::AgentTurn,
step_id,
entry: EntryKind::EffectIntent {
idempotency_key: IdempotencyKey::derive(exec, step_id, b"transfer"),
effect: EffectClass::ExactlyOnceGuarded,
hmac: None,
},
created_at_ms: 100,
}
}
#[tokio::test]
async fn open_execution_is_fresh_then_resume() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
assert!(
!backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap()
);
assert!(
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap()
);
}
#[tokio::test]
async fn open_execution_exclusive_is_fresh_then_resume() {
let dir = tempfile::tempdir().unwrap();
let db_path = dir.path().join("durable.db");
let backend = LocalBackend::open(&db_path.to_string_lossy(), 1_048_576)
.await
.unwrap();
backend.init().await.unwrap();
let exec = ExecutionId::new();
let (is_resume, lock) = backend
.open_execution_exclusive(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
assert!(!is_resume);
assert!(lock.is_some(), "a file-backed backend must derive a lock");
drop(lock);
let (is_resume, _lock) = backend
.open_execution_exclusive(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
assert!(is_resume);
}
#[tokio::test]
async fn open_execution_exclusive_rejects_concurrent_second_holder() {
let dir = tempfile::tempdir().unwrap();
let db_path = dir.path().join("durable.db");
let url = db_path.to_string_lossy().into_owned();
let backend_a = LocalBackend::open(&url, 1_048_576).await.unwrap();
backend_a.init().await.unwrap();
let backend_b = LocalBackend::open(&url, 1_048_576).await.unwrap();
let exec = ExecutionId::new();
let (_, _lock_a) = backend_a
.open_execution_exclusive(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
let err = backend_b
.open_execution_exclusive(exec, ExecutionKind::AgentTurn)
.await
.expect_err("a second concurrent holder must be rejected");
assert!(
matches!(err, DurableError::ExecutionLocked { execution_id, .. } if execution_id == exec),
"expected ExecutionLocked, got {err:?}"
);
}
#[tokio::test]
async fn open_execution_exclusive_on_memory_backend_returns_no_lock() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
let (is_resume, lock) = backend
.open_execution_exclusive(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
assert!(!is_resume);
assert!(lock.is_none());
}
#[tokio::test]
async fn list_executions_summarizes_and_filters() {
let backend = mem_backend(1_048_576).await;
let turn = ExecutionId::new();
let dag = ExecutionId::new();
backend
.open_execution(turn, ExecutionKind::AgentTurn)
.await
.unwrap();
backend
.open_execution(dag, ExecutionKind::DagRun)
.await
.unwrap();
backend.append(step_result(turn, 0, b"a")).await.unwrap();
backend.append(step_result(turn, 1, b"b")).await.unwrap();
backend.append(step_result(dag, 0, b"c")).await.unwrap();
backend
.finalize(turn, ExecutionStatus::Completed)
.await
.unwrap();
let all = backend.list_executions(None, None, 10).await.unwrap();
assert_eq!(all.len(), 2);
let turn_row = all
.iter()
.find(|e| e.execution_id == turn)
.expect("turn present");
assert_eq!(turn_row.kind, "agent_turn");
assert_eq!(turn_row.status, ExecutionStatus::Completed);
assert_eq!(turn_row.step_count, 2);
assert!(turn_row.finalized_at_ms.is_some());
let dag_row = all
.iter()
.find(|e| e.execution_id == dag)
.expect("dag present");
assert_eq!(dag_row.status, ExecutionStatus::Running);
assert_eq!(dag_row.step_count, 1);
assert!(dag_row.finalized_at_ms.is_none());
let running = backend
.list_executions(Some("running"), None, 10)
.await
.unwrap();
assert_eq!(running.len(), 1);
assert_eq!(running[0].execution_id, dag);
let dags = backend
.list_executions(None, Some("dag_run"), 10)
.await
.unwrap();
assert_eq!(dags.len(), 1);
assert_eq!(dags[0].execution_id, dag);
let one = backend.list_executions(None, None, 1).await.unwrap();
assert_eq!(one.len(), 1);
}
#[tokio::test]
async fn append_and_read_round_trips_step_result() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
let seq = backend
.append(step_result(exec, 0, b"hello"))
.await
.unwrap();
assert_eq!(seq.value(), 1, "first append takes seq 1");
let entries = backend.read_execution(exec).await.unwrap();
assert_eq!(entries.len(), 1);
match &entries[0].entry {
EntryKind::StepResult {
payload, effect, ..
} => {
assert_eq!(payload.as_ref(), b"hello");
assert_eq!(*effect, EffectClass::Idempotent);
}
other => panic!("unexpected entry kind: {other:?}"),
}
assert_eq!(entries[0].seq, Some(seq));
}
#[tokio::test]
async fn cipher_seals_payload_at_rest_but_round_trips() {
let backend = mem_backend(1_048_576)
.await
.with_cipher(Arc::new(XorCipher));
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backend
.append(step_result(exec, 0, b"secret-payload"))
.await
.unwrap();
let (stored,): (Option<Vec<u8>>,) = zeph_db::query_as(sql!(
"SELECT payload FROM durable_journal WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.fetch_one(backend.pool())
.await
.unwrap();
let stored = stored.expect("payload present");
assert_ne!(
stored.as_slice(),
b"secret-payload",
"payload must be sealed at rest"
);
let entries = backend.read_execution(exec).await.unwrap();
match &entries[0].entry {
EntryKind::StepResult { payload, .. } => {
assert_eq!(payload.as_ref(), b"secret-payload");
}
other => panic!("unexpected entry kind: {other:?}"),
}
}
#[tokio::test]
async fn control_entry_hmac_is_stamped_only_when_keyed() {
let exec = ExecutionId::new();
let unkeyed = mem_backend(1_048_576).await;
unkeyed
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
unkeyed.append(effect_intent(exec, 0)).await.unwrap();
match &unkeyed.read_execution(exec).await.unwrap()[0].entry {
EntryKind::EffectIntent { hmac, .. } => assert!(hmac.is_none()),
other => panic!("unexpected entry kind: {other:?}"),
}
let keyed = mem_backend(1_048_576).await.with_hmac_key([7u8; 32]);
let exec2 = ExecutionId::new();
keyed
.open_execution(exec2, ExecutionKind::AgentTurn)
.await
.unwrap();
keyed.append(effect_intent(exec2, 0)).await.unwrap();
match &keyed.read_execution(exec2).await.unwrap()[0].entry {
EntryKind::EffectIntent { hmac, .. } => {
assert!(
hmac.is_some(),
"keyed backend stamps a row HMAC over control entries"
);
}
other => panic!("unexpected entry kind: {other:?}"),
}
}
#[tokio::test]
async fn read_execution_rejects_control_hmac_under_wrong_key() {
let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
let exec = ExecutionId::new();
writer
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
writer.append(effect_intent(exec, 0)).await.unwrap();
let wrong_key_reader =
LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([2u8; 32]);
assert_matches!(
wrong_key_reader.read_execution(exec).await,
Err(DurableError::ControlIntegrity)
);
let right_key_reader =
LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([1u8; 32]);
assert!(right_key_reader.read_execution(exec).await.is_ok());
}
#[tokio::test]
async fn read_execution_rejects_missing_hmac_on_keyed_backend() {
let writer = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
writer
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
writer.append(effect_intent(exec, 0)).await.unwrap();
let keyed_reader =
LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([3u8; 32]);
assert_matches!(
keyed_reader.read_execution(exec).await,
Err(DurableError::ControlIntegrity)
);
}
#[tokio::test]
async fn read_execution_rejects_stamped_hmac_on_unkeyed_backend() {
let writer = mem_backend(1_048_576).await.with_hmac_key([4u8; 32]);
let exec = ExecutionId::new();
writer
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
writer.append(effect_intent(exec, 0)).await.unwrap();
let unkeyed_reader = LocalBackend::new(writer.pool().clone(), 1_048_576);
assert_matches!(
unkeyed_reader.read_execution(exec).await,
Err(DurableError::ControlIntegrity)
);
}
#[tokio::test]
async fn promise_and_timer_entries_fail_closed() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
let timer = JournalEntry {
seq: None,
execution_id: exec,
kind: ExecutionKind::AgentTurn,
step_id: StepId::new(0),
entry: EntryKind::TimerArmed {
timer_id: crate::TimerId::new(),
due_at_ms: 1_000,
hmac: None,
},
created_at_ms: 0,
};
assert_matches!(
backend.append(timer).await,
Err(DurableError::UnsupportedEntryKind {
kind: "timer_armed"
})
);
}
#[tokio::test]
async fn payload_over_limit_is_rejected_fail_closed() {
let backend = mem_backend(8).await;
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
let big = vec![0u8; 64];
assert_matches!(
backend.append(step_result(exec, 0, &big)).await,
Err(DurableError::PayloadTooLarge { .. })
);
}
#[tokio::test]
async fn finalize_marks_terminal_status_and_time() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backend
.finalize(exec, ExecutionStatus::Completed)
.await
.unwrap();
let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
"SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.fetch_one(backend.pool())
.await
.unwrap();
assert_eq!(status, "completed");
assert!(finalized.is_some(), "a terminal status stamps finalized_at");
}
#[tokio::test]
async fn finalize_is_a_noop_once_already_terminal() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backend
.finalize(exec, ExecutionStatus::Completed)
.await
.unwrap();
backend
.finalize(exec, ExecutionStatus::Failed)
.await
.unwrap();
let (status,): (String,) = zeph_db::query_as(sql!(
"SELECT status FROM durable_executions WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.fetch_one(backend.pool())
.await
.unwrap();
assert_eq!(
status, "completed",
"the first terminal status must stick; a later finalize call is a no-op"
);
}
#[tokio::test]
async fn finalize_after_abort_is_a_noop() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backend
.finalize(exec, ExecutionStatus::Aborted)
.await
.unwrap();
backend
.finalize(exec, ExecutionStatus::Completed)
.await
.unwrap();
let (status,): (String,) = zeph_db::query_as(sql!(
"SELECT status FROM durable_executions WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.fetch_one(backend.pool())
.await
.unwrap();
assert_eq!(
status, "aborted",
"an aborted execution must not be overwritten by a later Completed/Failed call"
);
}
#[tokio::test]
async fn reopening_a_finalized_execution_resets_it_to_running() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backend
.finalize(exec, ExecutionStatus::Completed)
.await
.unwrap();
let is_resume = backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
assert!(is_resume, "the row already existed, so this is a resume");
let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
"SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.fetch_one(backend.pool())
.await
.unwrap();
assert_eq!(
status, "running",
"reopening a completed execution must un-finalize it"
);
assert!(
finalized.is_none(),
"reopening must clear the stale finalized_at"
);
}
#[tokio::test]
async fn reopening_a_failed_execution_resets_it_to_running() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backend
.finalize(exec, ExecutionStatus::Failed)
.await
.unwrap();
let is_resume = backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
assert!(is_resume, "the row already existed, so this is a resume");
let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
"SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.fetch_one(backend.pool())
.await
.unwrap();
assert_eq!(
status, "running",
"reopening a failed execution must un-finalize it"
);
assert!(
finalized.is_none(),
"reopening must clear the stale finalized_at"
);
}
#[tokio::test]
async fn reopening_an_aborted_execution_un_finalizes_it() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backend
.finalize(exec, ExecutionStatus::Aborted)
.await
.unwrap();
let is_resume = backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
assert!(is_resume, "the row already existed, so this is a resume");
let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
"SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.fetch_one(backend.pool())
.await
.unwrap();
assert_eq!(
status, "running",
"reopening an aborted execution must un-finalize it (INV-16)"
);
assert!(
finalized.is_none(),
"reopening must clear the stale finalized_at"
);
}
#[tokio::test]
async fn reopen_of_a_row_deleted_out_from_under_it_starts_fresh() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backend
.finalize(exec, ExecutionStatus::Completed)
.await
.unwrap();
zeph_db::query(sql!(
"DELETE FROM durable_executions WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.execute(backend.pool())
.await
.unwrap();
let is_resume = backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
assert!(
!is_resume,
"a row deleted by a concurrent prune must be reported as a fresh execution, not a resume"
);
let (status,): (String,) = zeph_db::query_as(sql!(
"SELECT status FROM durable_executions WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.fetch_one(backend.pool())
.await
.unwrap();
assert_eq!(status, "running", "the fresh row starts running");
}
#[tokio::test]
async fn prune_does_not_delete_a_row_reopened_since_it_was_finalized() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backend.append(step_result(exec, 0, b"x")).await.unwrap();
zeph_db::query(sql!(
"UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.execute(backend.pool())
.await
.unwrap();
let is_resume = backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
assert!(is_resume);
let policy = RetentionPolicy {
ttl_completed_secs: 1,
prune_batch_size: 10,
..RetentionPolicy::default()
};
let deleted = backend.prune(&policy).await.unwrap();
assert_eq!(
deleted, 0,
"a reopened (un-finalized) execution must not be pruned"
);
assert_eq!(
backend.read_execution(exec).await.unwrap().len(),
1,
"the execution's journal must survive"
);
}
#[tokio::test]
async fn concurrent_prune_and_reopen_never_lose_or_corrupt_the_row() {
let dir = tempfile::tempdir().unwrap();
let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
backend.init().await.unwrap();
let policy = RetentionPolicy {
ttl_completed_secs: 1,
prune_batch_size: 10,
..RetentionPolicy::default()
};
for _ in 0..20 {
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backend.append(step_result(exec, 0, b"x")).await.unwrap();
zeph_db::query(sql!(
"UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.execute(backend.pool())
.await
.unwrap();
let reopen_backend = backend.clone();
let reopen = tokio::spawn(async move {
reopen_backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
});
let prune_backend = backend.clone();
let policy_for_task = policy.clone();
let prune = tokio::spawn(async move { prune_backend.prune(&policy_for_task).await });
let (reopen_result, prune_result) = tokio::join!(reopen, prune);
reopen_result
.expect("reopen task must not panic")
.expect("reopen must not error under concurrent prune");
prune_result
.expect("prune task must not panic")
.expect("prune must not error under a concurrent reopen");
let (status,): (String,) = zeph_db::query_as(sql!(
"SELECT status FROM durable_executions WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.fetch_one(backend.pool())
.await
.expect(
"the row must exist under either race outcome — reopened-running, or \
deleted-then-reinserted-fresh-running by reopen's fallback",
);
assert_eq!(
status, "running",
"whichever task wins, the row must end up running — never left completed \
(orphaned from a live journal) or absent"
);
}
}
#[tokio::test]
async fn max_seq_reflects_committed_appends() {
let backend = mem_backend(1_048_576).await;
assert_eq!(
backend.max_seq().await.unwrap(),
None,
"empty journal has no max seq"
);
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
for step in 0..3 {
backend.append(step_result(exec, step, b"x")).await.unwrap();
}
assert_eq!(backend.max_seq().await.unwrap(), Some(JournalSeq::new(3)));
}
#[tokio::test]
async fn append_batch_group_commits_every_entry() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
let batch = vec![
step_result(exec, 0, b"a"),
step_result(exec, 1, b"b"),
step_result(exec, 2, b"c"),
];
backend.append_batch(&batch).await.unwrap();
assert_eq!(backend.read_execution(exec).await.unwrap().len(), 3);
}
#[tokio::test]
async fn read_execution_range_bounds_the_segment() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
for step in 0..5 {
backend.append(step_result(exec, step, b"x")).await.unwrap();
}
let segment = backend.read_execution_range(exec, 2, 2).await.unwrap();
assert_eq!(segment.len(), 2);
assert_eq!(segment[0].step_id, StepId::new(2));
assert_eq!(segment[1].step_id, StepId::new(3));
}
#[tokio::test]
async fn lookup_committed_result_finds_by_idem_key() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
let entry = step_result(exec, 0, b"committed");
let idem_key = match &entry.entry {
EntryKind::StepResult {
idempotency_key, ..
} => *idempotency_key,
other => panic!("unexpected entry kind: {other:?}"),
};
backend.append(entry).await.unwrap();
let found = backend
.lookup_committed_result(exec, idem_key)
.await
.unwrap()
.expect("committed result is located by its idempotency key");
match &found.entry {
EntryKind::StepResult { payload, .. } => assert_eq!(payload.as_ref(), b"committed"),
other => panic!("unexpected entry kind: {other:?}"),
}
let absent = IdempotencyKey::derive(exec, StepId::new(99), b"never");
assert!(
backend
.lookup_committed_result(exec, absent)
.await
.unwrap()
.is_none()
);
}
#[tokio::test]
async fn capabilities_describe_the_local_profile() {
let backend = mem_backend(4096).await;
let caps = backend.capabilities();
assert!(caps.parallel_steps);
assert!(
!caps.cross_process,
"the SQLite local backend is in-process"
);
assert_eq!(caps.max_payload, 4096);
}
#[tokio::test]
async fn promise_insert_state_and_resolve_round_trip() {
let backend = mem_backend(1_048_576)
.await
.with_cipher(Arc::new(XorCipher));
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
let promise = PromiseId::derive(exec, StepId::new(0));
backend
.insert_promise(promise, exec, [9u8; 32], 100)
.await
.unwrap();
let pending = backend.promise_state(promise).await.unwrap().unwrap();
assert!(!pending.resolved);
assert_eq!(pending.execution_id, exec);
assert_eq!(pending.resolver_token_hash, [9u8; 32]);
assert!(
backend
.resolve_promise(promise, exec, b"answer", 200)
.await
.unwrap()
);
assert!(
!backend
.resolve_promise(promise, exec, b"again", 300)
.await
.unwrap()
);
let resolved = backend.promise_state(promise).await.unwrap().unwrap();
assert!(resolved.resolved);
let sealed = resolved.payload.expect("resolved payload present");
assert_ne!(sealed.as_slice(), b"answer", "payload is sealed at rest");
let opened = backend
.open_promise_payload(promise, exec, &sealed)
.unwrap();
assert_eq!(opened.as_ref(), b"answer");
}
#[tokio::test]
async fn claim_promise_notification_is_single_winner() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
let promise = PromiseId::derive(exec, StepId::new(0));
backend
.insert_promise(promise, exec, [9u8; 32], 100)
.await
.unwrap();
assert!(
backend
.claim_promise_notification(promise, 200)
.await
.unwrap()
);
assert!(
!backend
.claim_promise_notification(promise, 300)
.await
.unwrap()
);
}
#[tokio::test]
async fn timer_arm_due_and_fire() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
let past = TimerId::derive(exec, StepId::new(0));
let future = TimerId::derive(exec, StepId::new(1));
backend.arm_timer(past, exec, 1_000, 0).await.unwrap();
backend
.arm_timer(future, exec, 9_000_000_000_000, 0)
.await
.unwrap();
let due = backend.due_timers(5_000).await.unwrap();
assert_eq!(due, vec![past]);
assert!(backend.mark_timer_fired(past).await.unwrap());
assert!(
!backend.mark_timer_fired(past).await.unwrap(),
"second fire is a no-op"
);
assert_eq!(
backend.timer_state(past).await.unwrap(),
Some((1_000, true))
);
assert!(backend.due_timers(5_000).await.unwrap().is_empty());
}
#[tokio::test]
async fn prune_deletes_terminal_executions_past_ttl() {
let backend = mem_backend(1_048_576).await;
let old = ExecutionId::new();
backend
.open_execution(old, ExecutionKind::AgentTurn)
.await
.unwrap();
backend.append(step_result(old, 0, b"x")).await.unwrap();
zeph_db::query(sql!(
"UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
))
.bind(old.as_uuid().to_string())
.execute(backend.pool())
.await
.unwrap();
let live = ExecutionId::new();
backend
.open_execution(live, ExecutionKind::AgentTurn)
.await
.unwrap();
backend.append(step_result(live, 0, b"y")).await.unwrap();
let policy = RetentionPolicy {
ttl_completed_secs: 1,
prune_batch_size: 10,
..RetentionPolicy::default()
};
let deleted = backend.prune(&policy).await.unwrap();
assert_eq!(deleted, 1, "only the aged terminal execution is pruned");
assert!(backend.read_execution(old).await.unwrap().is_empty());
assert!(
backend
.promise_state(PromiseId::derive(old, StepId::new(0)))
.await
.unwrap()
.is_none()
);
assert_eq!(backend.read_execution(live).await.unwrap().len(), 1);
}
async fn backdate_updated_at(backend: &LocalBackend, id: ExecutionId, updated_at_ms: i64) {
zeph_db::query(sql!(
"UPDATE durable_executions SET updated_at = ? WHERE execution_id = ?"
))
.bind(updated_at_ms)
.bind(id.as_uuid().to_string())
.execute(backend.pool())
.await
.unwrap();
}
#[tokio::test]
async fn sweep_orphans_disabled_when_threshold_is_zero() {
let dir = tempfile::tempdir().unwrap();
let backend =
LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
.await
.unwrap();
backend.init().await.unwrap();
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backdate_updated_at(&backend, exec, 0).await;
let policy = RetentionPolicy {
stale_running_after_secs: 0,
..RetentionPolicy::default()
};
let aborted = backend.sweep_orphans(&policy).await.unwrap();
assert_eq!(
aborted, 0,
"stale_running_after_secs = 0 disables the sweep"
);
let (status,): (String,) = zeph_db::query_as(sql!(
"SELECT status FROM durable_executions WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.fetch_one(backend.pool())
.await
.unwrap();
assert_eq!(status, "running");
}
#[tokio::test]
async fn sweep_orphans_is_a_documented_no_op_on_memory_backend() {
let backend = mem_backend(1_048_576).await;
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backdate_updated_at(&backend, exec, 0).await;
let policy = RetentionPolicy {
stale_running_after_secs: 1,
..RetentionPolicy::default()
};
let aborted = backend.sweep_orphans(&policy).await.unwrap();
assert_eq!(
aborted, 0,
"a lock_dir=None backend must never abort on staleness alone"
);
let (status,): (String,) = zeph_db::query_as(sql!(
"SELECT status FROM durable_executions WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.fetch_one(backend.pool())
.await
.unwrap();
assert_eq!(status, "running");
}
#[tokio::test]
async fn sweep_orphans_aborts_a_stale_running_execution_with_no_live_owner() {
let dir = tempfile::tempdir().unwrap();
let backend =
LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
.await
.unwrap();
backend.init().await.unwrap();
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backdate_updated_at(&backend, exec, 0).await;
let policy = RetentionPolicy {
stale_running_after_secs: 1,
..RetentionPolicy::default()
};
let aborted = backend.sweep_orphans(&policy).await.unwrap();
assert_eq!(aborted, 1);
let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
"SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.fetch_one(backend.pool())
.await
.unwrap();
assert_eq!(status, "aborted");
assert!(finalized.is_some());
}
#[tokio::test]
async fn sweep_orphans_skips_an_execution_whose_lock_is_held_by_a_live_owner() {
let dir = tempfile::tempdir().unwrap();
let db_path = dir.path().join("durable.db");
let url = db_path.to_string_lossy().into_owned();
let owner = LocalBackend::open(&url, 1_048_576).await.unwrap();
owner.init().await.unwrap();
let sweeper = LocalBackend::open(&url, 1_048_576).await.unwrap();
let exec = ExecutionId::new();
let (_, _lock) = owner
.open_execution_exclusive(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backdate_updated_at(&owner, exec, 0).await;
let policy = RetentionPolicy {
stale_running_after_secs: 1,
..RetentionPolicy::default()
};
let aborted = sweeper.sweep_orphans(&policy).await.unwrap();
assert_eq!(aborted, 0, "a live-held lock must never be swept");
let (status,): (String,) = zeph_db::query_as(sql!(
"SELECT status FROM durable_executions WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.fetch_one(owner.pool())
.await
.unwrap();
assert_eq!(status, "running");
}
#[tokio::test]
async fn sweep_orphans_leaves_a_fresh_running_execution_untouched() {
let dir = tempfile::tempdir().unwrap();
let backend =
LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
.await
.unwrap();
backend.init().await.unwrap();
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
let policy = RetentionPolicy {
stale_running_after_secs: 3600,
..RetentionPolicy::default()
};
let aborted = backend.sweep_orphans(&policy).await.unwrap();
assert_eq!(aborted, 0);
}
#[tokio::test]
async fn count_orphans_matches_sweep_without_mutating() {
let dir = tempfile::tempdir().unwrap();
let backend =
LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
.await
.unwrap();
backend.init().await.unwrap();
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backdate_updated_at(&backend, exec, 0).await;
let policy = RetentionPolicy {
stale_running_after_secs: 1,
..RetentionPolicy::default()
};
let counted = backend.count_orphans(&policy).await.unwrap();
assert_eq!(counted, 1);
let (status,): (String,) = zeph_db::query_as(sql!(
"SELECT status FROM durable_executions WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.fetch_one(backend.pool())
.await
.unwrap();
assert_eq!(status, "running");
let aborted = backend.sweep_orphans(&policy).await.unwrap();
assert_eq!(
aborted, counted,
"sweep must abort exactly what count_orphans counted"
);
}
#[tokio::test]
async fn sweep_orphans_processes_every_batch_when_candidates_straddle_the_batch_size() {
let dir = tempfile::tempdir().unwrap();
let backend =
LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
.await
.unwrap();
backend.init().await.unwrap();
let batch_size = 2u64;
let candidate_count = batch_size + 1; let mut execs = Vec::new();
for _ in 0..candidate_count {
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backdate_updated_at(&backend, exec, 0).await;
execs.push(exec);
}
let policy = RetentionPolicy {
stale_running_after_secs: 1,
prune_batch_size: batch_size,
..RetentionPolicy::default()
};
let aborted = backend.sweep_orphans(&policy).await.unwrap();
assert_eq!(
aborted, candidate_count,
"every candidate must be aborted, including the one past the first batch"
);
for exec in execs {
let (status,): (String,) = zeph_db::query_as(sql!(
"SELECT status FROM durable_executions WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.fetch_one(backend.pool())
.await
.unwrap();
assert_eq!(status, "aborted");
}
}
#[tokio::test]
async fn sweep_orphans_terminates_when_lock_held_candidates_exceed_batch_size() {
let dir = tempfile::tempdir().unwrap();
let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
let owner = LocalBackend::open(&db_url, 1_048_576).await.unwrap();
owner.init().await.unwrap();
let sweeper = LocalBackend::open(&db_url, 1_048_576).await.unwrap();
let batch_size = 2u64;
let candidate_count = batch_size * 2 + 1; let mut locks = Vec::new();
for _ in 0..candidate_count {
let exec = ExecutionId::new();
let (_, lock) = owner
.open_execution_exclusive(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backdate_updated_at(&owner, exec, 0).await;
locks.push(lock); }
let policy = RetentionPolicy {
stale_running_after_secs: 1,
prune_batch_size: batch_size,
..RetentionPolicy::default()
};
let aborted = tokio::time::timeout(
std::time::Duration::from_secs(10),
sweeper.sweep_orphans(&policy),
)
.await
.expect(
"sweep_orphans must terminate even when lock-held candidates exceed prune_batch_size \
(#6254 C1) — it hung instead of returning",
)
.unwrap();
assert_eq!(aborted, 0, "every candidate's lock is held by a live owner");
drop(locks);
}
#[tokio::test]
async fn concurrent_sweep_and_reopen_race_never_corrupts_the_row() {
let dir = tempfile::tempdir().unwrap();
let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
backend.init().await.unwrap();
let policy = RetentionPolicy {
stale_running_after_secs: 1,
prune_batch_size: 10,
..RetentionPolicy::default()
};
for _ in 0..20 {
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
backdate_updated_at(&backend, exec, 0).await;
let sweep_backend = backend.clone();
let policy_for_task = policy.clone();
let sweep =
tokio::spawn(async move { sweep_backend.sweep_orphans(&policy_for_task).await });
let reopen_backend = backend.clone();
let reopen = tokio::spawn(async move {
reopen_backend
.open_execution_exclusive(exec, ExecutionKind::AgentTurn)
.await
});
let (sweep_result, reopen_result) = tokio::join!(sweep, reopen);
let aborted = sweep_result
.expect("sweep task must not panic")
.expect("sweep must not error under a concurrent reopen");
assert!(aborted <= 1, "at most one candidate row exists per trial");
match reopen_result.expect("reopen task must not panic") {
Ok((_is_resume, _lock)) => {
let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
"SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
))
.bind(exec.as_uuid().to_string())
.fetch_one(backend.pool())
.await
.unwrap();
assert_eq!(status, "running");
assert!(finalized.is_none());
backend
.finalize(exec, ExecutionStatus::Completed)
.await
.unwrap();
}
Err(DurableError::ExecutionLocked { .. }) => {
}
Err(e) => panic!(
"reopen must only ever fail with ExecutionLocked under this race, got {e:?}"
),
}
}
}
#[tokio::test]
async fn checkpoint_fold_compacts_idempotent_prefix_and_replays() {
let backend = mem_backend(1_048_576)
.await
.with_cipher(Arc::new(XorCipher));
let exec = ExecutionId::new();
backend
.open_execution(exec, ExecutionKind::AgentTurn)
.await
.unwrap();
for step in 0..5 {
backend
.append(step_result(exec, step, format!("v{step}").as_bytes()))
.await
.unwrap();
}
let folded = backend.checkpoint_fold(exec, 3).await.unwrap();
assert_eq!(folded, 3);
let remaining = backend.read_execution(exec).await.unwrap();
let step_results: Vec<u32> = remaining
.iter()
.filter(|e| matches!(e.entry, EntryKind::StepResult { .. }))
.map(|e| e.step_id.value())
.collect();
assert_eq!(step_results, vec![3, 4], "folded step rows are deleted");
assert!(
remaining
.iter()
.any(|e| matches!(e.entry, EntryKind::Checkpoint { .. })),
"a checkpoint entry replaces the folded prefix"
);
let preloaded = backend.read_checkpoints(exec).await.unwrap();
assert_eq!(preloaded.len(), 3);
for (i, entry) in preloaded.iter().enumerate() {
let step = u32::try_from(i).unwrap();
assert_eq!(entry.step_id, StepId::new(step));
match &entry.entry {
EntryKind::StepResult {
payload,
idempotency_key,
..
} => {
assert_eq!(payload.as_ref(), format!("v{step}").as_bytes());
assert_eq!(
*idempotency_key,
IdempotencyKey::derive(exec, StepId::new(step), b"tool:read")
);
}
other => panic!("unexpected folded entry: {other:?}"),
}
}
}
}