1use std::fmt;
38use std::fmt::Write as _;
39use std::path::PathBuf;
40use std::sync::Arc;
41use std::time::{SystemTime, UNIX_EPOCH};
42
43use bytes::Bytes;
44use zeph_db::{DbPool, sql};
45
46use crate::backend::execution_lock::ExecutionLock;
47use crate::backend::{BackendCapabilities, ExecutionBackend, ExecutionSummary, RedactedEntry};
48use crate::cipher::{EntryKindTag, PayloadAad, PayloadCipher, ensure_payload_within_limit};
49use crate::config::RetentionPolicy;
50use crate::error::DurableError;
51use crate::ids::{
52 ExecutionId, ExecutionKind, IdempotencyKey, JournalSeq, PromiseId, StepId, TimerId,
53};
54use crate::journal::{EntryKind, ExecutionStatus, Journal, JournalEntry};
55use crate::promise::PromiseRecord;
56use crate::retention::{CheckpointSnapshot, FoldedStep, decode_checkpoint, encode_checkpoint};
57use crate::waiters::NotifyRegistry;
58use tracing::Instrument as _;
59
60const SEAL_OVERHEAD_SLACK: u64 = 128;
68
69type ExecutionRow = (String, String, String, i64, i64, Option<i64>, i64);
71
72type RedactedRow = (
74 i64,
75 i64,
76 String,
77 Option<Vec<u8>>,
78 Option<String>,
79 Option<i64>,
80 i64,
81);
82
83fn idem_key_prefix(bytes: &[u8]) -> String {
85 bytes.iter().take(8).fold(String::new(), |mut acc, b| {
86 let _ = write!(acc, "{b:02x}");
87 acc
88 })
89}
90
91pub struct LocalBackend {
109 pool: DbPool,
110 cipher: Option<Arc<dyn PayloadCipher>>,
111 hmac_key: Option<[u8; 32]>,
112 max_payload_bytes: u64,
113 promise_waiters: NotifyRegistry,
115 timer_waiters: NotifyRegistry,
117 lock_dir: Option<PathBuf>,
123 orphan_sweep_warned: std::sync::atomic::AtomicBool,
127}
128
129impl fmt::Debug for LocalBackend {
130 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
132 f.debug_struct("LocalBackend")
133 .field("cipher", &self.cipher.as_ref().map(|_| "<cipher>"))
134 .field("hmac_key", &self.hmac_key.as_ref().map(|_| "<redacted>"))
135 .field("max_payload_bytes", &self.max_payload_bytes)
136 .finish_non_exhaustive()
137 }
138}
139
140impl LocalBackend {
141 #[must_use]
147 pub fn new(pool: DbPool, max_payload_bytes: u64) -> Self {
148 Self {
149 pool,
150 cipher: None,
151 hmac_key: None,
152 max_payload_bytes,
153 promise_waiters: NotifyRegistry::default(),
154 timer_waiters: NotifyRegistry::default(),
155 lock_dir: None,
156 orphan_sweep_warned: std::sync::atomic::AtomicBool::new(false),
157 }
158 }
159
160 pub async fn open(path: &str, max_payload_bytes: u64) -> Result<Self, DurableError> {
174 let pool = zeph_db::DbConfig {
175 url: path.to_string(),
176 pool_size: 5,
177 }
178 .connect()
179 .await
180 .map_err(|e| DurableError::storage("open", e))?;
181 let mut backend = Self::new(pool, max_payload_bytes);
182 backend.lock_dir = lock_dir_for_path(path);
183 Ok(backend)
184 }
185
186 #[must_use]
188 pub fn with_cipher(mut self, cipher: Arc<dyn PayloadCipher>) -> Self {
189 self.cipher = Some(cipher);
190 self
191 }
192
193 #[must_use]
196 pub fn with_hmac_key(mut self, key: [u8; 32]) -> Self {
197 self.hmac_key = Some(key);
198 self
199 }
200
201 #[must_use]
203 pub fn pool(&self) -> &DbPool {
204 &self.pool
205 }
206
207 pub async fn init(&self) -> Result<(), DurableError> {
215 zeph_db::run_migrations(&self.pool)
216 .await
217 .map_err(|e| DurableError::storage("init", e))?;
218 Ok(())
219 }
220
221 pub async fn list_executions(
236 &self,
237 status: Option<&str>,
238 kind: Option<&str>,
239 limit: i64,
240 ) -> Result<Vec<ExecutionSummary>, DurableError> {
241 let span = tracing::info_span!(
242 "durable.backend.list",
243 status = status.unwrap_or("*"),
244 kind = kind.unwrap_or("*"),
245 count = tracing::field::Empty,
246 );
247 async move {
248 let rows: Vec<ExecutionRow> =
252 zeph_db::query_as(sql!(
253 "SELECT
254 e.execution_id,
255 e.kind,
256 e.status,
257 e.created_at,
258 e.updated_at,
259 e.finalized_at,
260 (SELECT COUNT(*) FROM durable_journal j WHERE j.execution_id = e.execution_id)
261 FROM durable_executions e
262 WHERE e.status = COALESCE(?, e.status)
263 AND e.kind = COALESCE(?, e.kind)
264 ORDER BY e.created_at DESC
265 LIMIT ?"
266 ))
267 .bind(status)
268 .bind(kind)
269 .bind(limit)
270 .fetch_all(&self.pool)
271 .await
272 .map_err(|e| DurableError::storage("list", e))?;
273 tracing::Span::current().record("count", rows.len());
274 rows.into_iter()
275 .map(|(id, kind, status, created, updated, finalized, steps)| {
276 Ok(ExecutionSummary {
277 execution_id: parse_execution_id(&id)?,
278 kind,
279 status: ExecutionStatus::from_tag(&status).ok_or(DurableError::Decode {
280 context: "execution status is not a recognized CHECK-constrained value",
281 })?,
282 created_at_ms: created,
283 updated_at_ms: updated,
284 finalized_at_ms: finalized,
285 step_count: steps.max(0).cast_unsigned(),
286 })
287 })
288 .collect()
289 }
290 .instrument(span)
291 .await
292 }
293
294 pub async fn read_execution_redacted(
307 &self,
308 id: ExecutionId,
309 ) -> Result<Vec<RedactedEntry>, DurableError> {
310 let exec = id.as_uuid().to_string();
311 let rows: Vec<RedactedRow> = zeph_db::query_as(sql!(
312 "SELECT seq, step_id, entry_kind, idem_key, effect_class, LENGTH(payload), created_at
313 FROM durable_journal WHERE execution_id = ? ORDER BY seq"
314 ))
315 .bind(&exec)
316 .fetch_all(&self.pool)
317 .await
318 .map_err(|e| DurableError::storage("read_redacted", e))?;
319 Ok(rows
320 .into_iter()
321 .map(
322 |(seq, step, entry_kind, idem, effect_class, payload_len, created)| RedactedEntry {
323 seq,
324 step_id: StepId::new(u32::try_from(step).unwrap_or(0)),
325 entry_kind,
326 effect_class,
327 idem_key_prefix: idem.as_deref().map(idem_key_prefix),
328 payload_len: payload_len.unwrap_or(0).max(0).cast_unsigned(),
329 created_at_ms: created,
330 },
331 )
332 .collect())
333 }
334
335 pub async fn count_prunable(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
344 let cutoffs = crate::retention::PruneCutoffs::from_policy(policy, now_unix_millis());
345 let (count,): (i64,) = zeph_db::query_as(sql!(
346 "SELECT COUNT(*) FROM durable_executions
347 WHERE finalized_at IS NOT NULL
348 AND ( (status = 'completed' AND finalized_at <= ?)
349 OR (status IN ('failed', 'aborted') AND finalized_at <= ?) )"
350 ))
351 .bind(cutoffs.completed_before_ms)
352 .bind(cutoffs.failed_before_ms)
353 .fetch_one(&self.pool)
354 .await
355 .map_err(|e| DurableError::storage("count_prunable", e))?;
356 Ok(count.max(0).cast_unsigned())
357 }
358
359 pub async fn count_orphans(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
372 if policy.stale_running_after_secs == 0 {
373 return Ok(0);
374 }
375 let Some(lock_dir) = self.lock_dir.clone() else {
376 return Ok(0);
377 };
378 let cutoff_ms = orphan_cutoff_ms(policy, now_unix_millis());
379 let candidates: Vec<(String,)> = zeph_db::query_as(sql!(
380 "SELECT execution_id FROM durable_executions WHERE status = 'running' AND updated_at <= ?"
381 ))
382 .bind(cutoff_ms)
383 .fetch_all(&self.pool)
384 .await
385 .map_err(|e| DurableError::storage("count_orphans", e))?;
386 let mut count = 0u64;
387 for (exec_str,) in &candidates {
388 let Ok(execution_id) = parse_execution_id(exec_str) else {
389 continue;
390 };
391 if ExecutionLock::acquire(&lock_dir, execution_id).is_ok() {
392 count += 1;
393 }
394 }
395 Ok(count)
396 }
397
398 pub async fn open_execution(
431 &self,
432 id: ExecutionId,
433 kind: ExecutionKind,
434 ) -> Result<bool, DurableError> {
435 let span = tracing::info_span!(
436 "durable.backend.open",
437 execution_id = %id.as_uuid(),
438 kind = kind.as_str(),
439 is_resume = tracing::field::Empty,
440 );
441 async move {
442 let exec = id.as_uuid().to_string();
443
444 let reopened = zeph_db::query(sql!(
448 "UPDATE durable_executions SET status = 'running', updated_at = ?, finalized_at = NULL
449 WHERE execution_id = ? AND status IN ('completed', 'failed', 'aborted')"
450 ))
451 .bind(now_unix_millis())
452 .bind(&exec)
453 .execute(&self.pool)
454 .await
455 .map_err(|e| DurableError::storage("open", e))?;
456 if reopened.rows_affected() > 0 {
457 tracing::Span::current().record("is_resume", true);
458 return Ok(true);
459 }
460
461 let existing: Option<(String,)> = zeph_db::query_as(sql!(
467 "SELECT status FROM durable_executions WHERE execution_id = ?"
468 ))
469 .bind(&exec)
470 .fetch_optional(&self.pool)
471 .await
472 .map_err(|e| DurableError::storage("open", e))?;
473 if existing.is_some() {
474 tracing::Span::current().record("is_resume", true);
475 return Ok(true);
476 }
477 let now = now_unix_millis();
478 zeph_db::query(sql!(
479 "INSERT INTO durable_executions
480 (execution_id, kind, status, created_at, updated_at, finalized_at)
481 VALUES (?, ?, 'running', ?, ?, NULL)"
482 ))
483 .bind(&exec)
484 .bind(kind.as_str())
485 .bind(now)
486 .bind(now)
487 .execute(&self.pool)
488 .await
489 .map_err(|e| DurableError::storage("open", e))?;
490 tracing::Span::current().record("is_resume", false);
491 Ok(false)
492 }
493 .instrument(span)
494 .await
495 }
496
497 pub async fn open_execution_exclusive(
519 &self,
520 id: ExecutionId,
521 kind: ExecutionKind,
522 ) -> Result<(bool, Option<ExecutionLock>), DurableError> {
523 let lock = self
524 .lock_dir
525 .as_deref()
526 .map(|dir| ExecutionLock::acquire(dir, id))
527 .transpose()?;
528 let is_resume = self.open_execution(id, kind).await?;
529 Ok((is_resume, lock))
530 }
531
532 pub(crate) async fn append_batch(&self, entries: &[JournalEntry]) -> Result<(), DurableError> {
545 if entries.is_empty() {
546 return Ok(());
547 }
548 let mut rows = Vec::with_capacity(entries.len());
549 for entry in entries {
550 rows.push(self.prepare_row(entry)?);
551 }
552 let insert = sql!(
556 "INSERT INTO durable_journal
557 (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
558 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
559 );
560 let mut tx = zeph_db::begin_write(&self.pool)
561 .await
562 .map_err(|e| DurableError::storage("append_batch", e))?;
563 for row in rows {
564 zeph_db::query(insert)
565 .bind(row.execution_id)
566 .bind(row.step_id)
567 .bind(row.entry_kind)
568 .bind(row.idem_key)
569 .bind(row.effect_class)
570 .bind(row.payload)
571 .bind(row.payload_version)
572 .bind(row.hmac)
573 .bind(row.created_at)
574 .execute(&mut *tx)
575 .await
576 .map_err(|e| DurableError::storage("append_batch", e))?;
577 }
578 tx.commit()
579 .await
580 .map_err(|e| DurableError::storage("append_batch", e))?;
581 Ok(())
582 }
583
584 pub(crate) async fn lookup_committed_result(
598 &self,
599 id: ExecutionId,
600 idem_key: IdempotencyKey,
601 ) -> Result<Option<JournalEntry>, DurableError> {
602 let span = tracing::info_span!(
603 "durable.journal.lookup_idem",
604 execution_id = %id.as_uuid(),
605 found = tracing::field::Empty,
606 );
607 async move {
608 let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
609 "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
610 FROM durable_journal
611 WHERE execution_id = ? AND idem_key = ? AND entry_kind = 'step_result'
612 ORDER BY seq LIMIT 1"
613 ))
614 .bind(id.as_uuid().to_string())
615 .bind(idem_key.as_bytes().to_vec())
616 .fetch_all(&self.pool)
617 .await
618 .map_err(|e| DurableError::storage("lookup_idem", e))?;
619 let entry = self.rows_to_entries(id, rows).await?.into_iter().next();
620 tracing::Span::current().record("found", entry.is_some());
621 Ok(entry)
622 }
623 .instrument(span)
624 .await
625 }
626
627 pub(crate) async fn max_seq(&self) -> Result<Option<JournalSeq>, DurableError> {
637 let max: Option<i64> = zeph_db::query_scalar(sql!("SELECT MAX(seq) FROM durable_journal"))
638 .fetch_one(&self.pool)
639 .await
640 .map_err(|e| DurableError::storage("max_seq", e))?;
641 Ok(max.map(JournalSeq::new))
642 }
643
644 pub(crate) fn promise_waiters(&self) -> &NotifyRegistry {
646 &self.promise_waiters
647 }
648
649 pub(crate) fn timer_waiters(&self) -> &NotifyRegistry {
651 &self.timer_waiters
652 }
653
654 pub(crate) async fn insert_promise(
663 &self,
664 id: PromiseId,
665 execution_id: ExecutionId,
666 resolver_token_hash: [u8; 32],
667 created_at_ms: i64,
668 ) -> Result<(), DurableError> {
669 let span = tracing::info_span!("durable.promise.create", promise_id = %id.as_uuid());
670 async move {
671 zeph_db::query(sql!(
672 "INSERT INTO durable_promises
673 (promise_id, execution_id, resolver_token_hash, resolved, payload, created_at, resolved_at)
674 VALUES (?, ?, ?, 0, NULL, ?, NULL)"
675 ))
676 .bind(id.as_uuid().to_string())
677 .bind(execution_id.as_uuid().to_string())
678 .bind(resolver_token_hash.to_vec())
679 .bind(created_at_ms)
680 .execute(&self.pool)
681 .await
682 .map_err(|e| DurableError::storage("insert_promise", e))?;
683 Ok(())
684 }
685 .instrument(span)
686 .await
687 }
688
689 pub(crate) async fn promise_state(
696 &self,
697 id: PromiseId,
698 ) -> Result<Option<PromiseRecord>, DurableError> {
699 let row: Option<PromiseRowRead> = zeph_db::query_as(sql!(
700 "SELECT execution_id, resolver_token_hash, resolved, payload
701 FROM durable_promises WHERE promise_id = ?"
702 ))
703 .bind(id.as_uuid().to_string())
704 .fetch_optional(&self.pool)
705 .await
706 .map_err(|e| DurableError::storage("promise_state", e))?;
707 let Some((exec, hash, resolved, payload)) = row else {
708 return Ok(None);
709 };
710 Ok(Some(PromiseRecord {
711 execution_id: parse_execution_id(&exec)?,
712 resolver_token_hash: slice_to_array32(&hash, "promise resolver_token_hash")?,
713 resolved: resolved != 0,
714 payload,
715 }))
716 }
717
718 pub(crate) async fn resolve_promise(
729 &self,
730 id: PromiseId,
731 execution_id: ExecutionId,
732 value_plaintext: &[u8],
733 resolved_at_ms: i64,
734 ) -> Result<bool, DurableError> {
735 let span = tracing::info_span!("durable.promise.resolve", promise_id = %id.as_uuid());
736 async move {
737 ensure_payload_within_limit(value_plaintext.len(), self.max_payload_bytes)?;
738 let aad = promise_payload_aad(execution_id, id);
739 let sealed = self.seal_payload(value_plaintext, &aad)?;
740 let affected = zeph_db::query(sql!(
741 "UPDATE durable_promises SET resolved = 1, payload = ?, resolved_at = ?
742 WHERE promise_id = ? AND resolved = 0"
743 ))
744 .bind(sealed)
745 .bind(resolved_at_ms)
746 .bind(id.as_uuid().to_string())
747 .execute(&self.pool)
748 .await
749 .map_err(|e| DurableError::storage("resolve_promise", e))?
750 .rows_affected();
751 if affected > 0 {
752 self.promise_waiters.wake(id.as_uuid());
753 }
754 Ok(affected > 0)
755 }
756 .instrument(span)
757 .await
758 }
759
760 pub(crate) async fn claim_promise_notification(
773 &self,
774 id: PromiseId,
775 notified_at_ms: i64,
776 ) -> Result<bool, DurableError> {
777 let span = tracing::info_span!("durable.promise.claim_notify", promise_id = %id.as_uuid());
778 async move {
779 let affected = zeph_db::query(sql!(
780 "UPDATE durable_promises SET notified_at = ?
781 WHERE promise_id = ? AND notified_at IS NULL"
782 ))
783 .bind(notified_at_ms)
784 .bind(id.as_uuid().to_string())
785 .execute(&self.pool)
786 .await
787 .map_err(|e| DurableError::storage("claim_promise_notification", e))?
788 .rows_affected();
789 Ok(affected > 0)
790 }
791 .instrument(span)
792 .await
793 }
794
795 pub(crate) fn open_promise_payload(
802 &self,
803 id: PromiseId,
804 execution_id: ExecutionId,
805 sealed: &[u8],
806 ) -> Result<Bytes, DurableError> {
807 ensure_payload_within_limit(
808 sealed.len(),
809 self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
810 )?;
811 let aad = promise_payload_aad(execution_id, id);
812 self.open_payload(sealed, &aad)
813 }
814
815 pub(crate) async fn arm_timer(
823 &self,
824 id: TimerId,
825 execution_id: ExecutionId,
826 due_at_ms: i64,
827 created_at_ms: i64,
828 ) -> Result<(), DurableError> {
829 let span = tracing::info_span!("durable.timer.arm", timer_id = %id.as_uuid(), due_at_ms);
830 async move {
831 zeph_db::query(sql!(
832 "INSERT INTO durable_timers (timer_id, execution_id, due_at, fired, created_at)
833 VALUES (?, ?, ?, 0, ?)"
834 ))
835 .bind(id.as_uuid().to_string())
836 .bind(execution_id.as_uuid().to_string())
837 .bind(due_at_ms)
838 .bind(created_at_ms)
839 .execute(&self.pool)
840 .await
841 .map_err(|e| DurableError::storage("arm_timer", e))?;
842 Ok(())
843 }
844 .instrument(span)
845 .await
846 }
847
848 pub(crate) async fn timer_state(
854 &self,
855 id: TimerId,
856 ) -> Result<Option<(i64, bool)>, DurableError> {
857 let row: Option<(i64, i64)> = zeph_db::query_as(sql!(
858 "SELECT due_at, fired FROM durable_timers WHERE timer_id = ?"
859 ))
860 .bind(id.as_uuid().to_string())
861 .fetch_optional(&self.pool)
862 .await
863 .map_err(|e| DurableError::storage("timer_state", e))?;
864 Ok(row.map(|(due_at, fired)| (due_at, fired != 0)))
865 }
866
867 pub(crate) async fn due_timers(&self, now_ms: i64) -> Result<Vec<TimerId>, DurableError> {
877 let rows: Vec<(String,)> = zeph_db::query_as(sql!(
878 "SELECT timer_id FROM durable_timers WHERE fired = 0 AND due_at <= ? ORDER BY due_at"
879 ))
880 .bind(now_ms)
881 .fetch_all(&self.pool)
882 .await
883 .map_err(|e| DurableError::storage("due_timers", e))?;
884 rows.into_iter().map(|(id,)| parse_timer_id(&id)).collect()
885 }
886
887 pub(crate) async fn mark_timer_fired(&self, id: TimerId) -> Result<bool, DurableError> {
895 let span = tracing::info_span!("durable.timer.fire", timer_id = %id.as_uuid());
896 async move {
897 let affected = zeph_db::query(sql!(
898 "UPDATE durable_timers SET fired = 1 WHERE timer_id = ? AND fired = 0"
899 ))
900 .bind(id.as_uuid().to_string())
901 .execute(&self.pool)
902 .await
903 .map_err(|e| DurableError::storage("mark_timer_fired", e))?
904 .rows_affected();
905 if affected > 0 {
906 self.timer_waiters.wake(id.as_uuid());
907 }
908 Ok(affected > 0)
909 }
910 .instrument(span)
911 .await
912 }
913
914 fn open_foldable_steps(
920 &self,
921 execution_id: ExecutionId,
922 rows: Vec<FoldableRowRead>,
923 ) -> Result<Vec<FoldedStep>, DurableError> {
924 let mut folded = Vec::with_capacity(rows.len());
925 for (step_raw, idem, version, payload) in rows {
926 let step = u32::try_from(step_raw).map_err(|_| DurableError::Decode {
927 context: "checkpoint step_id out of u32 range",
928 })?;
929 let idem_bytes = idem.ok_or(DurableError::Decode {
930 context: "checkpoint step result missing idem_key",
931 })?;
932 let idem_key =
933 IdempotencyKey::from_bytes(slice_to_array32(&idem_bytes, "checkpoint idem_key")?);
934 let sealed = payload.ok_or(DurableError::Decode {
935 context: "checkpoint step result missing payload",
936 })?;
937 let aad = PayloadAad::new(
938 execution_id,
939 StepId::new(step),
940 EntryKindTag::StepResult,
941 Some(idem_key),
942 );
943 let plaintext = self.open_payload(&sealed, &aad)?;
944 let payload_version =
945 u8::try_from(version.unwrap_or(1)).map_err(|_| DurableError::Decode {
946 context: "checkpoint payload_version out of u8 range",
947 })?;
948 folded.push(FoldedStep {
949 step_id: step,
950 idem_key: *idem_key.as_bytes(),
951 payload_version,
952 payload: plaintext,
953 });
954 }
955 Ok(folded)
956 }
957
958 pub(crate) async fn checkpoint_fold(
972 &self,
973 execution_id: ExecutionId,
974 up_to_step: u32,
975 ) -> Result<u64, DurableError> {
976 let span = tracing::info_span!(
977 "durable.journal.checkpoint",
978 execution_id = %execution_id.as_uuid(),
979 folded_count = tracing::field::Empty,
980 );
981 async move {
982 let exec = execution_id.as_uuid().to_string();
983 let rows: Vec<FoldableRowRead> = zeph_db::query_as(sql!(
984 "SELECT step_id, idem_key, payload_version, payload FROM durable_journal
985 WHERE execution_id = ? AND entry_kind = 'step_result'
986 AND effect_class = 'idempotent' AND step_id < ?
987 ORDER BY step_id"
988 ))
989 .bind(&exec)
990 .bind(i64::from(up_to_step))
991 .fetch_all(&self.pool)
992 .await
993 .map_err(|e| DurableError::storage("checkpoint", e))?;
994 if rows.is_empty() {
995 return Ok(0);
996 }
997
998 let mut folded = self.open_foldable_steps(execution_id, rows)?;
1000 let lens: Vec<usize> = folded.iter().map(|s| s.payload.len()).collect();
1001 let take = crate::retention::fold_prefix_len(
1002 &lens,
1003 crate::retention::checkpoint_budget(self.max_payload_bytes),
1004 );
1005 if take == 0 {
1006 return Ok(0);
1009 }
1010 folded.truncate(take);
1011 let fold_end = folded.last().map_or(up_to_step, |s| s.step_id.saturating_add(1));
1012
1013 let snapshot = encode_checkpoint(&folded);
1014 let snap_aad =
1015 PayloadAad::new(execution_id, StepId::new(fold_end), EntryKindTag::Checkpoint, None);
1016 let sealed_snapshot = self.seal_payload(&snapshot, &snap_aad)?;
1017
1018 let mut tx = zeph_db::begin_write(&self.pool)
1019 .await
1020 .map_err(|e| DurableError::storage("checkpoint", e))?;
1021 zeph_db::query(sql!(
1022 "INSERT INTO durable_journal
1023 (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
1024 VALUES (?, ?, 'checkpoint', NULL, NULL, ?, ?, NULL, ?)"
1025 ))
1026 .bind(&exec)
1027 .bind(i64::from(fold_end))
1028 .bind(sealed_snapshot)
1029 .bind(i32::from(crate::step::PAYLOAD_VERSION))
1030 .bind(now_unix_millis())
1031 .execute(&mut *tx)
1032 .await
1033 .map_err(|e| DurableError::storage("checkpoint", e))?;
1034 zeph_db::query(sql!(
1035 "DELETE FROM durable_journal
1036 WHERE execution_id = ? AND entry_kind = 'step_result'
1037 AND effect_class = 'idempotent' AND step_id < ?"
1038 ))
1039 .bind(&exec)
1040 .bind(i64::from(fold_end))
1041 .execute(&mut *tx)
1042 .await
1043 .map_err(|e| DurableError::storage("checkpoint", e))?;
1044 tx.commit()
1045 .await
1046 .map_err(|e| DurableError::storage("checkpoint", e))?;
1047
1048 let count = folded.len() as u64;
1049 tracing::Span::current().record("folded_count", count);
1050 Ok(count)
1051 }
1052 .instrument(span)
1053 .await
1054 }
1055
1056 pub(crate) async fn read_checkpoints(
1069 &self,
1070 execution_id: ExecutionId,
1071 ) -> Result<Vec<JournalEntry>, DurableError> {
1072 let rows: Vec<(i64, Option<Vec<u8>>)> = zeph_db::query_as(sql!(
1073 "SELECT step_id, payload FROM durable_journal
1074 WHERE execution_id = ? AND entry_kind = 'checkpoint' ORDER BY step_id"
1075 ))
1076 .bind(execution_id.as_uuid().to_string())
1077 .fetch_all(&self.pool)
1078 .await
1079 .map_err(|e| DurableError::storage("read_checkpoints", e))?;
1080 if rows.is_empty() {
1081 return Ok(Vec::new());
1082 }
1083 let mut folded: CheckpointSnapshot = Vec::new();
1084 for (up_to, payload) in rows {
1085 let up_to = u32::try_from(up_to).map_err(|_| DurableError::Decode {
1086 context: "checkpoint up_to_step out of u32 range",
1087 })?;
1088 let sealed = payload.ok_or(DurableError::Decode {
1089 context: "checkpoint entry missing snapshot payload",
1090 })?;
1091 ensure_payload_within_limit(
1092 sealed.len(),
1093 self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1094 )?;
1095 let aad = PayloadAad::new(
1096 execution_id,
1097 StepId::new(up_to),
1098 EntryKindTag::Checkpoint,
1099 None,
1100 );
1101 let plaintext = self.open_payload(&sealed, &aad)?;
1102 folded.extend(decode_checkpoint(&plaintext)?);
1103 }
1104 let kind = self.lookup_kind(execution_id).await?;
1107 let entries = folded
1108 .into_iter()
1109 .map(|step| JournalEntry {
1110 seq: None,
1111 execution_id,
1112 kind,
1113 step_id: StepId::new(step.step_id),
1114 entry: EntryKind::StepResult {
1115 idempotency_key: IdempotencyKey::from_bytes(step.idem_key),
1116 payload: step.payload,
1117 effect: crate::EffectClass::Idempotent,
1118 payload_version: step.payload_version,
1119 },
1120 created_at_ms: 0,
1121 })
1122 .collect();
1123 Ok(entries)
1124 }
1125
1126 async fn delete_prune_batch(
1143 &self,
1144 cutoffs: crate::retention::PruneCutoffs,
1145 batch: u64,
1146 ) -> Result<u64, DurableError> {
1147 let mut tx = zeph_db::begin_write(&self.pool)
1148 .await
1149 .map_err(|e| DurableError::storage("prune", e))?;
1150
1151 #[cfg(feature = "postgres")]
1157 zeph_db::query(sql!(
1158 "SELECT execution_id FROM durable_executions
1159 WHERE finalized_at IS NOT NULL
1160 AND ( (status = 'completed' AND finalized_at <= ?)
1161 OR (status IN ('failed', 'aborted') AND finalized_at <= ?) )
1162 ORDER BY finalized_at LIMIT ?
1163 FOR UPDATE"
1164 ))
1165 .bind(cutoffs.completed_before_ms)
1166 .bind(cutoffs.failed_before_ms)
1167 .bind(i64::try_from(batch).unwrap_or(i64::MAX))
1168 .execute(&mut *tx)
1169 .await
1170 .map_err(|e| DurableError::storage("prune", e))?;
1171
1172 let ids: Vec<(String,)> = zeph_db::query_as(sql!(
1173 "SELECT execution_id FROM durable_executions
1174 WHERE finalized_at IS NOT NULL
1175 AND ( (status = 'completed' AND finalized_at <= ?)
1176 OR (status IN ('failed', 'aborted') AND finalized_at <= ?) )
1177 ORDER BY finalized_at LIMIT ?"
1178 ))
1179 .bind(cutoffs.completed_before_ms)
1180 .bind(cutoffs.failed_before_ms)
1181 .bind(i64::try_from(batch).unwrap_or(i64::MAX))
1182 .fetch_all(&mut *tx)
1183 .await
1184 .map_err(|e| DurableError::storage("prune", e))?;
1185 if ids.is_empty() {
1186 tx.commit()
1187 .await
1188 .map_err(|e| DurableError::storage("prune", e))?;
1189 return Ok(0);
1190 }
1191 let journal = sql!("DELETE FROM durable_journal WHERE execution_id = ?");
1192 let promises = sql!("DELETE FROM durable_promises WHERE execution_id = ?");
1193 let timers = sql!("DELETE FROM durable_timers WHERE execution_id = ?");
1194 let executions = sql!(
1197 "DELETE FROM durable_executions
1198 WHERE execution_id = ?
1199 AND finalized_at IS NOT NULL
1200 AND ( (status = 'completed' AND finalized_at <= ?)
1201 OR (status IN ('failed', 'aborted') AND finalized_at <= ?) )"
1202 );
1203 let mut removed = 0u64;
1204 for (id,) in &ids {
1205 for stmt in [journal, promises, timers] {
1206 zeph_db::query(stmt)
1207 .bind(id)
1208 .execute(&mut *tx)
1209 .await
1210 .map_err(|e| DurableError::storage("prune", e))?;
1211 }
1212 let result = zeph_db::query(executions)
1213 .bind(id)
1214 .bind(cutoffs.completed_before_ms)
1215 .bind(cutoffs.failed_before_ms)
1216 .execute(&mut *tx)
1217 .await
1218 .map_err(|e| DurableError::storage("prune", e))?;
1219 removed += result.rows_affected();
1220 }
1221 tx.commit()
1222 .await
1223 .map_err(|e| DurableError::storage("prune", e))?;
1224 Ok(removed)
1225 }
1226
1227 async fn sweep_orphan_batch(
1246 &self,
1247 lock_dir: &std::path::Path,
1248 cutoff_ms: i64,
1249 batch: u64,
1250 cursor: Option<crate::retention::SweepCursor>,
1251 ) -> Result<crate::retention::SweepBatchOutcome, DurableError> {
1252 let (after_updated_at, after_exec) = cursor.map_or((i64::MIN, String::new()), |c| {
1256 (c.updated_at_ms, c.execution_id)
1257 });
1258
1259 let candidates: Vec<(String, i64)> = zeph_db::query_as(sql!(
1260 "SELECT execution_id, updated_at FROM durable_executions
1261 WHERE status = 'running' AND updated_at <= ?
1262 AND (updated_at > ? OR (updated_at = ? AND execution_id > ?))
1263 ORDER BY updated_at, execution_id LIMIT ?"
1264 ))
1265 .bind(cutoff_ms)
1266 .bind(after_updated_at)
1267 .bind(after_updated_at)
1268 .bind(&after_exec)
1269 .bind(i64::try_from(batch).unwrap_or(i64::MAX))
1270 .fetch_all(&self.pool)
1271 .await
1272 .map_err(|e| DurableError::storage("sweep_orphans", e))?;
1273
1274 let scanned = u64::try_from(candidates.len()).unwrap_or(u64::MAX);
1275 let next_cursor = candidates
1276 .last()
1277 .map(|(id, updated_at)| crate::retention::SweepCursor {
1278 updated_at_ms: *updated_at,
1279 execution_id: id.clone(),
1280 });
1281
1282 let now = now_unix_millis();
1283 let abort = sql!(
1284 "UPDATE durable_executions SET status = 'aborted', finalized_at = ?, updated_at = ?
1285 WHERE execution_id = ? AND status = 'running' AND finalized_at IS NULL"
1286 );
1287 let mut aborted = 0u64;
1288 for (exec_str, _updated_at) in &candidates {
1289 let Ok(execution_id) = parse_execution_id(exec_str) else {
1290 continue;
1291 };
1292 match ExecutionLock::acquire(lock_dir, execution_id) {
1293 Ok(_lock) => {
1294 let result = zeph_db::query(abort)
1295 .bind(now)
1296 .bind(now)
1297 .bind(exec_str)
1298 .execute(&self.pool)
1299 .await
1300 .map_err(|e| DurableError::storage("sweep_orphans", e))?;
1301 aborted += result.rows_affected();
1302 }
1304 Err(DurableError::ExecutionLocked { .. }) => {
1305 }
1307 Err(e) => return Err(e),
1308 }
1309 }
1310 Ok(crate::retention::SweepBatchOutcome {
1311 scanned,
1312 aborted,
1313 next_cursor,
1314 })
1315 }
1316
1317 fn seal_payload(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, DurableError> {
1319 match &self.cipher {
1320 Some(cipher) => Ok(cipher.seal(plaintext, aad)?),
1321 None => Ok(plaintext.to_vec()),
1322 }
1323 }
1324
1325 fn open_payload(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Bytes, DurableError> {
1327 match &self.cipher {
1328 Some(cipher) => Ok(Bytes::from(cipher.open(sealed, aad)?)),
1329 None => Ok(Bytes::copy_from_slice(sealed)),
1330 }
1331 }
1332
1333 fn control_hmac(
1338 &self,
1339 entry: &JournalEntry,
1340 idem_key: Option<&IdempotencyKey>,
1341 ) -> Option<Vec<u8>> {
1342 self.compute_control_hmac(
1343 entry.execution_id,
1344 entry.step_id,
1345 entry.entry.tag(),
1346 idem_key,
1347 )
1348 .map(|h| h.to_vec())
1349 }
1350
1351 fn compute_control_hmac(
1356 &self,
1357 execution_id: ExecutionId,
1358 step_id: StepId,
1359 tag: &'static str,
1360 idem_key: Option<&IdempotencyKey>,
1361 ) -> Option<[u8; 32]> {
1362 let key = self.hmac_key.as_ref()?;
1363 let mut input = Vec::with_capacity(16 + 4 + 16 + 32);
1364 input.extend_from_slice(execution_id.as_bytes());
1365 input.extend_from_slice(&step_id.value().to_le_bytes());
1366 input.extend_from_slice(tag.as_bytes());
1367 if let Some(k) = idem_key {
1368 input.extend_from_slice(k.as_bytes());
1369 }
1370 Some(*blake3::keyed_hash(key, &input).as_bytes())
1371 }
1372
1373 fn verify_control_hmac(
1391 &self,
1392 execution_id: ExecutionId,
1393 step_id: StepId,
1394 tag: &'static str,
1395 idem_key: Option<&IdempotencyKey>,
1396 stored: Option<[u8; 32]>,
1397 ) -> Result<(), DurableError> {
1398 let Some(expected) = self.compute_control_hmac(execution_id, step_id, tag, idem_key) else {
1399 return if stored.is_some() {
1400 Err(DurableError::ControlIntegrity)
1401 } else {
1402 Ok(())
1403 };
1404 };
1405 match stored {
1406 Some(stored) if blake3::Hash::from(expected) == blake3::Hash::from(stored) => Ok(()),
1407 _ => Err(DurableError::ControlIntegrity),
1408 }
1409 }
1410
1411 fn prepare_row(&self, entry: &JournalEntry) -> Result<JournalRow, DurableError> {
1413 let execution_id = entry.execution_id.as_uuid().to_string();
1414 let step_id = i64::from(entry.step_id.value());
1415 let created_at = entry.created_at_ms;
1416 let entry_kind = entry.entry.tag();
1417 match &entry.entry {
1418 EntryKind::StepResult {
1419 idempotency_key,
1420 payload,
1421 effect,
1422 payload_version,
1423 } => {
1424 ensure_payload_within_limit(payload.len(), self.max_payload_bytes)?;
1425 let aad = PayloadAad::new(
1426 entry.execution_id,
1427 entry.step_id,
1428 EntryKindTag::StepResult,
1429 Some(*idempotency_key),
1430 );
1431 let sealed = self.seal_payload(payload.as_ref(), &aad)?;
1432 Ok(JournalRow {
1433 execution_id,
1434 step_id,
1435 entry_kind,
1436 idem_key: Some(idempotency_key.as_bytes().to_vec()),
1437 effect_class: Some(effect.as_str()),
1438 payload: Some(sealed),
1439 payload_version: Some(i32::from(*payload_version)),
1440 hmac: None,
1441 created_at,
1442 })
1443 }
1444 EntryKind::EffectIntent {
1445 idempotency_key,
1446 effect,
1447 hmac: _,
1448 } => {
1449 let hmac = self.control_hmac(entry, Some(idempotency_key));
1452 Ok(JournalRow {
1453 execution_id,
1454 step_id,
1455 entry_kind,
1456 idem_key: Some(idempotency_key.as_bytes().to_vec()),
1457 effect_class: Some(effect.as_str()),
1458 payload: None,
1459 payload_version: None,
1460 hmac,
1461 created_at,
1462 })
1463 }
1464 EntryKind::PromiseCreated { .. }
1465 | EntryKind::PromiseResolved { .. }
1466 | EntryKind::TimerArmed { .. }
1467 | EntryKind::TimerFired { .. }
1468 | EntryKind::Checkpoint { .. } => {
1469 Err(DurableError::UnsupportedEntryKind { kind: entry_kind })
1470 }
1471 }
1472 }
1473
1474 async fn lookup_kind(&self, id: ExecutionId) -> Result<ExecutionKind, DurableError> {
1476 let kind: Option<String> = zeph_db::query_scalar(sql!(
1477 "SELECT kind FROM durable_executions WHERE execution_id = ?"
1478 ))
1479 .bind(id.as_uuid().to_string())
1480 .fetch_optional(&self.pool)
1481 .await
1482 .map_err(|e| DurableError::storage("read", e))?;
1483 let kind = kind.ok_or(DurableError::Decode {
1484 context: "journaled entries reference a missing execution row",
1485 })?;
1486 ExecutionKind::from_tag(&kind).ok_or(DurableError::Decode {
1487 context: "execution kind is not reconstructible (custom kind read-back unsupported)",
1488 })
1489 }
1490
1491 fn row_to_entry(
1493 &self,
1494 id: ExecutionId,
1495 kind: ExecutionKind,
1496 row: JournalRowRead,
1497 ) -> Result<JournalEntry, DurableError> {
1498 let (
1499 seq,
1500 step_id_raw,
1501 entry_kind,
1502 idem_key,
1503 effect_class,
1504 payload,
1505 payload_version,
1506 hmac,
1507 created_at,
1508 ) = row;
1509 let step_id =
1510 StepId::new(
1511 u32::try_from(step_id_raw).map_err(|_| DurableError::Decode {
1512 context: "step_id out of u32 range",
1513 })?,
1514 );
1515 let entry = match entry_kind.as_str() {
1516 "step_result" => {
1517 let idem_bytes = idem_key.ok_or(DurableError::Decode {
1518 context: "step_result idem_key missing",
1519 })?;
1520 let idem_key = IdempotencyKey::from_bytes(slice_to_array32(
1521 &idem_bytes,
1522 "step_result idem_key",
1523 )?);
1524 let effect = effect_class
1525 .as_deref()
1526 .and_then(crate::EffectClass::from_tag)
1527 .ok_or(DurableError::Decode {
1528 context: "step_result effect_class missing or invalid",
1529 })?;
1530 let sealed = payload.ok_or(DurableError::Decode {
1531 context: "step_result payload missing",
1532 })?;
1533 ensure_payload_within_limit(
1534 sealed.len(),
1535 self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1536 )?;
1537 let aad = PayloadAad::new(id, step_id, EntryKindTag::StepResult, Some(idem_key));
1538 let opened = self.open_payload(&sealed, &aad)?;
1539 let version = u8::try_from(payload_version.unwrap_or(1)).map_err(|_| {
1540 DurableError::Decode {
1541 context: "payload_version out of u8 range",
1542 }
1543 })?;
1544 EntryKind::StepResult {
1545 idempotency_key: idem_key,
1546 payload: opened,
1547 effect,
1548 payload_version: version,
1549 }
1550 }
1551 "effect_intent" => {
1552 let idem_bytes = idem_key.ok_or(DurableError::Decode {
1553 context: "effect_intent idem_key missing",
1554 })?;
1555 let idem_key = IdempotencyKey::from_bytes(slice_to_array32(
1556 &idem_bytes,
1557 "effect_intent idem_key",
1558 )?);
1559 let effect = effect_class
1560 .as_deref()
1561 .and_then(crate::EffectClass::from_tag)
1562 .ok_or(DurableError::Decode {
1563 context: "effect_intent effect_class missing or invalid",
1564 })?;
1565 let hmac = hmac
1566 .map(|bytes| slice_to_array32(&bytes, "effect_intent hmac"))
1567 .transpose()?;
1568 self.verify_control_hmac(
1569 id,
1570 step_id,
1571 EntryKindTag::EffectIntent.as_str(),
1572 Some(&idem_key),
1573 hmac,
1574 )?;
1575 EntryKind::EffectIntent {
1576 idempotency_key: idem_key,
1577 effect,
1578 hmac,
1579 }
1580 }
1581 "checkpoint" => self.checkpoint_entry(id, step_id, payload)?,
1582 other => {
1583 return Err(DurableError::UnsupportedEntryKind {
1584 kind: static_entry_tag(other),
1585 });
1586 }
1587 };
1588 Ok(JournalEntry {
1589 seq: Some(JournalSeq::new(seq)),
1590 execution_id: id,
1591 kind,
1592 step_id,
1593 entry,
1594 created_at_ms: created_at,
1595 })
1596 }
1597
1598 fn checkpoint_entry(
1603 &self,
1604 id: ExecutionId,
1605 step_id: StepId,
1606 payload: Option<Vec<u8>>,
1607 ) -> Result<EntryKind, DurableError> {
1608 let sealed = payload.ok_or(DurableError::Decode {
1609 context: "checkpoint entry missing snapshot payload",
1610 })?;
1611 ensure_payload_within_limit(
1612 sealed.len(),
1613 self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1614 )?;
1615 let aad = PayloadAad::new(id, step_id, EntryKindTag::Checkpoint, None);
1616 let snapshot = self.open_payload(&sealed, &aad)?;
1617 Ok(EntryKind::Checkpoint {
1618 up_to_step: step_id.value(),
1619 snapshot,
1620 })
1621 }
1622
1623 async fn rows_to_entries(
1625 &self,
1626 id: ExecutionId,
1627 rows: Vec<JournalRowRead>,
1628 ) -> Result<Vec<JournalEntry>, DurableError> {
1629 if rows.is_empty() {
1630 return Ok(Vec::new());
1631 }
1632 let kind = self.lookup_kind(id).await?;
1633 let mut entries = Vec::with_capacity(rows.len());
1634 for row in rows {
1635 entries.push(self.row_to_entry(id, kind, row)?);
1636 }
1637 Ok(entries)
1638 }
1639}
1640
1641impl Journal for LocalBackend {
1642 async fn append(&self, entry: JournalEntry) -> Result<JournalSeq, DurableError> {
1643 let span = tracing::info_span!(
1644 "durable.journal.append",
1645 execution_id = %entry.execution_id.as_uuid(),
1646 step_id = entry.step_id.value(),
1647 entry_kind = entry.entry.tag(),
1648 );
1649 async move {
1650 let row = self.prepare_row(&entry)?;
1651 let (seq,): (i64,) = zeph_db::query_as(sql!(
1652 "INSERT INTO durable_journal
1653 (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
1654 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
1655 RETURNING seq"
1656 ))
1657 .bind(row.execution_id)
1658 .bind(row.step_id)
1659 .bind(row.entry_kind)
1660 .bind(row.idem_key)
1661 .bind(row.effect_class)
1662 .bind(row.payload)
1663 .bind(row.payload_version)
1664 .bind(row.hmac)
1665 .bind(row.created_at)
1666 .fetch_one(&self.pool)
1667 .await
1668 .map_err(|e| DurableError::storage("append", e))?;
1669 Ok(JournalSeq::new(seq))
1670 }
1671 .instrument(span)
1672 .await
1673 }
1674
1675 async fn read_execution(&self, id: ExecutionId) -> Result<Vec<JournalEntry>, DurableError> {
1676 let span = tracing::info_span!(
1677 "durable.journal.read",
1678 execution_id = %id.as_uuid(),
1679 step_count = tracing::field::Empty,
1680 );
1681 async move {
1682 let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
1683 "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
1684 FROM durable_journal WHERE execution_id = ? ORDER BY seq"
1685 ))
1686 .bind(id.as_uuid().to_string())
1687 .fetch_all(&self.pool)
1688 .await
1689 .map_err(|e| DurableError::storage("read", e))?;
1690 let entries = self.rows_to_entries(id, rows).await?;
1691 tracing::Span::current().record("step_count", entries.len());
1692 Ok(entries)
1693 }
1694 .instrument(span)
1695 .await
1696 }
1697
1698 async fn read_execution_range(
1699 &self,
1700 id: ExecutionId,
1701 from_step_id: u32,
1702 limit: usize,
1703 ) -> Result<Vec<JournalEntry>, DurableError> {
1704 let span = tracing::info_span!(
1705 "durable.journal.read_segment",
1706 execution_id = %id.as_uuid(),
1707 from_step_id,
1708 count = tracing::field::Empty,
1709 );
1710 async move {
1711 let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
1712 "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
1713 FROM durable_journal WHERE execution_id = ? AND step_id >= ? ORDER BY step_id, seq LIMIT ?"
1714 ))
1715 .bind(id.as_uuid().to_string())
1716 .bind(i64::from(from_step_id))
1717 .bind(i64::try_from(limit).unwrap_or(i64::MAX))
1718 .fetch_all(&self.pool)
1719 .await
1720 .map_err(|e| DurableError::storage("read_segment", e))?;
1721 let entries = self.rows_to_entries(id, rows).await?;
1722 tracing::Span::current().record("count", entries.len());
1723 Ok(entries)
1724 }
1725 .instrument(span)
1726 .await
1727 }
1728
1729 async fn finalize(&self, id: ExecutionId, status: ExecutionStatus) -> Result<(), DurableError> {
1730 let span = tracing::info_span!(
1731 "durable.journal.finalize",
1732 execution_id = %id.as_uuid(),
1733 status = status.as_str(),
1734 );
1735 async move {
1736 let now = now_unix_millis();
1737 let finalized_at = (!status.is_running()).then_some(now);
1738 let mut tx = zeph_db::begin_write(&self.pool)
1739 .await
1740 .map_err(|e| DurableError::storage("finalize", e))?;
1741 zeph_db::query(sql!(
1746 "UPDATE durable_executions SET status = ?, updated_at = ?, finalized_at = ?
1747 WHERE execution_id = ? AND status = 'running'"
1748 ))
1749 .bind(status.as_str())
1750 .bind(now)
1751 .bind(finalized_at)
1752 .bind(id.as_uuid().to_string())
1753 .execute(&mut *tx)
1754 .await
1755 .map_err(|e| DurableError::storage("finalize", e))?;
1756 tx.commit()
1757 .await
1758 .map_err(|e| DurableError::storage("finalize", e))?;
1759 Ok(())
1760 }
1761 .instrument(span)
1762 .await
1763 }
1764
1765 async fn prune(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
1766 let now = now_unix_millis();
1767 crate::retention::prune_in_batches(policy, now, |cutoffs, batch| {
1768 self.delete_prune_batch(cutoffs, batch)
1769 })
1770 .await
1771 }
1772
1773 async fn sweep_orphans(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
1775 if policy.stale_running_after_secs == 0 {
1776 return Ok(0);
1777 }
1778 let Some(lock_dir) = self.lock_dir.clone() else {
1779 if !self
1780 .orphan_sweep_warned
1781 .swap(true, std::sync::atomic::Ordering::Relaxed)
1782 {
1783 tracing::warn!(
1784 "durable: crash-orphan sweep requires an on-disk advisory-lock dir; orphan \
1785 reclamation disabled for this backend (Postgres/:memory:/non-Unix)"
1786 );
1787 }
1788 return Ok(0);
1789 };
1790 let cutoff_ms = orphan_cutoff_ms(policy, now_unix_millis());
1791 crate::retention::sweep_orphans_in_batches(
1792 policy.prune_batch_size,
1793 cutoff_ms,
1794 |cutoff, batch, cursor| self.sweep_orphan_batch(&lock_dir, cutoff, batch, cursor),
1795 )
1796 .await
1797 }
1798}
1799
1800impl crate::sealed::Sealed for LocalBackend {}
1801
1802impl ExecutionBackend for LocalBackend {
1803 fn capabilities(&self) -> BackendCapabilities {
1804 BackendCapabilities {
1805 parallel_steps: true,
1806 cross_process: cfg!(feature = "postgres"),
1808 max_payload: usize::try_from(self.max_payload_bytes).unwrap_or(usize::MAX),
1809 }
1810 }
1811
1812 async fn lookup_committed_result(
1813 &self,
1814 id: ExecutionId,
1815 idem_key: IdempotencyKey,
1816 ) -> Result<Option<JournalEntry>, DurableError> {
1817 LocalBackend::lookup_committed_result(self, id, idem_key).await
1818 }
1819}
1820
1821struct JournalRow {
1823 execution_id: String,
1824 step_id: i64,
1825 entry_kind: &'static str,
1826 idem_key: Option<Vec<u8>>,
1827 effect_class: Option<&'static str>,
1828 payload: Option<Vec<u8>>,
1829 payload_version: Option<i32>,
1830 hmac: Option<Vec<u8>>,
1831 created_at: i64,
1832}
1833
1834type JournalRowRead = (
1842 i64,
1843 i64,
1844 String,
1845 Option<Vec<u8>>,
1846 Option<String>,
1847 Option<Vec<u8>>,
1848 Option<i32>,
1849 Option<Vec<u8>>,
1850 i64,
1851);
1852
1853type PromiseRowRead = (String, Vec<u8>, i64, Option<Vec<u8>>);
1856
1857type FoldableRowRead = (i64, Option<Vec<u8>>, Option<i32>, Option<Vec<u8>>);
1860
1861#[cfg(feature = "sqlite")]
1868fn lock_dir_for_path(path: &str) -> Option<std::path::PathBuf> {
1869 (path != ":memory:").then(|| std::path::PathBuf::from(format!("{path}.locks")))
1870}
1871
1872#[cfg(not(feature = "sqlite"))]
1873fn lock_dir_for_path(_path: &str) -> Option<std::path::PathBuf> {
1874 None
1875}
1876
1877#[cfg(all(test, not(feature = "sqlite")))]
1883mod postgres_lock_dir_tests {
1884 use super::lock_dir_for_path;
1885
1886 #[test]
1887 fn postgres_url_never_derives_a_lock_dir() {
1888 assert_eq!(
1889 lock_dir_for_path("postgres://user:secret@host/db"),
1890 None,
1891 "a Postgres connection URL (which may embed credentials) must never be used to mint \
1892 an on-disk lock directory name"
1893 );
1894 assert_eq!(lock_dir_for_path(":memory:"), None);
1895 }
1896}
1897
1898pub(crate) fn now_unix_millis() -> i64 {
1900 SystemTime::now()
1901 .duration_since(UNIX_EPOCH)
1902 .map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX))
1903}
1904
1905fn orphan_cutoff_ms(policy: &RetentionPolicy, now_ms: i64) -> i64 {
1908 let threshold =
1909 i64::try_from(policy.stale_running_after_secs.saturating_mul(1000)).unwrap_or(i64::MAX);
1910 now_ms.saturating_sub(threshold)
1911}
1912
1913fn slice_to_array32(bytes: &[u8], field: &'static str) -> Result<[u8; 32], DurableError> {
1915 <[u8; 32]>::try_from(bytes).map_err(|_| DurableError::Decode { context: field })
1916}
1917
1918fn parse_execution_id(text: &str) -> Result<ExecutionId, DurableError> {
1920 uuid::Uuid::parse_str(text)
1921 .map(ExecutionId::from_uuid)
1922 .map_err(|_| DurableError::Decode {
1923 context: "execution_id is not a valid UUID",
1924 })
1925}
1926
1927fn parse_timer_id(text: &str) -> Result<TimerId, DurableError> {
1929 uuid::Uuid::parse_str(text)
1930 .map(TimerId::from_uuid)
1931 .map_err(|_| DurableError::Decode {
1932 context: "timer_id is not a valid UUID",
1933 })
1934}
1935
1936fn promise_payload_aad(execution_id: ExecutionId, promise_id: PromiseId) -> PayloadAad {
1941 let binding = IdempotencyKey::derive(
1942 execution_id,
1943 StepId::new(0),
1944 promise_id.as_uuid().as_bytes(),
1945 );
1946 PayloadAad::new(
1947 execution_id,
1948 StepId::new(0),
1949 EntryKindTag::PromiseResolved,
1950 Some(binding),
1951 )
1952}
1953
1954fn static_entry_tag(tag: &str) -> &'static str {
1956 match tag {
1957 "promise_created" => "promise_created",
1958 "promise_resolved" => "promise_resolved",
1959 "timer_armed" => "timer_armed",
1960 "timer_fired" => "timer_fired",
1961 "checkpoint" => "checkpoint",
1962 _ => "unknown",
1963 }
1964}
1965
1966#[cfg(all(test, feature = "sqlite"))]
1971mod tests {
1972 use std::assert_matches;
1973
1974 use super::*;
1975 use crate::cipher::CipherError;
1976 use crate::effect::EffectClass;
1977
1978 struct XorCipher;
1981 const XOR_MASK: u8 = 0x5A;
1982
1983 impl PayloadCipher for XorCipher {
1984 fn seal(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
1985 let tag = blake3::hash(&aad.canonical_bytes());
1986 let mut out = tag.as_bytes()[..8].to_vec();
1987 out.extend(plaintext.iter().map(|b| b ^ XOR_MASK));
1988 Ok(out)
1989 }
1990
1991 fn open(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
1992 if sealed.len() < 8 {
1993 return Err(CipherError::Malformed {
1994 context: "sealed blob shorter than the aad tag",
1995 });
1996 }
1997 let expected = blake3::hash(&aad.canonical_bytes());
1998 if sealed[..8] != expected.as_bytes()[..8] {
1999 return Err(CipherError::Authentication);
2000 }
2001 Ok(sealed[8..].iter().map(|b| b ^ XOR_MASK).collect())
2002 }
2003 }
2004
2005 async fn mem_backend(max_payload_bytes: u64) -> LocalBackend {
2006 let backend = LocalBackend::open(":memory:", max_payload_bytes)
2007 .await
2008 .expect("open in-memory backend");
2009 backend.init().await.expect("apply migrations");
2010 backend
2011 }
2012
2013 fn step_result(exec: ExecutionId, step: u32, payload: &[u8]) -> JournalEntry {
2014 let step_id = StepId::new(step);
2015 JournalEntry {
2016 seq: None,
2017 execution_id: exec,
2018 kind: ExecutionKind::AgentTurn,
2019 step_id,
2020 entry: EntryKind::StepResult {
2021 idempotency_key: IdempotencyKey::derive(exec, step_id, b"tool:read"),
2022 payload: Bytes::copy_from_slice(payload),
2023 effect: EffectClass::Idempotent,
2024 payload_version: 1,
2025 },
2026 created_at_ms: 100,
2027 }
2028 }
2029
2030 fn effect_intent(exec: ExecutionId, step: u32) -> JournalEntry {
2031 let step_id = StepId::new(step);
2032 JournalEntry {
2033 seq: None,
2034 execution_id: exec,
2035 kind: ExecutionKind::AgentTurn,
2036 step_id,
2037 entry: EntryKind::EffectIntent {
2038 idempotency_key: IdempotencyKey::derive(exec, step_id, b"transfer"),
2039 effect: EffectClass::ExactlyOnceGuarded,
2040 hmac: None,
2041 },
2042 created_at_ms: 100,
2043 }
2044 }
2045
2046 #[tokio::test]
2047 async fn open_execution_is_fresh_then_resume() {
2048 let backend = mem_backend(1_048_576).await;
2049 let exec = ExecutionId::new();
2050 assert!(
2051 !backend
2052 .open_execution(exec, ExecutionKind::AgentTurn)
2053 .await
2054 .unwrap()
2055 );
2056 assert!(
2057 backend
2058 .open_execution(exec, ExecutionKind::AgentTurn)
2059 .await
2060 .unwrap()
2061 );
2062 }
2063
2064 #[tokio::test]
2065 async fn open_execution_exclusive_is_fresh_then_resume() {
2066 let dir = tempfile::tempdir().unwrap();
2069 let db_path = dir.path().join("durable.db");
2070 let backend = LocalBackend::open(&db_path.to_string_lossy(), 1_048_576)
2071 .await
2072 .unwrap();
2073 backend.init().await.unwrap();
2074
2075 let exec = ExecutionId::new();
2076 let (is_resume, lock) = backend
2077 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
2078 .await
2079 .unwrap();
2080 assert!(!is_resume);
2081 assert!(lock.is_some(), "a file-backed backend must derive a lock");
2082 drop(lock);
2083
2084 let (is_resume, _lock) = backend
2085 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
2086 .await
2087 .unwrap();
2088 assert!(is_resume);
2089 }
2090
2091 #[tokio::test]
2096 async fn open_execution_exclusive_rejects_concurrent_second_holder() {
2097 let dir = tempfile::tempdir().unwrap();
2098 let db_path = dir.path().join("durable.db");
2099 let url = db_path.to_string_lossy().into_owned();
2100
2101 let backend_a = LocalBackend::open(&url, 1_048_576).await.unwrap();
2102 backend_a.init().await.unwrap();
2103 let backend_b = LocalBackend::open(&url, 1_048_576).await.unwrap();
2104
2105 let exec = ExecutionId::new();
2106 let (_, _lock_a) = backend_a
2107 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
2108 .await
2109 .unwrap();
2110
2111 let err = backend_b
2112 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
2113 .await
2114 .expect_err("a second concurrent holder must be rejected");
2115 assert!(
2116 matches!(err, DurableError::ExecutionLocked { execution_id, .. } if execution_id == exec),
2117 "expected ExecutionLocked, got {err:?}"
2118 );
2119 }
2120
2121 #[tokio::test]
2122 async fn open_execution_exclusive_on_memory_backend_returns_no_lock() {
2123 let backend = mem_backend(1_048_576).await;
2126 let exec = ExecutionId::new();
2127 let (is_resume, lock) = backend
2128 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
2129 .await
2130 .unwrap();
2131 assert!(!is_resume);
2132 assert!(lock.is_none());
2133 }
2134
2135 #[tokio::test]
2136 async fn list_executions_summarizes_and_filters() {
2137 let backend = mem_backend(1_048_576).await;
2138 let turn = ExecutionId::new();
2139 let dag = ExecutionId::new();
2140 backend
2141 .open_execution(turn, ExecutionKind::AgentTurn)
2142 .await
2143 .unwrap();
2144 backend
2145 .open_execution(dag, ExecutionKind::DagRun)
2146 .await
2147 .unwrap();
2148 backend.append(step_result(turn, 0, b"a")).await.unwrap();
2149 backend.append(step_result(turn, 1, b"b")).await.unwrap();
2150 backend.append(step_result(dag, 0, b"c")).await.unwrap();
2151 backend
2152 .finalize(turn, ExecutionStatus::Completed)
2153 .await
2154 .unwrap();
2155
2156 let all = backend.list_executions(None, None, 10).await.unwrap();
2158 assert_eq!(all.len(), 2);
2159
2160 let turn_row = all
2161 .iter()
2162 .find(|e| e.execution_id == turn)
2163 .expect("turn present");
2164 assert_eq!(turn_row.kind, "agent_turn");
2165 assert_eq!(turn_row.status, ExecutionStatus::Completed);
2166 assert_eq!(turn_row.step_count, 2);
2167 assert!(turn_row.finalized_at_ms.is_some());
2168
2169 let dag_row = all
2170 .iter()
2171 .find(|e| e.execution_id == dag)
2172 .expect("dag present");
2173 assert_eq!(dag_row.status, ExecutionStatus::Running);
2174 assert_eq!(dag_row.step_count, 1);
2175 assert!(dag_row.finalized_at_ms.is_none());
2176
2177 let running = backend
2179 .list_executions(Some("running"), None, 10)
2180 .await
2181 .unwrap();
2182 assert_eq!(running.len(), 1);
2183 assert_eq!(running[0].execution_id, dag);
2184
2185 let dags = backend
2187 .list_executions(None, Some("dag_run"), 10)
2188 .await
2189 .unwrap();
2190 assert_eq!(dags.len(), 1);
2191 assert_eq!(dags[0].execution_id, dag);
2192
2193 let one = backend.list_executions(None, None, 1).await.unwrap();
2195 assert_eq!(one.len(), 1);
2196 }
2197
2198 #[tokio::test]
2199 async fn append_and_read_round_trips_step_result() {
2200 let backend = mem_backend(1_048_576).await;
2201 let exec = ExecutionId::new();
2202 backend
2203 .open_execution(exec, ExecutionKind::AgentTurn)
2204 .await
2205 .unwrap();
2206
2207 let seq = backend
2208 .append(step_result(exec, 0, b"hello"))
2209 .await
2210 .unwrap();
2211 assert_eq!(seq.value(), 1, "first append takes seq 1");
2212
2213 let entries = backend.read_execution(exec).await.unwrap();
2214 assert_eq!(entries.len(), 1);
2215 match &entries[0].entry {
2216 EntryKind::StepResult {
2217 payload, effect, ..
2218 } => {
2219 assert_eq!(payload.as_ref(), b"hello");
2220 assert_eq!(*effect, EffectClass::Idempotent);
2221 }
2222 other => panic!("unexpected entry kind: {other:?}"),
2223 }
2224 assert_eq!(entries[0].seq, Some(seq));
2225 }
2226
2227 #[tokio::test]
2228 async fn cipher_seals_payload_at_rest_but_round_trips() {
2229 let backend = mem_backend(1_048_576)
2230 .await
2231 .with_cipher(Arc::new(XorCipher));
2232 let exec = ExecutionId::new();
2233 backend
2234 .open_execution(exec, ExecutionKind::AgentTurn)
2235 .await
2236 .unwrap();
2237 backend
2238 .append(step_result(exec, 0, b"secret-payload"))
2239 .await
2240 .unwrap();
2241
2242 let (stored,): (Option<Vec<u8>>,) = zeph_db::query_as(sql!(
2244 "SELECT payload FROM durable_journal WHERE execution_id = ?"
2245 ))
2246 .bind(exec.as_uuid().to_string())
2247 .fetch_one(backend.pool())
2248 .await
2249 .unwrap();
2250 let stored = stored.expect("payload present");
2251 assert_ne!(
2252 stored.as_slice(),
2253 b"secret-payload",
2254 "payload must be sealed at rest"
2255 );
2256
2257 let entries = backend.read_execution(exec).await.unwrap();
2259 match &entries[0].entry {
2260 EntryKind::StepResult { payload, .. } => {
2261 assert_eq!(payload.as_ref(), b"secret-payload");
2262 }
2263 other => panic!("unexpected entry kind: {other:?}"),
2264 }
2265 }
2266
2267 #[tokio::test]
2268 async fn control_entry_hmac_is_stamped_only_when_keyed() {
2269 let exec = ExecutionId::new();
2270
2271 let unkeyed = mem_backend(1_048_576).await;
2272 unkeyed
2273 .open_execution(exec, ExecutionKind::AgentTurn)
2274 .await
2275 .unwrap();
2276 unkeyed.append(effect_intent(exec, 0)).await.unwrap();
2277 match &unkeyed.read_execution(exec).await.unwrap()[0].entry {
2278 EntryKind::EffectIntent { hmac, .. } => assert!(hmac.is_none()),
2279 other => panic!("unexpected entry kind: {other:?}"),
2280 }
2281
2282 let keyed = mem_backend(1_048_576).await.with_hmac_key([7u8; 32]);
2283 let exec2 = ExecutionId::new();
2284 keyed
2285 .open_execution(exec2, ExecutionKind::AgentTurn)
2286 .await
2287 .unwrap();
2288 keyed.append(effect_intent(exec2, 0)).await.unwrap();
2289 match &keyed.read_execution(exec2).await.unwrap()[0].entry {
2290 EntryKind::EffectIntent { hmac, .. } => {
2291 assert!(
2292 hmac.is_some(),
2293 "keyed backend stamps a row HMAC over control entries"
2294 );
2295 }
2296 other => panic!("unexpected entry kind: {other:?}"),
2297 }
2298 }
2299
2300 #[tokio::test]
2307 async fn read_execution_rejects_control_hmac_under_wrong_key() {
2308 let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
2309 let exec = ExecutionId::new();
2310 writer
2311 .open_execution(exec, ExecutionKind::AgentTurn)
2312 .await
2313 .unwrap();
2314 writer.append(effect_intent(exec, 0)).await.unwrap();
2315
2316 let wrong_key_reader =
2317 LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([2u8; 32]);
2318 assert_matches!(
2319 wrong_key_reader.read_execution(exec).await,
2320 Err(DurableError::ControlIntegrity)
2321 );
2322
2323 let right_key_reader =
2325 LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([1u8; 32]);
2326 assert!(right_key_reader.read_execution(exec).await.is_ok());
2327 }
2328
2329 #[tokio::test]
2334 async fn read_execution_rejects_missing_hmac_on_keyed_backend() {
2335 let writer = mem_backend(1_048_576).await;
2336 let exec = ExecutionId::new();
2337 writer
2338 .open_execution(exec, ExecutionKind::AgentTurn)
2339 .await
2340 .unwrap();
2341 writer.append(effect_intent(exec, 0)).await.unwrap();
2342
2343 let keyed_reader =
2344 LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([3u8; 32]);
2345 assert_matches!(
2346 keyed_reader.read_execution(exec).await,
2347 Err(DurableError::ControlIntegrity)
2348 );
2349 }
2350
2351 #[tokio::test]
2360 async fn read_execution_rejects_stamped_hmac_on_unkeyed_backend() {
2361 let writer = mem_backend(1_048_576).await.with_hmac_key([4u8; 32]);
2362 let exec = ExecutionId::new();
2363 writer
2364 .open_execution(exec, ExecutionKind::AgentTurn)
2365 .await
2366 .unwrap();
2367 writer.append(effect_intent(exec, 0)).await.unwrap();
2368
2369 let unkeyed_reader = LocalBackend::new(writer.pool().clone(), 1_048_576);
2370 assert_matches!(
2371 unkeyed_reader.read_execution(exec).await,
2372 Err(DurableError::ControlIntegrity)
2373 );
2374 }
2375
2376 #[tokio::test]
2377 async fn promise_and_timer_entries_fail_closed() {
2378 let backend = mem_backend(1_048_576).await;
2379 let exec = ExecutionId::new();
2380 backend
2381 .open_execution(exec, ExecutionKind::AgentTurn)
2382 .await
2383 .unwrap();
2384 let timer = JournalEntry {
2385 seq: None,
2386 execution_id: exec,
2387 kind: ExecutionKind::AgentTurn,
2388 step_id: StepId::new(0),
2389 entry: EntryKind::TimerArmed {
2390 timer_id: crate::TimerId::new(),
2391 due_at_ms: 1_000,
2392 hmac: None,
2393 },
2394 created_at_ms: 0,
2395 };
2396 assert_matches!(
2397 backend.append(timer).await,
2398 Err(DurableError::UnsupportedEntryKind {
2399 kind: "timer_armed"
2400 })
2401 );
2402 }
2403
2404 #[tokio::test]
2405 async fn payload_over_limit_is_rejected_fail_closed() {
2406 let backend = mem_backend(8).await;
2407 let exec = ExecutionId::new();
2408 backend
2409 .open_execution(exec, ExecutionKind::AgentTurn)
2410 .await
2411 .unwrap();
2412 let big = vec![0u8; 64];
2413 assert_matches!(
2414 backend.append(step_result(exec, 0, &big)).await,
2415 Err(DurableError::PayloadTooLarge { .. })
2416 );
2417 }
2418
2419 #[tokio::test]
2420 async fn finalize_marks_terminal_status_and_time() {
2421 let backend = mem_backend(1_048_576).await;
2422 let exec = ExecutionId::new();
2423 backend
2424 .open_execution(exec, ExecutionKind::AgentTurn)
2425 .await
2426 .unwrap();
2427 backend
2428 .finalize(exec, ExecutionStatus::Completed)
2429 .await
2430 .unwrap();
2431
2432 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
2433 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
2434 ))
2435 .bind(exec.as_uuid().to_string())
2436 .fetch_one(backend.pool())
2437 .await
2438 .unwrap();
2439 assert_eq!(status, "completed");
2440 assert!(finalized.is_some(), "a terminal status stamps finalized_at");
2441 }
2442
2443 #[tokio::test]
2444 async fn finalize_is_a_noop_once_already_terminal() {
2445 let backend = mem_backend(1_048_576).await;
2448 let exec = ExecutionId::new();
2449 backend
2450 .open_execution(exec, ExecutionKind::AgentTurn)
2451 .await
2452 .unwrap();
2453 backend
2454 .finalize(exec, ExecutionStatus::Completed)
2455 .await
2456 .unwrap();
2457
2458 backend
2460 .finalize(exec, ExecutionStatus::Failed)
2461 .await
2462 .unwrap();
2463
2464 let (status,): (String,) = zeph_db::query_as(sql!(
2465 "SELECT status FROM durable_executions WHERE execution_id = ?"
2466 ))
2467 .bind(exec.as_uuid().to_string())
2468 .fetch_one(backend.pool())
2469 .await
2470 .unwrap();
2471 assert_eq!(
2472 status, "completed",
2473 "the first terminal status must stick; a later finalize call is a no-op"
2474 );
2475 }
2476
2477 #[tokio::test]
2478 async fn finalize_after_abort_is_a_noop() {
2479 let backend = mem_backend(1_048_576).await;
2483 let exec = ExecutionId::new();
2484 backend
2485 .open_execution(exec, ExecutionKind::AgentTurn)
2486 .await
2487 .unwrap();
2488 backend
2489 .finalize(exec, ExecutionStatus::Aborted)
2490 .await
2491 .unwrap();
2492
2493 backend
2494 .finalize(exec, ExecutionStatus::Completed)
2495 .await
2496 .unwrap();
2497
2498 let (status,): (String,) = zeph_db::query_as(sql!(
2499 "SELECT status FROM durable_executions WHERE execution_id = ?"
2500 ))
2501 .bind(exec.as_uuid().to_string())
2502 .fetch_one(backend.pool())
2503 .await
2504 .unwrap();
2505 assert_eq!(
2506 status, "aborted",
2507 "an aborted execution must not be overwritten by a later Completed/Failed call"
2508 );
2509 }
2510
2511 #[tokio::test]
2512 async fn reopening_a_finalized_execution_resets_it_to_running() {
2513 let backend = mem_backend(1_048_576).await;
2517 let exec = ExecutionId::new();
2518 backend
2519 .open_execution(exec, ExecutionKind::AgentTurn)
2520 .await
2521 .unwrap();
2522 backend
2523 .finalize(exec, ExecutionStatus::Completed)
2524 .await
2525 .unwrap();
2526
2527 let is_resume = backend
2528 .open_execution(exec, ExecutionKind::AgentTurn)
2529 .await
2530 .unwrap();
2531 assert!(is_resume, "the row already existed, so this is a resume");
2532
2533 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
2534 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
2535 ))
2536 .bind(exec.as_uuid().to_string())
2537 .fetch_one(backend.pool())
2538 .await
2539 .unwrap();
2540 assert_eq!(
2541 status, "running",
2542 "reopening a completed execution must un-finalize it"
2543 );
2544 assert!(
2545 finalized.is_none(),
2546 "reopening must clear the stale finalized_at"
2547 );
2548 }
2549
2550 #[tokio::test]
2551 async fn reopening_a_failed_execution_resets_it_to_running() {
2552 let backend = mem_backend(1_048_576).await;
2556 let exec = ExecutionId::new();
2557 backend
2558 .open_execution(exec, ExecutionKind::AgentTurn)
2559 .await
2560 .unwrap();
2561 backend
2562 .finalize(exec, ExecutionStatus::Failed)
2563 .await
2564 .unwrap();
2565
2566 let is_resume = backend
2567 .open_execution(exec, ExecutionKind::AgentTurn)
2568 .await
2569 .unwrap();
2570 assert!(is_resume, "the row already existed, so this is a resume");
2571
2572 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
2573 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
2574 ))
2575 .bind(exec.as_uuid().to_string())
2576 .fetch_one(backend.pool())
2577 .await
2578 .unwrap();
2579 assert_eq!(
2580 status, "running",
2581 "reopening a failed execution must un-finalize it"
2582 );
2583 assert!(
2584 finalized.is_none(),
2585 "reopening must clear the stale finalized_at"
2586 );
2587 }
2588
2589 #[tokio::test]
2590 async fn reopening_an_aborted_execution_un_finalizes_it() {
2591 let backend = mem_backend(1_048_576).await;
2598 let exec = ExecutionId::new();
2599 backend
2600 .open_execution(exec, ExecutionKind::AgentTurn)
2601 .await
2602 .unwrap();
2603 backend
2604 .finalize(exec, ExecutionStatus::Aborted)
2605 .await
2606 .unwrap();
2607
2608 let is_resume = backend
2609 .open_execution(exec, ExecutionKind::AgentTurn)
2610 .await
2611 .unwrap();
2612 assert!(is_resume, "the row already existed, so this is a resume");
2613
2614 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
2615 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
2616 ))
2617 .bind(exec.as_uuid().to_string())
2618 .fetch_one(backend.pool())
2619 .await
2620 .unwrap();
2621 assert_eq!(
2622 status, "running",
2623 "reopening an aborted execution must un-finalize it (INV-16)"
2624 );
2625 assert!(
2626 finalized.is_none(),
2627 "reopening must clear the stale finalized_at"
2628 );
2629 }
2630
2631 #[tokio::test]
2632 async fn reopen_of_a_row_deleted_out_from_under_it_starts_fresh() {
2633 let backend = mem_backend(1_048_576).await;
2639 let exec = ExecutionId::new();
2640 backend
2641 .open_execution(exec, ExecutionKind::AgentTurn)
2642 .await
2643 .unwrap();
2644 backend
2645 .finalize(exec, ExecutionStatus::Completed)
2646 .await
2647 .unwrap();
2648
2649 zeph_db::query(sql!(
2651 "DELETE FROM durable_executions WHERE execution_id = ?"
2652 ))
2653 .bind(exec.as_uuid().to_string())
2654 .execute(backend.pool())
2655 .await
2656 .unwrap();
2657
2658 let is_resume = backend
2659 .open_execution(exec, ExecutionKind::AgentTurn)
2660 .await
2661 .unwrap();
2662 assert!(
2663 !is_resume,
2664 "a row deleted by a concurrent prune must be reported as a fresh execution, not a resume"
2665 );
2666
2667 let (status,): (String,) = zeph_db::query_as(sql!(
2668 "SELECT status FROM durable_executions WHERE execution_id = ?"
2669 ))
2670 .bind(exec.as_uuid().to_string())
2671 .fetch_one(backend.pool())
2672 .await
2673 .unwrap();
2674 assert_eq!(status, "running", "the fresh row starts running");
2675 }
2676
2677 #[tokio::test]
2678 async fn prune_does_not_delete_a_row_reopened_since_it_was_finalized() {
2679 let backend = mem_backend(1_048_576).await;
2683 let exec = ExecutionId::new();
2684 backend
2685 .open_execution(exec, ExecutionKind::AgentTurn)
2686 .await
2687 .unwrap();
2688 backend.append(step_result(exec, 0, b"x")).await.unwrap();
2689 zeph_db::query(sql!(
2690 "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
2691 ))
2692 .bind(exec.as_uuid().to_string())
2693 .execute(backend.pool())
2694 .await
2695 .unwrap();
2696
2697 let is_resume = backend
2699 .open_execution(exec, ExecutionKind::AgentTurn)
2700 .await
2701 .unwrap();
2702 assert!(is_resume);
2703
2704 let policy = RetentionPolicy {
2705 ttl_completed_secs: 1,
2706 prune_batch_size: 10,
2707 ..RetentionPolicy::default()
2708 };
2709 let deleted = backend.prune(&policy).await.unwrap();
2710 assert_eq!(
2711 deleted, 0,
2712 "a reopened (un-finalized) execution must not be pruned"
2713 );
2714 assert_eq!(
2715 backend.read_execution(exec).await.unwrap().len(),
2716 1,
2717 "the execution's journal must survive"
2718 );
2719 }
2720
2721 #[tokio::test]
2722 async fn concurrent_prune_and_reopen_never_lose_or_corrupt_the_row() {
2723 let dir = tempfile::tempdir().unwrap();
2738 let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
2739 let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
2740 backend.init().await.unwrap();
2741
2742 let policy = RetentionPolicy {
2743 ttl_completed_secs: 1,
2744 prune_batch_size: 10,
2745 ..RetentionPolicy::default()
2746 };
2747
2748 for _ in 0..20 {
2749 let exec = ExecutionId::new();
2750 backend
2751 .open_execution(exec, ExecutionKind::AgentTurn)
2752 .await
2753 .unwrap();
2754 backend.append(step_result(exec, 0, b"x")).await.unwrap();
2755 zeph_db::query(sql!(
2757 "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
2758 ))
2759 .bind(exec.as_uuid().to_string())
2760 .execute(backend.pool())
2761 .await
2762 .unwrap();
2763
2764 let reopen_backend = backend.clone();
2765 let reopen = tokio::spawn(async move {
2766 reopen_backend
2767 .open_execution(exec, ExecutionKind::AgentTurn)
2768 .await
2769 });
2770 let prune_backend = backend.clone();
2771 let policy_for_task = policy.clone();
2772 let prune = tokio::spawn(async move { prune_backend.prune(&policy_for_task).await });
2773
2774 let (reopen_result, prune_result) = tokio::join!(reopen, prune);
2775 reopen_result
2776 .expect("reopen task must not panic")
2777 .expect("reopen must not error under concurrent prune");
2778 prune_result
2779 .expect("prune task must not panic")
2780 .expect("prune must not error under a concurrent reopen");
2781
2782 let (status,): (String,) = zeph_db::query_as(sql!(
2783 "SELECT status FROM durable_executions WHERE execution_id = ?"
2784 ))
2785 .bind(exec.as_uuid().to_string())
2786 .fetch_one(backend.pool())
2787 .await
2788 .expect(
2789 "the row must exist under either race outcome — reopened-running, or \
2790 deleted-then-reinserted-fresh-running by reopen's fallback",
2791 );
2792 assert_eq!(
2793 status, "running",
2794 "whichever task wins, the row must end up running — never left completed \
2795 (orphaned from a live journal) or absent"
2796 );
2797 }
2798 }
2799
2800 #[tokio::test]
2801 async fn max_seq_reflects_committed_appends() {
2802 let backend = mem_backend(1_048_576).await;
2803 assert_eq!(
2804 backend.max_seq().await.unwrap(),
2805 None,
2806 "empty journal has no max seq"
2807 );
2808
2809 let exec = ExecutionId::new();
2810 backend
2811 .open_execution(exec, ExecutionKind::AgentTurn)
2812 .await
2813 .unwrap();
2814 for step in 0..3 {
2815 backend.append(step_result(exec, step, b"x")).await.unwrap();
2816 }
2817 assert_eq!(backend.max_seq().await.unwrap(), Some(JournalSeq::new(3)));
2818 }
2819
2820 #[tokio::test]
2821 async fn append_batch_group_commits_every_entry() {
2822 let backend = mem_backend(1_048_576).await;
2823 let exec = ExecutionId::new();
2824 backend
2825 .open_execution(exec, ExecutionKind::AgentTurn)
2826 .await
2827 .unwrap();
2828 let batch = vec![
2829 step_result(exec, 0, b"a"),
2830 step_result(exec, 1, b"b"),
2831 step_result(exec, 2, b"c"),
2832 ];
2833 backend.append_batch(&batch).await.unwrap();
2834 assert_eq!(backend.read_execution(exec).await.unwrap().len(), 3);
2835 }
2836
2837 #[tokio::test]
2838 async fn read_execution_range_bounds_the_segment() {
2839 let backend = mem_backend(1_048_576).await;
2840 let exec = ExecutionId::new();
2841 backend
2842 .open_execution(exec, ExecutionKind::AgentTurn)
2843 .await
2844 .unwrap();
2845 for step in 0..5 {
2846 backend.append(step_result(exec, step, b"x")).await.unwrap();
2847 }
2848 let segment = backend.read_execution_range(exec, 2, 2).await.unwrap();
2849 assert_eq!(segment.len(), 2);
2850 assert_eq!(segment[0].step_id, StepId::new(2));
2851 assert_eq!(segment[1].step_id, StepId::new(3));
2852 }
2853
2854 #[tokio::test]
2855 async fn lookup_committed_result_finds_by_idem_key() {
2856 let backend = mem_backend(1_048_576).await;
2857 let exec = ExecutionId::new();
2858 backend
2859 .open_execution(exec, ExecutionKind::AgentTurn)
2860 .await
2861 .unwrap();
2862 let entry = step_result(exec, 0, b"committed");
2863 let idem_key = match &entry.entry {
2864 EntryKind::StepResult {
2865 idempotency_key, ..
2866 } => *idempotency_key,
2867 other => panic!("unexpected entry kind: {other:?}"),
2868 };
2869 backend.append(entry).await.unwrap();
2870
2871 let found = backend
2872 .lookup_committed_result(exec, idem_key)
2873 .await
2874 .unwrap()
2875 .expect("committed result is located by its idempotency key");
2876 match &found.entry {
2877 EntryKind::StepResult { payload, .. } => assert_eq!(payload.as_ref(), b"committed"),
2878 other => panic!("unexpected entry kind: {other:?}"),
2879 }
2880
2881 let absent = IdempotencyKey::derive(exec, StepId::new(99), b"never");
2883 assert!(
2884 backend
2885 .lookup_committed_result(exec, absent)
2886 .await
2887 .unwrap()
2888 .is_none()
2889 );
2890 }
2891
2892 #[tokio::test]
2893 async fn capabilities_describe_the_local_profile() {
2894 let backend = mem_backend(4096).await;
2895 let caps = backend.capabilities();
2896 assert!(caps.parallel_steps);
2897 assert!(
2898 !caps.cross_process,
2899 "the SQLite local backend is in-process"
2900 );
2901 assert_eq!(caps.max_payload, 4096);
2902 }
2903
2904 #[tokio::test]
2905 async fn promise_insert_state_and_resolve_round_trip() {
2906 let backend = mem_backend(1_048_576)
2907 .await
2908 .with_cipher(Arc::new(XorCipher));
2909 let exec = ExecutionId::new();
2910 backend
2911 .open_execution(exec, ExecutionKind::AgentTurn)
2912 .await
2913 .unwrap();
2914 let promise = PromiseId::derive(exec, StepId::new(0));
2915 backend
2916 .insert_promise(promise, exec, [9u8; 32], 100)
2917 .await
2918 .unwrap();
2919
2920 let pending = backend.promise_state(promise).await.unwrap().unwrap();
2921 assert!(!pending.resolved);
2922 assert_eq!(pending.execution_id, exec);
2923 assert_eq!(pending.resolver_token_hash, [9u8; 32]);
2924
2925 assert!(
2927 backend
2928 .resolve_promise(promise, exec, b"answer", 200)
2929 .await
2930 .unwrap()
2931 );
2932 assert!(
2933 !backend
2934 .resolve_promise(promise, exec, b"again", 300)
2935 .await
2936 .unwrap()
2937 );
2938
2939 let resolved = backend.promise_state(promise).await.unwrap().unwrap();
2940 assert!(resolved.resolved);
2941 let sealed = resolved.payload.expect("resolved payload present");
2942 assert_ne!(sealed.as_slice(), b"answer", "payload is sealed at rest");
2943 let opened = backend
2944 .open_promise_payload(promise, exec, &sealed)
2945 .unwrap();
2946 assert_eq!(opened.as_ref(), b"answer");
2947 }
2948
2949 #[tokio::test]
2950 async fn claim_promise_notification_is_single_winner() {
2951 let backend = mem_backend(1_048_576).await;
2952 let exec = ExecutionId::new();
2953 backend
2954 .open_execution(exec, ExecutionKind::AgentTurn)
2955 .await
2956 .unwrap();
2957 let promise = PromiseId::derive(exec, StepId::new(0));
2958 backend
2959 .insert_promise(promise, exec, [9u8; 32], 100)
2960 .await
2961 .unwrap();
2962
2963 assert!(
2965 backend
2966 .claim_promise_notification(promise, 200)
2967 .await
2968 .unwrap()
2969 );
2970 assert!(
2972 !backend
2973 .claim_promise_notification(promise, 300)
2974 .await
2975 .unwrap()
2976 );
2977 }
2978
2979 #[tokio::test]
2980 async fn timer_arm_due_and_fire() {
2981 let backend = mem_backend(1_048_576).await;
2982 let exec = ExecutionId::new();
2983 backend
2984 .open_execution(exec, ExecutionKind::AgentTurn)
2985 .await
2986 .unwrap();
2987 let past = TimerId::derive(exec, StepId::new(0));
2988 let future = TimerId::derive(exec, StepId::new(1));
2989 backend.arm_timer(past, exec, 1_000, 0).await.unwrap();
2990 backend
2991 .arm_timer(future, exec, 9_000_000_000_000, 0)
2992 .await
2993 .unwrap();
2994
2995 let due = backend.due_timers(5_000).await.unwrap();
2997 assert_eq!(due, vec![past]);
2998
2999 assert!(backend.mark_timer_fired(past).await.unwrap());
3000 assert!(
3001 !backend.mark_timer_fired(past).await.unwrap(),
3002 "second fire is a no-op"
3003 );
3004 assert_eq!(
3005 backend.timer_state(past).await.unwrap(),
3006 Some((1_000, true))
3007 );
3008 assert!(backend.due_timers(5_000).await.unwrap().is_empty());
3010 }
3011
3012 #[tokio::test]
3013 async fn prune_deletes_terminal_executions_past_ttl() {
3014 let backend = mem_backend(1_048_576).await;
3015 let old = ExecutionId::new();
3017 backend
3018 .open_execution(old, ExecutionKind::AgentTurn)
3019 .await
3020 .unwrap();
3021 backend.append(step_result(old, 0, b"x")).await.unwrap();
3022 zeph_db::query(sql!(
3024 "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
3025 ))
3026 .bind(old.as_uuid().to_string())
3027 .execute(backend.pool())
3028 .await
3029 .unwrap();
3030
3031 let live = ExecutionId::new();
3032 backend
3033 .open_execution(live, ExecutionKind::AgentTurn)
3034 .await
3035 .unwrap();
3036 backend.append(step_result(live, 0, b"y")).await.unwrap();
3037
3038 let policy = RetentionPolicy {
3039 ttl_completed_secs: 1,
3040 prune_batch_size: 10,
3041 ..RetentionPolicy::default()
3042 };
3043 let deleted = backend.prune(&policy).await.unwrap();
3044 assert_eq!(deleted, 1, "only the aged terminal execution is pruned");
3045
3046 assert!(backend.read_execution(old).await.unwrap().is_empty());
3048 assert!(
3049 backend
3050 .promise_state(PromiseId::derive(old, StepId::new(0)))
3051 .await
3052 .unwrap()
3053 .is_none()
3054 );
3055 assert_eq!(backend.read_execution(live).await.unwrap().len(), 1);
3056 }
3057
3058 async fn backdate_updated_at(backend: &LocalBackend, id: ExecutionId, updated_at_ms: i64) {
3060 zeph_db::query(sql!(
3061 "UPDATE durable_executions SET updated_at = ? WHERE execution_id = ?"
3062 ))
3063 .bind(updated_at_ms)
3064 .bind(id.as_uuid().to_string())
3065 .execute(backend.pool())
3066 .await
3067 .unwrap();
3068 }
3069
3070 #[tokio::test]
3071 async fn sweep_orphans_disabled_when_threshold_is_zero() {
3072 let dir = tempfile::tempdir().unwrap();
3075 let backend =
3076 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
3077 .await
3078 .unwrap();
3079 backend.init().await.unwrap();
3080
3081 let exec = ExecutionId::new();
3082 backend
3083 .open_execution(exec, ExecutionKind::AgentTurn)
3084 .await
3085 .unwrap();
3086 backdate_updated_at(&backend, exec, 0).await;
3087
3088 let policy = RetentionPolicy {
3089 stale_running_after_secs: 0,
3090 ..RetentionPolicy::default()
3091 };
3092 let aborted = backend.sweep_orphans(&policy).await.unwrap();
3093 assert_eq!(
3094 aborted, 0,
3095 "stale_running_after_secs = 0 disables the sweep"
3096 );
3097
3098 let (status,): (String,) = zeph_db::query_as(sql!(
3099 "SELECT status FROM durable_executions WHERE execution_id = ?"
3100 ))
3101 .bind(exec.as_uuid().to_string())
3102 .fetch_one(backend.pool())
3103 .await
3104 .unwrap();
3105 assert_eq!(status, "running");
3106 }
3107
3108 #[tokio::test]
3109 async fn sweep_orphans_is_a_documented_no_op_on_memory_backend() {
3110 let backend = mem_backend(1_048_576).await;
3113 let exec = ExecutionId::new();
3114 backend
3115 .open_execution(exec, ExecutionKind::AgentTurn)
3116 .await
3117 .unwrap();
3118 backdate_updated_at(&backend, exec, 0).await;
3119
3120 let policy = RetentionPolicy {
3121 stale_running_after_secs: 1,
3122 ..RetentionPolicy::default()
3123 };
3124 let aborted = backend.sweep_orphans(&policy).await.unwrap();
3125 assert_eq!(
3126 aborted, 0,
3127 "a lock_dir=None backend must never abort on staleness alone"
3128 );
3129
3130 let (status,): (String,) = zeph_db::query_as(sql!(
3131 "SELECT status FROM durable_executions WHERE execution_id = ?"
3132 ))
3133 .bind(exec.as_uuid().to_string())
3134 .fetch_one(backend.pool())
3135 .await
3136 .unwrap();
3137 assert_eq!(status, "running");
3138 }
3139
3140 #[tokio::test]
3141 async fn sweep_orphans_aborts_a_stale_running_execution_with_no_live_owner() {
3142 let dir = tempfile::tempdir().unwrap();
3144 let backend =
3145 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
3146 .await
3147 .unwrap();
3148 backend.init().await.unwrap();
3149
3150 let exec = ExecutionId::new();
3151 backend
3152 .open_execution(exec, ExecutionKind::AgentTurn)
3153 .await
3154 .unwrap();
3155 backdate_updated_at(&backend, exec, 0).await;
3158
3159 let policy = RetentionPolicy {
3160 stale_running_after_secs: 1,
3161 ..RetentionPolicy::default()
3162 };
3163 let aborted = backend.sweep_orphans(&policy).await.unwrap();
3164 assert_eq!(aborted, 1);
3165
3166 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3167 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3168 ))
3169 .bind(exec.as_uuid().to_string())
3170 .fetch_one(backend.pool())
3171 .await
3172 .unwrap();
3173 assert_eq!(status, "aborted");
3174 assert!(finalized.is_some());
3175 }
3176
3177 #[tokio::test]
3178 async fn sweep_orphans_skips_an_execution_whose_lock_is_held_by_a_live_owner() {
3179 let dir = tempfile::tempdir().unwrap();
3183 let db_path = dir.path().join("durable.db");
3184 let url = db_path.to_string_lossy().into_owned();
3185
3186 let owner = LocalBackend::open(&url, 1_048_576).await.unwrap();
3187 owner.init().await.unwrap();
3188 let sweeper = LocalBackend::open(&url, 1_048_576).await.unwrap();
3189
3190 let exec = ExecutionId::new();
3191 let (_, _lock) = owner
3192 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3193 .await
3194 .unwrap();
3195 backdate_updated_at(&owner, exec, 0).await;
3196
3197 let policy = RetentionPolicy {
3198 stale_running_after_secs: 1,
3199 ..RetentionPolicy::default()
3200 };
3201 let aborted = sweeper.sweep_orphans(&policy).await.unwrap();
3202 assert_eq!(aborted, 0, "a live-held lock must never be swept");
3203
3204 let (status,): (String,) = zeph_db::query_as(sql!(
3205 "SELECT status FROM durable_executions WHERE execution_id = ?"
3206 ))
3207 .bind(exec.as_uuid().to_string())
3208 .fetch_one(owner.pool())
3209 .await
3210 .unwrap();
3211 assert_eq!(status, "running");
3212 }
3213
3214 #[tokio::test]
3215 async fn sweep_orphans_leaves_a_fresh_running_execution_untouched() {
3216 let dir = tempfile::tempdir().unwrap();
3218 let backend =
3219 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
3220 .await
3221 .unwrap();
3222 backend.init().await.unwrap();
3223
3224 let exec = ExecutionId::new();
3225 backend
3226 .open_execution(exec, ExecutionKind::AgentTurn)
3227 .await
3228 .unwrap();
3229
3230 let policy = RetentionPolicy {
3231 stale_running_after_secs: 3600,
3232 ..RetentionPolicy::default()
3233 };
3234 let aborted = backend.sweep_orphans(&policy).await.unwrap();
3235 assert_eq!(aborted, 0);
3236 }
3237
3238 #[tokio::test]
3239 async fn count_orphans_matches_sweep_without_mutating() {
3240 let dir = tempfile::tempdir().unwrap();
3241 let backend =
3242 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
3243 .await
3244 .unwrap();
3245 backend.init().await.unwrap();
3246
3247 let exec = ExecutionId::new();
3248 backend
3249 .open_execution(exec, ExecutionKind::AgentTurn)
3250 .await
3251 .unwrap();
3252 backdate_updated_at(&backend, exec, 0).await;
3253
3254 let policy = RetentionPolicy {
3255 stale_running_after_secs: 1,
3256 ..RetentionPolicy::default()
3257 };
3258 let counted = backend.count_orphans(&policy).await.unwrap();
3259 assert_eq!(counted, 1);
3260
3261 let (status,): (String,) = zeph_db::query_as(sql!(
3263 "SELECT status FROM durable_executions WHERE execution_id = ?"
3264 ))
3265 .bind(exec.as_uuid().to_string())
3266 .fetch_one(backend.pool())
3267 .await
3268 .unwrap();
3269 assert_eq!(status, "running");
3270
3271 let aborted = backend.sweep_orphans(&policy).await.unwrap();
3272 assert_eq!(
3273 aborted, counted,
3274 "sweep must abort exactly what count_orphans counted"
3275 );
3276 }
3277
3278 #[tokio::test]
3284 async fn sweep_orphans_processes_every_batch_when_candidates_straddle_the_batch_size() {
3285 let dir = tempfile::tempdir().unwrap();
3286 let backend =
3287 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
3288 .await
3289 .unwrap();
3290 backend.init().await.unwrap();
3291
3292 let batch_size = 2u64;
3293 let candidate_count = batch_size + 1; let mut execs = Vec::new();
3295 for _ in 0..candidate_count {
3296 let exec = ExecutionId::new();
3297 backend
3298 .open_execution(exec, ExecutionKind::AgentTurn)
3299 .await
3300 .unwrap();
3301 backdate_updated_at(&backend, exec, 0).await;
3302 execs.push(exec);
3303 }
3304
3305 let policy = RetentionPolicy {
3306 stale_running_after_secs: 1,
3307 prune_batch_size: batch_size,
3308 ..RetentionPolicy::default()
3309 };
3310 let aborted = backend.sweep_orphans(&policy).await.unwrap();
3311 assert_eq!(
3312 aborted, candidate_count,
3313 "every candidate must be aborted, including the one past the first batch"
3314 );
3315
3316 for exec in execs {
3317 let (status,): (String,) = zeph_db::query_as(sql!(
3318 "SELECT status FROM durable_executions WHERE execution_id = ?"
3319 ))
3320 .bind(exec.as_uuid().to_string())
3321 .fetch_one(backend.pool())
3322 .await
3323 .unwrap();
3324 assert_eq!(status, "aborted");
3325 }
3326 }
3327
3328 #[tokio::test]
3341 async fn sweep_orphans_terminates_when_lock_held_candidates_exceed_batch_size() {
3342 let dir = tempfile::tempdir().unwrap();
3343 let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
3344
3345 let owner = LocalBackend::open(&db_url, 1_048_576).await.unwrap();
3346 owner.init().await.unwrap();
3347 let sweeper = LocalBackend::open(&db_url, 1_048_576).await.unwrap();
3348
3349 let batch_size = 2u64;
3350 let candidate_count = batch_size * 2 + 1; let mut locks = Vec::new();
3352 for _ in 0..candidate_count {
3353 let exec = ExecutionId::new();
3354 let (_, lock) = owner
3355 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3356 .await
3357 .unwrap();
3358 backdate_updated_at(&owner, exec, 0).await;
3359 locks.push(lock); }
3361
3362 let policy = RetentionPolicy {
3363 stale_running_after_secs: 1,
3364 prune_batch_size: batch_size,
3365 ..RetentionPolicy::default()
3366 };
3367
3368 let aborted = tokio::time::timeout(
3369 std::time::Duration::from_secs(10),
3370 sweeper.sweep_orphans(&policy),
3371 )
3372 .await
3373 .expect(
3374 "sweep_orphans must terminate even when lock-held candidates exceed prune_batch_size \
3375 (#6254 C1) — it hung instead of returning",
3376 )
3377 .unwrap();
3378
3379 assert_eq!(aborted, 0, "every candidate's lock is held by a live owner");
3380 drop(locks);
3381 }
3382
3383 #[tokio::test]
3392 async fn concurrent_sweep_and_reopen_race_never_corrupts_the_row() {
3393 let dir = tempfile::tempdir().unwrap();
3394 let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
3395 let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
3396 backend.init().await.unwrap();
3397
3398 let policy = RetentionPolicy {
3399 stale_running_after_secs: 1,
3400 prune_batch_size: 10,
3401 ..RetentionPolicy::default()
3402 };
3403
3404 for _ in 0..20 {
3405 let exec = ExecutionId::new();
3406 backend
3407 .open_execution(exec, ExecutionKind::AgentTurn)
3408 .await
3409 .unwrap();
3410 backdate_updated_at(&backend, exec, 0).await;
3411
3412 let sweep_backend = backend.clone();
3413 let policy_for_task = policy.clone();
3414 let sweep =
3415 tokio::spawn(async move { sweep_backend.sweep_orphans(&policy_for_task).await });
3416
3417 let reopen_backend = backend.clone();
3418 let reopen = tokio::spawn(async move {
3419 reopen_backend
3420 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3421 .await
3422 });
3423
3424 let (sweep_result, reopen_result) = tokio::join!(sweep, reopen);
3425 let aborted = sweep_result
3426 .expect("sweep task must not panic")
3427 .expect("sweep must not error under a concurrent reopen");
3428 assert!(aborted <= 1, "at most one candidate row exists per trial");
3429
3430 match reopen_result.expect("reopen task must not panic") {
3431 Ok((_is_resume, _lock)) => {
3432 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3436 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3437 ))
3438 .bind(exec.as_uuid().to_string())
3439 .fetch_one(backend.pool())
3440 .await
3441 .unwrap();
3442 assert_eq!(status, "running");
3443 assert!(finalized.is_none());
3444 backend
3452 .finalize(exec, ExecutionStatus::Completed)
3453 .await
3454 .unwrap();
3455 }
3456 Err(DurableError::ExecutionLocked { .. }) => {
3457 }
3459 Err(e) => panic!(
3460 "reopen must only ever fail with ExecutionLocked under this race, got {e:?}"
3461 ),
3462 }
3463 }
3464 }
3465
3466 #[tokio::test]
3467 async fn checkpoint_fold_compacts_idempotent_prefix_and_replays() {
3468 let backend = mem_backend(1_048_576)
3469 .await
3470 .with_cipher(Arc::new(XorCipher));
3471 let exec = ExecutionId::new();
3472 backend
3473 .open_execution(exec, ExecutionKind::AgentTurn)
3474 .await
3475 .unwrap();
3476 for step in 0..5 {
3477 backend
3478 .append(step_result(exec, step, format!("v{step}").as_bytes()))
3479 .await
3480 .unwrap();
3481 }
3482
3483 let folded = backend.checkpoint_fold(exec, 3).await.unwrap();
3485 assert_eq!(folded, 3);
3486
3487 let remaining = backend.read_execution(exec).await.unwrap();
3489 let step_results: Vec<u32> = remaining
3490 .iter()
3491 .filter(|e| matches!(e.entry, EntryKind::StepResult { .. }))
3492 .map(|e| e.step_id.value())
3493 .collect();
3494 assert_eq!(step_results, vec![3, 4], "folded step rows are deleted");
3495 assert!(
3496 remaining
3497 .iter()
3498 .any(|e| matches!(e.entry, EntryKind::Checkpoint { .. })),
3499 "a checkpoint entry replaces the folded prefix"
3500 );
3501
3502 let preloaded = backend.read_checkpoints(exec).await.unwrap();
3504 assert_eq!(preloaded.len(), 3);
3505 for (i, entry) in preloaded.iter().enumerate() {
3506 let step = u32::try_from(i).unwrap();
3507 assert_eq!(entry.step_id, StepId::new(step));
3508 match &entry.entry {
3509 EntryKind::StepResult {
3510 payload,
3511 idempotency_key,
3512 ..
3513 } => {
3514 assert_eq!(payload.as_ref(), format!("v{step}").as_bytes());
3515 assert_eq!(
3516 *idempotency_key,
3517 IdempotencyKey::derive(exec, StepId::new(step), b"tool:read")
3518 );
3519 }
3520 other => panic!("unexpected folded entry: {other:?}"),
3521 }
3522 }
3523 }
3524}