1use std::fmt;
38use std::fmt::Write as _;
39use std::path::PathBuf;
40use std::sync::Arc;
41use std::time::{SystemTime, UNIX_EPOCH};
42
43use bytes::Bytes;
44use zeph_db::{DbPool, sql};
45
46use crate::backend::execution_lock::ExecutionLock;
47use crate::backend::{BackendCapabilities, ExecutionBackend, ExecutionSummary, RedactedEntry};
48use crate::cipher::{EntryKindTag, PayloadAad, PayloadCipher, ensure_payload_within_limit};
49use crate::config::RetentionPolicy;
50use crate::error::DurableError;
51use crate::ids::{
52 ExecutionId, ExecutionKind, IdempotencyKey, JournalSeq, PromiseId, StepId, TimerId,
53};
54use crate::journal::{EntryKind, ExecutionStatus, Journal, JournalEntry};
55use crate::promise::PromiseRecord;
56use crate::retention::{CheckpointSnapshot, FoldedStep, decode_checkpoint, encode_checkpoint};
57use crate::waiters::NotifyRegistry;
58use tracing::Instrument as _;
59
60const SEAL_OVERHEAD_SLACK: u64 = 128;
68
69type ExecutionRow = (String, String, String, i64, i64, Option<i64>, i64);
71
72type RedactedRow = (
74 i64,
75 i64,
76 String,
77 Option<Vec<u8>>,
78 Option<String>,
79 Option<i64>,
80 i64,
81);
82
83fn idem_key_prefix(bytes: &[u8]) -> String {
85 bytes.iter().take(8).fold(String::new(), |mut acc, b| {
86 let _ = write!(acc, "{b:02x}");
87 acc
88 })
89}
90
91#[derive(Debug, Clone, Copy, PartialEq, Eq)]
96pub enum CancelOutcome {
97 Canceled,
100 AlreadyTerminal {
103 status: ExecutionStatus,
105 },
106 NotFound,
108 LiveOwner {
114 pid: u32,
116 },
117 LivenessUnverifiable,
121}
122
123pub struct LocalBackend {
141 pool: DbPool,
142 cipher: Option<Arc<dyn PayloadCipher>>,
143 hmac_key: Option<[u8; 32]>,
144 previous_hmac_key: Option<[u8; 32]>,
149 hwm_key: Option<HwmKeySlot>,
157 hwm_key_previous: Option<HwmKeySlot>,
160 max_payload_bytes: u64,
161 promise_waiters: NotifyRegistry,
163 timer_waiters: NotifyRegistry,
165 lock_dir: Option<PathBuf>,
171 orphan_sweep_warned: std::sync::atomic::AtomicBool,
175 integrity_sealed: bool,
182 integrity_grandfather: std::collections::HashSet<ExecutionId>,
190}
191
192#[derive(Clone, Copy)]
198struct HwmKeySlot {
199 epoch: u32,
200 key: [u8; 32],
201}
202
203impl fmt::Debug for LocalBackend {
204 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
206 f.debug_struct("LocalBackend")
207 .field("cipher", &self.cipher.as_ref().map(|_| "<cipher>"))
208 .field("hmac_key", &self.hmac_key.as_ref().map(|_| "<redacted>"))
209 .field(
210 "previous_hmac_key",
211 &self.previous_hmac_key.as_ref().map(|_| "<redacted>"),
212 )
213 .field("hwm_key_epoch", &self.hwm_key.as_ref().map(|s| s.epoch))
214 .field("max_payload_bytes", &self.max_payload_bytes)
215 .finish_non_exhaustive()
216 }
217}
218
219impl LocalBackend {
221 #[must_use]
227 pub fn new(pool: DbPool, max_payload_bytes: u64) -> Self {
228 Self {
229 pool,
230 cipher: None,
231 hmac_key: None,
232 previous_hmac_key: None,
233 hwm_key: None,
234 hwm_key_previous: None,
235 max_payload_bytes,
236 promise_waiters: NotifyRegistry::default(),
237 timer_waiters: NotifyRegistry::default(),
238 lock_dir: None,
239 orphan_sweep_warned: std::sync::atomic::AtomicBool::new(false),
240 integrity_sealed: false,
241 integrity_grandfather: std::collections::HashSet::new(),
242 }
243 }
244
245 pub async fn open(path: &str, max_payload_bytes: u64) -> Result<Self, DurableError> {
259 let pool = zeph_db::DbConfig {
260 url: path.to_string(),
261 pool_size: 5,
262 }
263 .connect()
264 .await
265 .map_err(|e| DurableError::storage("open", e))?;
266 let mut backend = Self::new(pool, max_payload_bytes);
267 backend.lock_dir = lock_dir_for_path(path);
268 Ok(backend)
269 }
270
271 #[must_use]
273 pub fn with_cipher(mut self, cipher: Arc<dyn PayloadCipher>) -> Self {
274 self.cipher = Some(cipher);
275 self
276 }
277
278 #[must_use]
281 pub fn with_hmac_key(mut self, key: [u8; 32]) -> Self {
282 self.hmac_key = Some(key);
283 self
284 }
285
286 #[must_use]
297 pub fn with_previous_hmac_key(mut self, key: [u8; 32]) -> Self {
298 self.previous_hmac_key = Some(key);
299 self
300 }
301
302 #[must_use]
311 pub fn with_hwm_key(mut self, epoch: u32, key: [u8; 32]) -> Self {
312 self.hwm_key = Some(HwmKeySlot { epoch, key });
313 self
314 }
315
316 #[must_use]
321 pub fn with_previous_hwm_key(mut self, epoch: u32, key: [u8; 32]) -> Self {
322 self.hwm_key_previous = Some(HwmKeySlot { epoch, key });
323 self
324 }
325
326 #[must_use]
332 pub fn with_integrity_sealed(mut self, sealed: bool) -> Self {
333 self.integrity_sealed = sealed;
334 self
335 }
336
337 #[must_use]
343 pub fn with_grandfather(mut self, ids: std::collections::HashSet<ExecutionId>) -> Self {
344 self.integrity_grandfather = ids;
345 self
346 }
347
348 #[must_use]
350 pub fn pool(&self) -> &DbPool {
351 &self.pool
352 }
353
354 pub async fn init(&self) -> Result<(), DurableError> {
362 zeph_db::run_migrations(&self.pool)
363 .await
364 .map_err(|e| DurableError::storage("init", e))?;
365 Ok(())
366 }
367}
368
369impl LocalBackend {
371 pub async fn list_executions(
386 &self,
387 status: Option<&str>,
388 kind: Option<&str>,
389 limit: i64,
390 ) -> Result<Vec<ExecutionSummary>, DurableError> {
391 let span = tracing::info_span!(
392 "durable.backend.list",
393 status = status.unwrap_or("*"),
394 kind = kind.unwrap_or("*"),
395 count = tracing::field::Empty,
396 );
397 async move {
398 let rows: Vec<ExecutionRow> =
402 zeph_db::query_as(sql!(
403 "SELECT
404 e.execution_id,
405 e.kind,
406 e.status,
407 e.created_at,
408 e.updated_at,
409 e.finalized_at,
410 (SELECT COUNT(*) FROM durable_journal j WHERE j.execution_id = e.execution_id)
411 FROM durable_executions e
412 WHERE e.status = COALESCE(?, e.status)
413 AND e.kind = COALESCE(?, e.kind)
414 ORDER BY e.created_at DESC
415 LIMIT ?"
416 ))
417 .bind(status)
418 .bind(kind)
419 .bind(limit)
420 .fetch_all(&self.pool)
421 .await
422 .map_err(|e| DurableError::storage("list", e))?;
423 tracing::Span::current().record("count", rows.len());
424 rows.into_iter()
425 .map(|(id, kind, status, created, updated, finalized, steps)| {
426 Ok(ExecutionSummary {
427 execution_id: parse_execution_id(&id)?,
428 kind,
429 status: ExecutionStatus::from_tag(&status).ok_or(DurableError::Decode {
430 context: "execution status is not a recognized CHECK-constrained value",
431 })?,
432 created_at_ms: created,
433 updated_at_ms: updated,
434 finalized_at_ms: finalized,
435 step_count: steps.max(0).cast_unsigned(),
436 })
437 })
438 .collect()
439 }
440 .instrument(span)
441 .await
442 }
443
444 pub async fn execution_status(
455 &self,
456 id: ExecutionId,
457 ) -> Result<Option<ExecutionStatus>, DurableError> {
458 let row: Option<(String,)> = zeph_db::query_as(sql!(
459 "SELECT status FROM durable_executions WHERE execution_id = ?"
460 ))
461 .bind(id.as_uuid().to_string())
462 .fetch_optional(&self.pool)
463 .await
464 .map_err(|e| DurableError::storage("execution_status", e))?;
465 row.map(|(status,)| {
466 ExecutionStatus::from_tag(&status).ok_or(DurableError::Decode {
467 context: "execution status is not a recognized CHECK-constrained value",
468 })
469 })
470 .transpose()
471 }
472
473 pub async fn read_execution_redacted(
486 &self,
487 id: ExecutionId,
488 ) -> Result<Vec<RedactedEntry>, DurableError> {
489 let exec = id.as_uuid().to_string();
490 let rows: Vec<RedactedRow> = zeph_db::query_as(sql!(
491 "SELECT seq, step_id, entry_kind, idem_key, effect_class, LENGTH(payload), created_at
492 FROM durable_journal WHERE execution_id = ? ORDER BY seq"
493 ))
494 .bind(&exec)
495 .fetch_all(&self.pool)
496 .await
497 .map_err(|e| DurableError::storage("read_redacted", e))?;
498 Ok(rows
499 .into_iter()
500 .map(
501 |(seq, step, entry_kind, idem, effect_class, payload_len, created)| RedactedEntry {
502 seq,
503 step_id: StepId::new(u32::try_from(step).unwrap_or(0)),
504 entry_kind,
505 effect_class,
506 idem_key_prefix: idem.as_deref().map(idem_key_prefix),
507 payload_len: payload_len.unwrap_or(0).max(0).cast_unsigned(),
508 created_at_ms: created,
509 },
510 )
511 .collect())
512 }
513
514 pub async fn open_execution(
558 &self,
559 id: ExecutionId,
560 kind: ExecutionKind,
561 ) -> Result<bool, DurableError> {
562 let span = tracing::info_span!(
563 "durable.backend.open",
564 execution_id = %id.as_uuid(),
565 kind = kind.as_str(),
566 is_resume = tracing::field::Empty,
567 );
568 async move {
569 let exec = id.as_uuid().to_string();
570
571 let reopened = zeph_db::query(sql!(
575 "UPDATE durable_executions SET status = 'running', updated_at = ?, finalized_at = NULL
576 WHERE execution_id = ? AND status IN ('completed', 'failed', 'aborted')"
577 ))
578 .bind(now_unix_millis())
579 .bind(&exec)
580 .execute(&self.pool)
581 .await
582 .map_err(|e| DurableError::storage("open", e))?;
583 if reopened.rows_affected() > 0 {
584 self.verify_high_water_mark(id).await?;
585 tracing::Span::current().record("is_resume", true);
586 return Ok(true);
587 }
588
589 let existing: Option<(String,)> = zeph_db::query_as(sql!(
596 "SELECT status FROM durable_executions WHERE execution_id = ?"
597 ))
598 .bind(&exec)
599 .fetch_optional(&self.pool)
600 .await
601 .map_err(|e| DurableError::storage("open", e))?;
602 if let Some((status,)) = existing {
603 if status == "canceled" {
604 return Err(DurableError::ExecutionCanceled { execution_id: id });
605 }
606 self.verify_high_water_mark(id).await?;
607 tracing::Span::current().record("is_resume", true);
608 return Ok(true);
609 }
610 let now = now_unix_millis();
611 zeph_db::query(sql!(
612 "INSERT INTO durable_executions
613 (execution_id, kind, status, created_at, updated_at, finalized_at)
614 VALUES (?, ?, 'running', ?, ?, NULL)"
615 ))
616 .bind(&exec)
617 .bind(kind.as_str())
618 .bind(now)
619 .bind(now)
620 .execute(&self.pool)
621 .await
622 .map_err(|e| DurableError::storage("open", e))?;
623 tracing::Span::current().record("is_resume", false);
624 Ok(false)
625 }
626 .instrument(span)
627 .await
628 }
629
630 pub async fn open_execution_exclusive(
652 &self,
653 id: ExecutionId,
654 kind: ExecutionKind,
655 ) -> Result<(bool, Option<ExecutionLock>), DurableError> {
656 let lock = self
657 .lock_dir
658 .as_deref()
659 .map(|dir| ExecutionLock::acquire(dir, id))
660 .transpose()?;
661 let is_resume = self.open_execution(id, kind).await?;
662 Ok((is_resume, lock))
663 }
664
665 pub async fn cancel_execution(&self, id: ExecutionId) -> Result<CancelOutcome, DurableError> {
706 let span = tracing::info_span!(
707 "durable.backend.cancel",
708 execution_id = %id.as_uuid(),
709 prior_status = tracing::field::Empty,
710 path = tracing::field::Empty,
711 );
712 async move {
713 let exec = id.as_uuid().to_string();
714
715 if let Some(lock_dir) = self.lock_dir.clone() {
716 let _lock = match ExecutionLock::acquire(&lock_dir, id) {
717 Ok(lock) => lock,
718 Err(DurableError::ExecutionLocked { holder_pid, .. }) => {
719 tracing::Span::current().record("path", "live_owner_refused");
720 return Ok(CancelOutcome::LiveOwner { pid: holder_pid });
721 }
722 Err(e) => return Err(e),
723 };
724 let outcome = self.cancel_write(&exec).await?;
725 tracing::Span::current().record("path", "immediate");
726 record_prior_status(outcome);
727 return Ok(outcome);
728 }
730
731 if self.capabilities().cross_process {
732 tracing::Span::current().record("path", "unverifiable");
733 return Ok(CancelOutcome::LivenessUnverifiable);
734 }
735
736 tracing::Span::current().record("path", "no_lock_dir_single_process");
737 let outcome = self.cancel_write(&exec).await?;
738 record_prior_status(outcome);
739 Ok(outcome)
740 }
741 .instrument(span)
742 .await
743 }
744
745 async fn cancel_write(&self, exec: &str) -> Result<CancelOutcome, DurableError> {
749 let now = now_unix_millis();
750 let mut tx = zeph_db::begin_write(&self.pool)
751 .await
752 .map_err(|e| DurableError::storage("cancel", e))?;
753 let result = zeph_db::query(sql!(
754 "UPDATE durable_executions SET status = 'canceled', finalized_at = ?, updated_at = ?
755 WHERE execution_id = ? AND status = 'running'"
756 ))
757 .bind(now)
758 .bind(now)
759 .bind(exec)
760 .execute(&mut *tx)
761 .await
762 .map_err(|e| DurableError::storage("cancel", e))?;
763 if result.rows_affected() > 0 {
764 tx.commit()
765 .await
766 .map_err(|e| DurableError::storage("cancel", e))?;
767 return Ok(CancelOutcome::Canceled);
768 }
769
770 let existing: Option<(String,)> = zeph_db::query_as(sql!(
774 "SELECT status FROM durable_executions WHERE execution_id = ?"
775 ))
776 .bind(exec)
777 .fetch_optional(&mut *tx)
778 .await
779 .map_err(|e| DurableError::storage("cancel", e))?;
780 tx.commit()
781 .await
782 .map_err(|e| DurableError::storage("cancel", e))?;
783 match existing {
784 None => Ok(CancelOutcome::NotFound),
785 Some((status,)) => {
786 let status = ExecutionStatus::from_tag(&status).ok_or(DurableError::Decode {
787 context: "unrecognized durable_executions.status value",
788 })?;
789 Ok(CancelOutcome::AlreadyTerminal { status })
790 }
791 }
792 }
793
794 pub(crate) async fn append_batch(&self, entries: &[JournalEntry]) -> Result<(), DurableError> {
807 if entries.is_empty() {
808 return Ok(());
809 }
810 let mut rows = Vec::with_capacity(entries.len());
811 for entry in entries {
812 rows.push(self.prepare_row(entry)?);
813 }
814 let insert = sql!(
818 "INSERT INTO durable_journal
819 (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
820 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
821 );
822 let mut tx = zeph_db::begin_write(&self.pool)
823 .await
824 .map_err(|e| DurableError::storage("append_batch", e))?;
825 for (entry, row) in entries.iter().zip(rows) {
826 zeph_db::query(insert)
827 .bind(row.execution_id)
828 .bind(row.step_id)
829 .bind(row.entry_kind)
830 .bind(row.idem_key)
831 .bind(row.effect_class)
832 .bind(row.payload)
833 .bind(row.payload_version)
834 .bind(row.hmac)
835 .bind(row.created_at)
836 .execute(&mut *tx)
837 .await
838 .map_err(|e| DurableError::storage("append_batch", e))?;
839 if matches!(entry.entry, EntryKind::StepResult { .. }) {
840 self.bump_hwm_for_step_result(&mut tx, entry.execution_id, entry.step_id)
841 .await?;
842 }
843 }
844 tx.commit()
845 .await
846 .map_err(|e| DurableError::storage("append_batch", e))?;
847 Ok(())
848 }
849
850 pub(crate) async fn lookup_committed_result(
864 &self,
865 id: ExecutionId,
866 idem_key: IdempotencyKey,
867 ) -> Result<Option<JournalEntry>, DurableError> {
868 let span = tracing::info_span!(
869 "durable.journal.lookup_idem",
870 execution_id = %id.as_uuid(),
871 found = tracing::field::Empty,
872 );
873 async move {
874 let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
875 "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
876 FROM durable_journal
877 WHERE execution_id = ? AND idem_key = ? AND entry_kind = 'step_result'
878 ORDER BY seq LIMIT 1"
879 ))
880 .bind(id.as_uuid().to_string())
881 .bind(idem_key.as_bytes().to_vec())
882 .fetch_all(&self.pool)
883 .await
884 .map_err(|e| DurableError::storage("lookup_idem", e))?;
885 let entry = self.rows_to_entries(id, rows).await?.into_iter().next();
886 tracing::Span::current().record("found", entry.is_some());
887 Ok(entry)
888 }
889 .instrument(span)
890 .await
891 }
892
893 pub(crate) async fn max_seq(&self) -> Result<Option<JournalSeq>, DurableError> {
903 let max: Option<i64> = zeph_db::query_scalar(sql!("SELECT MAX(seq) FROM durable_journal"))
904 .fetch_one(&self.pool)
905 .await
906 .map_err(|e| DurableError::storage("max_seq", e))?;
907 Ok(max.map(JournalSeq::new))
908 }
909
910 pub(crate) fn promise_waiters(&self) -> &NotifyRegistry {
912 &self.promise_waiters
913 }
914
915 pub(crate) fn timer_waiters(&self) -> &NotifyRegistry {
917 &self.timer_waiters
918 }
919
920 pub(crate) async fn insert_promise(
929 &self,
930 id: PromiseId,
931 execution_id: ExecutionId,
932 resolver_token_hash: [u8; 32],
933 created_at_ms: i64,
934 ) -> Result<(), DurableError> {
935 let span = tracing::info_span!("durable.promise.create", promise_id = %id.as_uuid());
936 async move {
937 zeph_db::query(sql!(
938 "INSERT INTO durable_promises
939 (promise_id, execution_id, resolver_token_hash, resolved, payload, created_at, resolved_at)
940 VALUES (?, ?, ?, 0, NULL, ?, NULL)"
941 ))
942 .bind(id.as_uuid().to_string())
943 .bind(execution_id.as_uuid().to_string())
944 .bind(resolver_token_hash.to_vec())
945 .bind(created_at_ms)
946 .execute(&self.pool)
947 .await
948 .map_err(|e| DurableError::storage("insert_promise", e))?;
949 Ok(())
950 }
951 .instrument(span)
952 .await
953 }
954
955 pub(crate) async fn promise_state(
962 &self,
963 id: PromiseId,
964 ) -> Result<Option<PromiseRecord>, DurableError> {
965 let row: Option<PromiseRowRead> = zeph_db::query_as(sql!(
966 "SELECT execution_id, resolver_token_hash, resolved, payload
967 FROM durable_promises WHERE promise_id = ?"
968 ))
969 .bind(id.as_uuid().to_string())
970 .fetch_optional(&self.pool)
971 .await
972 .map_err(|e| DurableError::storage("promise_state", e))?;
973 let Some((exec, hash, resolved, payload)) = row else {
974 return Ok(None);
975 };
976 Ok(Some(PromiseRecord {
977 execution_id: parse_execution_id(&exec)?,
978 resolver_token_hash: slice_to_array32(&hash, "promise resolver_token_hash")?,
979 resolved: resolved != 0,
980 payload,
981 }))
982 }
983
984 pub(crate) async fn resolve_promise(
995 &self,
996 id: PromiseId,
997 execution_id: ExecutionId,
998 value_plaintext: &[u8],
999 resolved_at_ms: i64,
1000 ) -> Result<bool, DurableError> {
1001 let span = tracing::info_span!("durable.promise.resolve", promise_id = %id.as_uuid());
1002 async move {
1003 ensure_payload_within_limit(value_plaintext.len(), self.max_payload_bytes)?;
1004 let aad = promise_payload_aad(execution_id, id);
1005 let sealed = self.seal_payload(value_plaintext, &aad)?;
1006 let affected = zeph_db::query(sql!(
1007 "UPDATE durable_promises SET resolved = 1, payload = ?, resolved_at = ?
1008 WHERE promise_id = ? AND resolved = 0"
1009 ))
1010 .bind(sealed)
1011 .bind(resolved_at_ms)
1012 .bind(id.as_uuid().to_string())
1013 .execute(&self.pool)
1014 .await
1015 .map_err(|e| DurableError::storage("resolve_promise", e))?
1016 .rows_affected();
1017 if affected > 0 {
1018 self.promise_waiters.wake(id.as_uuid());
1019 }
1020 Ok(affected > 0)
1021 }
1022 .instrument(span)
1023 .await
1024 }
1025
1026 pub(crate) async fn claim_promise_notification(
1039 &self,
1040 id: PromiseId,
1041 notified_at_ms: i64,
1042 ) -> Result<bool, DurableError> {
1043 let span = tracing::info_span!("durable.promise.claim_notify", promise_id = %id.as_uuid());
1044 async move {
1045 let affected = zeph_db::query(sql!(
1046 "UPDATE durable_promises SET notified_at = ?
1047 WHERE promise_id = ? AND notified_at IS NULL"
1048 ))
1049 .bind(notified_at_ms)
1050 .bind(id.as_uuid().to_string())
1051 .execute(&self.pool)
1052 .await
1053 .map_err(|e| DurableError::storage("claim_promise_notification", e))?
1054 .rows_affected();
1055 Ok(affected > 0)
1056 }
1057 .instrument(span)
1058 .await
1059 }
1060
1061 pub(crate) fn open_promise_payload(
1068 &self,
1069 id: PromiseId,
1070 execution_id: ExecutionId,
1071 sealed: &[u8],
1072 ) -> Result<Bytes, DurableError> {
1073 ensure_payload_within_limit(
1074 sealed.len(),
1075 self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1076 )?;
1077 let aad = promise_payload_aad(execution_id, id);
1078 self.open_payload(sealed, &aad)
1079 }
1080
1081 pub(crate) async fn arm_timer(
1089 &self,
1090 id: TimerId,
1091 execution_id: ExecutionId,
1092 due_at_ms: i64,
1093 created_at_ms: i64,
1094 ) -> Result<(), DurableError> {
1095 let span = tracing::info_span!("durable.timer.arm", timer_id = %id.as_uuid(), due_at_ms);
1096 async move {
1097 zeph_db::query(sql!(
1098 "INSERT INTO durable_timers (timer_id, execution_id, due_at, fired, created_at)
1099 VALUES (?, ?, ?, 0, ?)"
1100 ))
1101 .bind(id.as_uuid().to_string())
1102 .bind(execution_id.as_uuid().to_string())
1103 .bind(due_at_ms)
1104 .bind(created_at_ms)
1105 .execute(&self.pool)
1106 .await
1107 .map_err(|e| DurableError::storage("arm_timer", e))?;
1108 Ok(())
1109 }
1110 .instrument(span)
1111 .await
1112 }
1113
1114 pub(crate) async fn timer_state(
1120 &self,
1121 id: TimerId,
1122 ) -> Result<Option<(i64, bool)>, DurableError> {
1123 let row: Option<(i64, i64)> = zeph_db::query_as(sql!(
1124 "SELECT due_at, fired FROM durable_timers WHERE timer_id = ?"
1125 ))
1126 .bind(id.as_uuid().to_string())
1127 .fetch_optional(&self.pool)
1128 .await
1129 .map_err(|e| DurableError::storage("timer_state", e))?;
1130 Ok(row.map(|(due_at, fired)| (due_at, fired != 0)))
1131 }
1132
1133 pub(crate) async fn due_timers(&self, now_ms: i64) -> Result<Vec<TimerId>, DurableError> {
1143 let rows: Vec<(String,)> = zeph_db::query_as(sql!(
1144 "SELECT timer_id FROM durable_timers WHERE fired = 0 AND due_at <= ? ORDER BY due_at"
1145 ))
1146 .bind(now_ms)
1147 .fetch_all(&self.pool)
1148 .await
1149 .map_err(|e| DurableError::storage("due_timers", e))?;
1150 rows.into_iter().map(|(id,)| parse_timer_id(&id)).collect()
1151 }
1152
1153 pub(crate) async fn mark_timer_fired(&self, id: TimerId) -> Result<bool, DurableError> {
1161 let span = tracing::info_span!("durable.timer.fire", timer_id = %id.as_uuid());
1162 async move {
1163 let affected = zeph_db::query(sql!(
1164 "UPDATE durable_timers SET fired = 1 WHERE timer_id = ? AND fired = 0"
1165 ))
1166 .bind(id.as_uuid().to_string())
1167 .execute(&self.pool)
1168 .await
1169 .map_err(|e| DurableError::storage("mark_timer_fired", e))?
1170 .rows_affected();
1171 if affected > 0 {
1172 self.timer_waiters.wake(id.as_uuid());
1173 }
1174 Ok(affected > 0)
1175 }
1176 .instrument(span)
1177 .await
1178 }
1179
1180 fn open_foldable_steps(
1186 &self,
1187 execution_id: ExecutionId,
1188 rows: Vec<FoldableRowRead>,
1189 ) -> Result<Vec<FoldedStep>, DurableError> {
1190 let mut folded = Vec::with_capacity(rows.len());
1191 for (step_raw, idem, version, payload) in rows {
1192 let step = u32::try_from(step_raw).map_err(|_| DurableError::Decode {
1193 context: "checkpoint step_id out of u32 range",
1194 })?;
1195 let idem_bytes = idem.ok_or(DurableError::Decode {
1196 context: "checkpoint step result missing idem_key",
1197 })?;
1198 let idem_key =
1199 IdempotencyKey::from_bytes(slice_to_array32(&idem_bytes, "checkpoint idem_key")?);
1200 let sealed = payload.ok_or(DurableError::Decode {
1201 context: "checkpoint step result missing payload",
1202 })?;
1203 let aad = PayloadAad::new(
1204 execution_id,
1205 StepId::new(step),
1206 EntryKindTag::StepResult,
1207 Some(idem_key),
1208 );
1209 let plaintext = self.open_payload(&sealed, &aad)?;
1210 let payload_version =
1211 u8::try_from(version.unwrap_or(1)).map_err(|_| DurableError::Decode {
1212 context: "checkpoint payload_version out of u8 range",
1213 })?;
1214 folded.push(FoldedStep {
1215 step_id: step,
1216 idem_key: *idem_key.as_bytes(),
1217 payload_version,
1218 payload: plaintext,
1219 });
1220 }
1221 Ok(folded)
1222 }
1223
1224 pub(crate) async fn checkpoint_fold(
1245 &self,
1246 execution_id: ExecutionId,
1247 up_to_step: u32,
1248 ) -> Result<u64, DurableError> {
1249 let span = tracing::info_span!(
1250 "durable.journal.checkpoint",
1251 execution_id = %execution_id.as_uuid(),
1252 folded_count = tracing::field::Empty,
1253 );
1254 async move {
1255 let exec = execution_id.as_uuid().to_string();
1256 let rows: Vec<FoldableRowRead> = zeph_db::query_as(sql!(
1257 "SELECT step_id, idem_key, payload_version, payload FROM durable_journal
1258 WHERE execution_id = ? AND entry_kind = 'step_result'
1259 AND effect_class = 'idempotent' AND step_id < ?
1260 ORDER BY step_id"
1261 ))
1262 .bind(&exec)
1263 .bind(i64::from(up_to_step))
1264 .fetch_all(&self.pool)
1265 .await
1266 .map_err(|e| DurableError::storage("checkpoint", e))?;
1267 if rows.is_empty() {
1268 return Ok(0);
1269 }
1270
1271 let mut folded = self.open_foldable_steps(execution_id, rows)?;
1273 let lens: Vec<usize> = folded.iter().map(|s| s.payload.len()).collect();
1274 let take = crate::retention::fold_prefix_len(
1275 &lens,
1276 crate::retention::checkpoint_budget(self.max_payload_bytes),
1277 );
1278 if take == 0 {
1279 return Ok(0);
1282 }
1283 folded.truncate(take);
1284 let fold_end = folded.last().map_or(up_to_step, |s| s.step_id.saturating_add(1));
1285
1286 let snapshot = encode_checkpoint(&folded);
1287 let snap_aad =
1288 PayloadAad::new(execution_id, StepId::new(fold_end), EntryKindTag::Checkpoint, None);
1289 let sealed_snapshot = self.seal_payload(&snapshot, &snap_aad)?;
1290
1291 let count = folded.len() as u64;
1296
1297 let mut tx = zeph_db::begin_write(&self.pool)
1298 .await
1299 .map_err(|e| DurableError::storage("checkpoint", e))?;
1300 zeph_db::query(sql!(
1301 "INSERT INTO durable_journal
1302 (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at, folded_count)
1303 VALUES (?, ?, 'checkpoint', NULL, NULL, ?, ?, NULL, ?, ?)"
1304 ))
1305 .bind(&exec)
1306 .bind(i64::from(fold_end))
1307 .bind(sealed_snapshot)
1308 .bind(i32::from(crate::step::PAYLOAD_VERSION))
1309 .bind(now_unix_millis())
1310 .bind(i64::try_from(count).unwrap_or(i64::MAX))
1311 .execute(&mut *tx)
1312 .await
1313 .map_err(|e| DurableError::storage("checkpoint", e))?;
1314 zeph_db::query(sql!(
1315 "DELETE FROM durable_journal
1316 WHERE execution_id = ? AND entry_kind = 'step_result'
1317 AND effect_class = 'idempotent' AND step_id < ?"
1318 ))
1319 .bind(&exec)
1320 .bind(i64::from(fold_end))
1321 .execute(&mut *tx)
1322 .await
1323 .map_err(|e| DurableError::storage("checkpoint", e))?;
1324 tx.commit()
1325 .await
1326 .map_err(|e| DurableError::storage("checkpoint", e))?;
1327
1328 tracing::Span::current().record("folded_count", count);
1329 Ok(count)
1330 }
1331 .instrument(span)
1332 .await
1333 }
1334
1335 pub(crate) async fn read_checkpoints(
1348 &self,
1349 execution_id: ExecutionId,
1350 ) -> Result<Vec<JournalEntry>, DurableError> {
1351 let rows: Vec<(i64, Option<Vec<u8>>)> = zeph_db::query_as(sql!(
1352 "SELECT step_id, payload FROM durable_journal
1353 WHERE execution_id = ? AND entry_kind = 'checkpoint' ORDER BY step_id"
1354 ))
1355 .bind(execution_id.as_uuid().to_string())
1356 .fetch_all(&self.pool)
1357 .await
1358 .map_err(|e| DurableError::storage("read_checkpoints", e))?;
1359 if rows.is_empty() {
1360 return Ok(Vec::new());
1361 }
1362 let mut folded: CheckpointSnapshot = Vec::new();
1363 for (up_to, payload) in rows {
1364 let up_to = u32::try_from(up_to).map_err(|_| DurableError::Decode {
1365 context: "checkpoint up_to_step out of u32 range",
1366 })?;
1367 let sealed = payload.ok_or(DurableError::Decode {
1368 context: "checkpoint entry missing snapshot payload",
1369 })?;
1370 ensure_payload_within_limit(
1371 sealed.len(),
1372 self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1373 )?;
1374 let aad = PayloadAad::new(
1375 execution_id,
1376 StepId::new(up_to),
1377 EntryKindTag::Checkpoint,
1378 None,
1379 );
1380 let plaintext = self.open_payload(&sealed, &aad)?;
1381 folded.extend(decode_checkpoint(&plaintext)?);
1382 }
1383 let kind = self.lookup_kind(execution_id).await?;
1386 let entries = folded
1387 .into_iter()
1388 .map(|step| JournalEntry {
1389 seq: None,
1390 execution_id,
1391 kind,
1392 step_id: StepId::new(step.step_id),
1393 entry: EntryKind::StepResult {
1394 idempotency_key: IdempotencyKey::from_bytes(step.idem_key),
1395 payload: step.payload,
1396 effect: crate::EffectClass::Idempotent,
1397 payload_version: step.payload_version,
1398 },
1399 created_at_ms: 0,
1400 })
1401 .collect();
1402 Ok(entries)
1403 }
1404
1405 pub async fn find_unsealed_resumable_executions(
1418 &self,
1419 ) -> Result<Vec<ExecutionId>, DurableError> {
1420 let rows: Vec<(String,)> = zeph_db::query_as(sql!(
1421 "SELECT e.execution_id FROM durable_executions e
1422 WHERE e.status = 'running'
1423 AND NOT EXISTS (
1424 SELECT 1 FROM durable_execution_integrity i WHERE i.execution_id = e.execution_id
1425 )
1426 AND (
1427 EXISTS (
1428 SELECT 1 FROM durable_journal j
1429 WHERE j.execution_id = e.execution_id AND j.entry_kind = 'step_result'
1430 )
1431 OR EXISTS (
1432 SELECT 1 FROM durable_journal j
1433 WHERE j.execution_id = e.execution_id AND j.entry_kind = 'checkpoint'
1434 AND j.folded_count > 0
1435 )
1436 )"
1437 ))
1438 .fetch_all(&self.pool)
1439 .await
1440 .map_err(|e| DurableError::storage("seal_integrity_scan", e))?;
1441
1442 rows.into_iter()
1443 .map(|(id,)| {
1444 ExecutionId::parse_str(&id).map_err(|_| DurableError::Decode {
1445 context: "malformed execution_id in durable_executions",
1446 })
1447 })
1448 .collect()
1449 }
1450
1451 async fn committed_step_result_count(
1457 &self,
1458 execution_id: ExecutionId,
1459 ) -> Result<u64, DurableError> {
1460 let exec = execution_id.as_uuid().to_string();
1461 let live_count: i64 = zeph_db::query_scalar(sql!(
1462 "SELECT COUNT(*) FROM durable_journal
1463 WHERE execution_id = ? AND entry_kind = 'step_result'"
1464 ))
1465 .bind(&exec)
1466 .fetch_one(&self.pool)
1467 .await
1468 .map_err(|e| DurableError::storage("hwm_verify", e))?;
1469 let folded_sum: i64 = zeph_db::query_scalar(sql!(
1470 "SELECT COALESCE(SUM(folded_count), 0) FROM durable_journal
1471 WHERE execution_id = ? AND entry_kind = 'checkpoint'"
1472 ))
1473 .bind(&exec)
1474 .fetch_one(&self.pool)
1475 .await
1476 .map_err(|e| DurableError::storage("hwm_verify", e))?;
1477 Ok(u64::try_from(live_count.saturating_add(folded_sum)).unwrap_or(0))
1478 }
1479
1480 fn prepare_row(&self, entry: &JournalEntry) -> Result<JournalRow, DurableError> {
1482 let execution_id = entry.execution_id.as_uuid().to_string();
1483 let step_id = i64::from(entry.step_id.value());
1484 let created_at = entry.created_at_ms;
1485 let entry_kind = entry.entry.tag();
1486 match &entry.entry {
1487 EntryKind::StepResult {
1488 idempotency_key,
1489 payload,
1490 effect,
1491 payload_version,
1492 } => {
1493 ensure_payload_within_limit(payload.len(), self.max_payload_bytes)?;
1494 let aad = PayloadAad::new(
1495 entry.execution_id,
1496 entry.step_id,
1497 EntryKindTag::StepResult,
1498 Some(*idempotency_key),
1499 );
1500 let sealed = self.seal_payload(payload.as_ref(), &aad)?;
1501 Ok(JournalRow {
1502 execution_id,
1503 step_id,
1504 entry_kind,
1505 idem_key: Some(idempotency_key.as_bytes().to_vec()),
1506 effect_class: Some(effect.as_str()),
1507 payload: Some(sealed),
1508 payload_version: Some(i32::from(*payload_version)),
1509 hmac: None,
1510 created_at,
1511 })
1512 }
1513 EntryKind::EffectIntent {
1514 idempotency_key,
1515 effect,
1516 hmac: _,
1517 } => {
1518 let hmac = self.control_hmac(entry, Some(idempotency_key));
1521 Ok(JournalRow {
1522 execution_id,
1523 step_id,
1524 entry_kind,
1525 idem_key: Some(idempotency_key.as_bytes().to_vec()),
1526 effect_class: Some(effect.as_str()),
1527 payload: None,
1528 payload_version: None,
1529 hmac,
1530 created_at,
1531 })
1532 }
1533 EntryKind::PromiseCreated { .. }
1534 | EntryKind::PromiseResolved { .. }
1535 | EntryKind::TimerArmed { .. }
1536 | EntryKind::TimerFired { .. }
1537 | EntryKind::Checkpoint { .. } => {
1538 Err(DurableError::UnsupportedEntryKind { kind: entry_kind })
1539 }
1540 }
1541 }
1542
1543 async fn lookup_kind(&self, id: ExecutionId) -> Result<ExecutionKind, DurableError> {
1545 let kind: Option<String> = zeph_db::query_scalar(sql!(
1546 "SELECT kind FROM durable_executions WHERE execution_id = ?"
1547 ))
1548 .bind(id.as_uuid().to_string())
1549 .fetch_optional(&self.pool)
1550 .await
1551 .map_err(|e| DurableError::storage("read", e))?;
1552 let kind = kind.ok_or(DurableError::Decode {
1553 context: "journaled entries reference a missing execution row",
1554 })?;
1555 ExecutionKind::from_tag(&kind).ok_or(DurableError::Decode {
1556 context: "execution kind is not reconstructible (custom kind read-back unsupported)",
1557 })
1558 }
1559
1560 fn row_to_entry(
1562 &self,
1563 id: ExecutionId,
1564 kind: ExecutionKind,
1565 row: JournalRowRead,
1566 ) -> Result<JournalEntry, DurableError> {
1567 let (
1568 seq,
1569 step_id_raw,
1570 entry_kind,
1571 idem_key,
1572 effect_class,
1573 payload,
1574 payload_version,
1575 hmac,
1576 created_at,
1577 ) = row;
1578 let step_id =
1579 StepId::new(
1580 u32::try_from(step_id_raw).map_err(|_| DurableError::Decode {
1581 context: "step_id out of u32 range",
1582 })?,
1583 );
1584 let entry = match entry_kind.as_str() {
1585 "step_result" => {
1586 let idem_bytes = idem_key.ok_or(DurableError::Decode {
1587 context: "step_result idem_key missing",
1588 })?;
1589 let idem_key = IdempotencyKey::from_bytes(slice_to_array32(
1590 &idem_bytes,
1591 "step_result idem_key",
1592 )?);
1593 let effect = effect_class
1594 .as_deref()
1595 .and_then(crate::EffectClass::from_tag)
1596 .ok_or(DurableError::Decode {
1597 context: "step_result effect_class missing or invalid",
1598 })?;
1599 let sealed = payload.ok_or(DurableError::Decode {
1600 context: "step_result payload missing",
1601 })?;
1602 ensure_payload_within_limit(
1603 sealed.len(),
1604 self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1605 )?;
1606 let aad = PayloadAad::new(id, step_id, EntryKindTag::StepResult, Some(idem_key));
1607 let opened = self.open_payload(&sealed, &aad)?;
1608 let version = u8::try_from(payload_version.unwrap_or(1)).map_err(|_| {
1609 DurableError::Decode {
1610 context: "payload_version out of u8 range",
1611 }
1612 })?;
1613 EntryKind::StepResult {
1614 idempotency_key: idem_key,
1615 payload: opened,
1616 effect,
1617 payload_version: version,
1618 }
1619 }
1620 "effect_intent" => {
1621 let idem_bytes = idem_key.ok_or(DurableError::Decode {
1622 context: "effect_intent idem_key missing",
1623 })?;
1624 let idem_key = IdempotencyKey::from_bytes(slice_to_array32(
1625 &idem_bytes,
1626 "effect_intent idem_key",
1627 )?);
1628 let effect = effect_class
1629 .as_deref()
1630 .and_then(crate::EffectClass::from_tag)
1631 .ok_or(DurableError::Decode {
1632 context: "effect_intent effect_class missing or invalid",
1633 })?;
1634 let hmac = hmac
1635 .map(|bytes| slice_to_array32(&bytes, "effect_intent hmac"))
1636 .transpose()?;
1637 self.verify_control_hmac(
1638 id,
1639 step_id,
1640 EntryKindTag::EffectIntent.as_str(),
1641 Some(&idem_key),
1642 hmac,
1643 )?;
1644 EntryKind::EffectIntent {
1645 idempotency_key: idem_key,
1646 effect,
1647 hmac,
1648 }
1649 }
1650 "checkpoint" => self.checkpoint_entry(id, step_id, payload)?,
1651 other => {
1652 return Err(DurableError::UnsupportedEntryKind {
1653 kind: static_entry_tag(other),
1654 });
1655 }
1656 };
1657 Ok(JournalEntry {
1658 seq: Some(JournalSeq::new(seq)),
1659 execution_id: id,
1660 kind,
1661 step_id,
1662 entry,
1663 created_at_ms: created_at,
1664 })
1665 }
1666
1667 fn checkpoint_entry(
1672 &self,
1673 id: ExecutionId,
1674 step_id: StepId,
1675 payload: Option<Vec<u8>>,
1676 ) -> Result<EntryKind, DurableError> {
1677 let sealed = payload.ok_or(DurableError::Decode {
1678 context: "checkpoint entry missing snapshot payload",
1679 })?;
1680 ensure_payload_within_limit(
1681 sealed.len(),
1682 self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1683 )?;
1684 let aad = PayloadAad::new(id, step_id, EntryKindTag::Checkpoint, None);
1685 let snapshot = self.open_payload(&sealed, &aad)?;
1686 Ok(EntryKind::Checkpoint {
1687 up_to_step: step_id.value(),
1688 snapshot,
1689 })
1690 }
1691
1692 async fn rows_to_entries(
1694 &self,
1695 id: ExecutionId,
1696 rows: Vec<JournalRowRead>,
1697 ) -> Result<Vec<JournalEntry>, DurableError> {
1698 if rows.is_empty() {
1699 return Ok(Vec::new());
1700 }
1701 let kind = self.lookup_kind(id).await?;
1702 let mut entries = Vec::with_capacity(rows.len());
1703 for row in rows {
1704 entries.push(self.row_to_entry(id, kind, row)?);
1705 }
1706 Ok(entries)
1707 }
1708}
1709
1710impl LocalBackend {
1712 fn control_hmac(
1717 &self,
1718 entry: &JournalEntry,
1719 idem_key: Option<&IdempotencyKey>,
1720 ) -> Option<Vec<u8>> {
1721 self.compute_control_hmac(
1722 entry.execution_id,
1723 entry.step_id,
1724 entry.entry.tag(),
1725 idem_key,
1726 )
1727 .map(|h| h.to_vec())
1728 }
1729
1730 fn compute_control_hmac(
1735 &self,
1736 execution_id: ExecutionId,
1737 step_id: StepId,
1738 tag: &'static str,
1739 idem_key: Option<&IdempotencyKey>,
1740 ) -> Option<[u8; 32]> {
1741 let key = self.hmac_key.as_ref()?;
1742 Some(Self::keyed_control_hmac(
1743 key,
1744 execution_id,
1745 step_id,
1746 tag,
1747 idem_key,
1748 ))
1749 }
1750
1751 fn keyed_control_hmac(
1754 key: &[u8; 32],
1755 execution_id: ExecutionId,
1756 step_id: StepId,
1757 tag: &'static str,
1758 idem_key: Option<&IdempotencyKey>,
1759 ) -> [u8; 32] {
1760 let mut input = Vec::with_capacity(16 + 4 + 16 + 32);
1761 input.extend_from_slice(execution_id.as_bytes());
1762 input.extend_from_slice(&step_id.value().to_le_bytes());
1763 input.extend_from_slice(tag.as_bytes());
1764 if let Some(k) = idem_key {
1765 input.extend_from_slice(k.as_bytes());
1766 }
1767 *blake3::keyed_hash(key, &input).as_bytes()
1768 }
1769
1770 fn verify_control_hmac(
1791 &self,
1792 execution_id: ExecutionId,
1793 step_id: StepId,
1794 tag: &'static str,
1795 idem_key: Option<&IdempotencyKey>,
1796 stored: Option<[u8; 32]>,
1797 ) -> Result<(), DurableError> {
1798 let Some(current_key) = self.hmac_key.as_ref() else {
1799 return if stored.is_some() {
1800 Err(DurableError::ControlIntegrity)
1801 } else {
1802 Ok(())
1803 };
1804 };
1805 let Some(stored) = stored else {
1806 return Err(DurableError::ControlIntegrity);
1807 };
1808 let expected_current =
1809 Self::keyed_control_hmac(current_key, execution_id, step_id, tag, idem_key);
1810 if blake3::Hash::from(expected_current) == blake3::Hash::from(stored) {
1811 return Ok(());
1812 }
1813 if let Some(previous_key) = self.previous_hmac_key.as_ref() {
1814 let expected_previous =
1815 Self::keyed_control_hmac(previous_key, execution_id, step_id, tag, idem_key);
1816 if blake3::Hash::from(expected_previous) == blake3::Hash::from(stored) {
1817 return Ok(());
1818 }
1819 }
1820 Err(DurableError::ControlIntegrity)
1821 }
1822
1823 fn compute_hwm_hmac(
1831 execution_id: ExecutionId,
1832 max_committed_step_id: u32,
1833 committed_result_count: u64,
1834 key_epoch: u32,
1835 key: &[u8; 32],
1836 ) -> [u8; 32] {
1837 let mut input = Vec::with_capacity(16 + 4 + 8 + 4);
1838 input.extend_from_slice(execution_id.as_bytes());
1839 input.extend_from_slice(&max_committed_step_id.to_le_bytes());
1840 input.extend_from_slice(&committed_result_count.to_le_bytes());
1841 input.extend_from_slice(&key_epoch.to_le_bytes());
1842 *blake3::keyed_hash(key, &input).as_bytes()
1843 }
1844
1845 fn resolve_hwm_key(&self, epoch: u32) -> Option<[u8; 32]> {
1854 if let Some(slot) = &self.hwm_key
1855 && slot.epoch == epoch
1856 {
1857 return Some(slot.key);
1858 }
1859 if let Some(slot) = &self.hwm_key_previous
1860 && slot.epoch == epoch
1861 {
1862 return Some(slot.key);
1863 }
1864 None
1865 }
1866
1867 async fn bump_hwm_for_step_result(
1878 &self,
1879 tx: &mut zeph_db::DbTransaction<'_>,
1880 execution_id: ExecutionId,
1881 step_id: StepId,
1882 ) -> Result<(), DurableError> {
1883 let Some(slot) = &self.hwm_key else {
1884 return Ok(());
1885 };
1886 let exec = execution_id.as_uuid().to_string();
1887 let existing: Option<(i64, i64)> = zeph_db::query_as(sql!(
1888 "SELECT max_committed_step_id, committed_result_count
1889 FROM durable_execution_integrity WHERE execution_id = ?"
1890 ))
1891 .bind(&exec)
1892 .fetch_optional(&mut **tx)
1893 .await
1894 .map_err(|e| DurableError::storage("hwm_bump", e))?;
1895 let (prev_max, prev_count) = existing.unwrap_or((0, 0));
1896 let new_max = prev_max.max(i64::from(step_id.value()));
1897 let new_count = prev_count.saturating_add(1);
1898 let hmac = Self::compute_hwm_hmac(
1899 execution_id,
1900 u32::try_from(new_max).unwrap_or(u32::MAX),
1901 u64::try_from(new_count).unwrap_or(u64::MAX),
1902 slot.epoch,
1903 &slot.key,
1904 );
1905 zeph_db::query(sql!(
1906 "INSERT INTO durable_execution_integrity
1907 (execution_id, key_epoch, max_committed_step_id, committed_result_count, hwm_hmac, updated_at)
1908 VALUES (?, ?, ?, ?, ?, ?)
1909 ON CONFLICT(execution_id) DO UPDATE SET
1910 key_epoch = excluded.key_epoch,
1911 max_committed_step_id = excluded.max_committed_step_id,
1912 committed_result_count = excluded.committed_result_count,
1913 hwm_hmac = excluded.hwm_hmac,
1914 updated_at = excluded.updated_at"
1915 ))
1916 .bind(&exec)
1917 .bind(i64::from(slot.epoch))
1918 .bind(new_max)
1919 .bind(new_count)
1920 .bind(hmac.to_vec())
1921 .bind(now_unix_millis())
1922 .execute(&mut **tx)
1923 .await
1924 .map_err(|e| DurableError::storage("hwm_bump", e))?;
1925 Ok(())
1926 }
1927
1928 async fn verify_high_water_mark(&self, execution_id: ExecutionId) -> Result<(), DurableError> {
1936 if self.hwm_key.is_none() {
1937 return Ok(());
1938 }
1939 if let Err(error) = self.check_high_water_mark(execution_id).await {
1940 if let Err(finalize_error) = self.finalize(execution_id, ExecutionStatus::Aborted).await
1941 {
1942 tracing::warn!(
1943 error = %finalize_error,
1944 "failed to mark HWM-integrity-failed execution aborted"
1945 );
1946 }
1947 return Err(error);
1948 }
1949 Ok(())
1950 }
1951
1952 async fn check_high_water_mark(&self, execution_id: ExecutionId) -> Result<(), DurableError> {
1969 let exec = execution_id.as_uuid().to_string();
1970 let stored: Option<(i64, i64, i64, Vec<u8>)> = zeph_db::query_as(sql!(
1971 "SELECT key_epoch, max_committed_step_id, committed_result_count, hwm_hmac
1972 FROM durable_execution_integrity WHERE execution_id = ?"
1973 ))
1974 .bind(&exec)
1975 .fetch_optional(&self.pool)
1976 .await
1977 .map_err(|e| DurableError::storage("hwm_verify", e))?;
1978 let Some((epoch_raw, max_step_raw, count_raw, hmac)) = stored else {
1979 if self.hwm_key.is_some()
1980 && self.integrity_sealed
1981 && !self.integrity_grandfather.contains(&execution_id)
1982 && self.committed_step_result_count(execution_id).await? >= 1
1983 {
1984 return Err(DurableError::HighWaterMarkIntegrity {
1985 execution_id,
1986 reason: "integrity_row_absent_post_seal",
1987 hint: "TAMPER: this backend is sealed against pre-feature integrity-row \
1988 absence, this execution is keyed and not grandfathered, and it has \
1989 committed StepResults — a legitimate keyed execution can never reach \
1990 this state (the integrity row is written atomically with its first \
1991 committed StepResult), so an absent row here means the row was \
1992 deleted outside the write path",
1993 });
1994 }
1995 return Ok(());
1996 };
1997
1998 let fail =
2003 |reason: &'static str, hint: &'static str| DurableError::HighWaterMarkIntegrity {
2004 execution_id,
2005 reason,
2006 hint,
2007 };
2008 let tamper = |reason: &'static str| {
2009 fail(
2010 reason,
2011 "TAMPER: the signed high-water-mark did not authenticate under any key this \
2012 backend holds for the recorded epoch",
2013 )
2014 };
2015
2016 let epoch = u32::try_from(epoch_raw).map_err(|_| tamper("hmac_mismatch"))?;
2017 let Some(key) = self.resolve_hwm_key(epoch) else {
2018 return Err(fail(
2019 "key_epoch_unresolvable",
2020 "possibly re-keyed: this execution's signed key_epoch is neither the current key \
2021 nor a registered previous rotation key — if ZEPH_DURABLE_KEY was recently \
2022 rotated, ensure the rotation window is still open (ZEPH_DURABLE_KEY_PREVIOUS \
2023 present and [durable] previous_key_id set); the window is closed permanently by \
2024 `zeph durable rotate-key --drop-previous`. The durable resume path cannot \
2025 proceed without it (no interactive override)",
2026 ));
2027 };
2028 let stored_hmac =
2029 <[u8; 32]>::try_from(hmac.as_slice()).map_err(|_| tamper("hmac_mismatch"))?;
2030 let max_step = u32::try_from(max_step_raw).unwrap_or(u32::MAX);
2031 let count = u64::try_from(count_raw).unwrap_or(u64::MAX);
2032 let expected = Self::compute_hwm_hmac(execution_id, max_step, count, epoch, &key);
2033 if blake3::Hash::from(expected) != blake3::Hash::from(stored_hmac) {
2034 return Err(tamper("hmac_mismatch"));
2035 }
2036
2037 let recomputed = self.committed_step_result_count(execution_id).await?;
2038 if recomputed != count {
2039 return Err(fail(
2040 "count_mismatch",
2041 "TAMPER: the recomputed committed-result count (surviving StepResult rows plus \
2042 every checkpoint's folded_count) disagrees with the signed value — a committed \
2043 result was likely deleted outside the write path",
2044 ));
2045 }
2046 Ok(())
2047 }
2048}
2049
2050impl LocalBackend {
2052 fn seal_payload(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, DurableError> {
2054 match &self.cipher {
2055 Some(cipher) => Ok(cipher.seal(plaintext, aad)?),
2056 None => Ok(plaintext.to_vec()),
2057 }
2058 }
2059
2060 fn open_payload(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Bytes, DurableError> {
2062 match &self.cipher {
2063 Some(cipher) => Ok(Bytes::from(cipher.open(sealed, aad)?)),
2064 None => Ok(Bytes::copy_from_slice(sealed)),
2065 }
2066 }
2067}
2068
2069impl LocalBackend {
2071 pub async fn count_prunable(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
2080 let cutoffs = crate::retention::PruneCutoffs::from_policy(policy, now_unix_millis());
2081 let (count,): (i64,) = zeph_db::query_as(sql!(
2082 "SELECT COUNT(*) FROM durable_executions
2083 WHERE finalized_at IS NOT NULL
2084 AND ( (status = 'completed' AND finalized_at <= ?)
2085 OR (status IN ('failed', 'aborted', 'canceled') AND finalized_at <= ?) )"
2086 ))
2087 .bind(cutoffs.completed_before_ms)
2088 .bind(cutoffs.failed_before_ms)
2089 .fetch_one(&self.pool)
2090 .await
2091 .map_err(|e| DurableError::storage("count_prunable", e))?;
2092 Ok(count.max(0).cast_unsigned())
2093 }
2094
2095 pub async fn count_orphans(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
2108 if policy.stale_running_after_secs == 0 {
2109 return Ok(0);
2110 }
2111 let Some(lock_dir) = self.lock_dir.clone() else {
2112 return Ok(0);
2113 };
2114 let cutoff_ms = orphan_cutoff_ms(policy, now_unix_millis());
2115 let candidates: Vec<(String,)> = zeph_db::query_as(sql!(
2116 "SELECT execution_id FROM durable_executions WHERE status = 'running' AND updated_at <= ?"
2117 ))
2118 .bind(cutoff_ms)
2119 .fetch_all(&self.pool)
2120 .await
2121 .map_err(|e| DurableError::storage("count_orphans", e))?;
2122 let mut count = 0u64;
2123 for (exec_str,) in &candidates {
2124 let Ok(execution_id) = parse_execution_id(exec_str) else {
2125 continue;
2126 };
2127 if ExecutionLock::acquire(&lock_dir, execution_id).is_ok() {
2128 count += 1;
2129 }
2130 }
2131 Ok(count)
2132 }
2133
2134 pub async fn count_sealed_under_key_id(&self, key_id: u8) -> Result<u64, DurableError> {
2159 #[cfg(feature = "postgres")]
2160 {
2161 let key_id_param = i32::from(key_id);
2162 let (journal_count,): (i64,) = zeph_db::query_as(sql!(
2163 "SELECT COUNT(*) FROM durable_journal
2164 WHERE payload IS NOT NULL AND get_byte(payload, 0) = ?"
2165 ))
2166 .bind(key_id_param)
2167 .fetch_one(&self.pool)
2168 .await
2169 .map_err(|e| DurableError::storage("count_sealed_under_key_id", e))?;
2170 let (promises_count,): (i64,) = zeph_db::query_as(sql!(
2171 "SELECT COUNT(*) FROM durable_promises
2172 WHERE payload IS NOT NULL AND get_byte(payload, 0) = ?"
2173 ))
2174 .bind(key_id_param)
2175 .fetch_one(&self.pool)
2176 .await
2177 .map_err(|e| DurableError::storage("count_sealed_under_key_id", e))?;
2178 Ok((journal_count.max(0) + promises_count.max(0)).cast_unsigned())
2179 }
2180 #[cfg(not(feature = "postgres"))]
2181 {
2182 let key_byte = vec![key_id];
2183 let (journal_count,): (i64,) = zeph_db::query_as(sql!(
2184 "SELECT COUNT(*) FROM durable_journal
2185 WHERE payload IS NOT NULL AND substr(payload, 1, 1) = ?"
2186 ))
2187 .bind(key_byte.clone())
2188 .fetch_one(&self.pool)
2189 .await
2190 .map_err(|e| DurableError::storage("count_sealed_under_key_id", e))?;
2191 let (promises_count,): (i64,) = zeph_db::query_as(sql!(
2192 "SELECT COUNT(*) FROM durable_promises
2193 WHERE payload IS NOT NULL AND substr(payload, 1, 1) = ?"
2194 ))
2195 .bind(key_byte)
2196 .fetch_one(&self.pool)
2197 .await
2198 .map_err(|e| DurableError::storage("count_sealed_under_key_id", e))?;
2199 Ok((journal_count.max(0) + promises_count.max(0)).cast_unsigned())
2200 }
2201 }
2202
2203 pub async fn count_control_entries_under_previous_hmac(&self) -> Result<u64, DurableError> {
2240 let rows: Vec<ControlHmacScanRow> = zeph_db::query_as(sql!(
2241 "SELECT execution_id, step_id, idem_key, hmac
2242 FROM durable_journal
2243 WHERE entry_kind = 'effect_intent' AND hmac IS NOT NULL"
2244 ))
2245 .fetch_all(&self.pool)
2246 .await
2247 .map_err(|e| DurableError::storage("count_control_entries_under_previous_hmac", e))?;
2248
2249 if rows.is_empty() {
2250 return Ok(0);
2251 }
2252
2253 let (Some(current_key), Some(previous_key)) =
2254 (self.hmac_key.as_ref(), self.previous_hmac_key.as_ref())
2255 else {
2256 return Err(DurableError::ControlIntegrity);
2257 };
2258
2259 let mut count = 0u64;
2260 for (execution_id_raw, step_id_raw, idem_key_raw, hmac_raw) in rows {
2261 let Ok(execution_id) = parse_execution_id(&execution_id_raw) else {
2262 continue;
2263 };
2264 let Ok(step_id_value) = u32::try_from(step_id_raw) else {
2265 continue;
2266 };
2267 let step_id = StepId::new(step_id_value);
2268 let idem_key = idem_key_raw
2269 .as_deref()
2270 .and_then(|b| slice_to_array32(b, "effect_intent idem_key").ok())
2271 .map(IdempotencyKey::from_bytes);
2272 let Ok(stored) = slice_to_array32(&hmac_raw, "effect_intent hmac") else {
2273 continue;
2274 };
2275
2276 let tag = EntryKindTag::EffectIntent.as_str();
2277 let expected_current = Self::keyed_control_hmac(
2278 current_key,
2279 execution_id,
2280 step_id,
2281 tag,
2282 idem_key.as_ref(),
2283 );
2284 if blake3::Hash::from(expected_current) == blake3::Hash::from(stored) {
2285 continue;
2286 }
2287 let expected_previous = Self::keyed_control_hmac(
2288 previous_key,
2289 execution_id,
2290 step_id,
2291 tag,
2292 idem_key.as_ref(),
2293 );
2294 if blake3::Hash::from(expected_previous) == blake3::Hash::from(stored) {
2295 count += 1;
2296 }
2297 }
2298 Ok(count)
2299 }
2300
2301 pub async fn count_integrity_rows_under_epoch(&self, epoch: u32) -> Result<u64, DurableError> {
2324 let count: i64 = zeph_db::query_scalar(sql!(
2325 "SELECT COUNT(*) FROM durable_execution_integrity WHERE key_epoch = ?"
2326 ))
2327 .bind(i64::from(epoch))
2328 .fetch_one(&self.pool)
2329 .await
2330 .map_err(|e| DurableError::storage("count_integrity_rows_under_epoch", e))?;
2331 Ok(count.max(0).cast_unsigned())
2332 }
2333
2334 async fn delete_prune_batch(
2351 &self,
2352 cutoffs: crate::retention::PruneCutoffs,
2353 batch: u64,
2354 ) -> Result<u64, DurableError> {
2355 let mut tx = zeph_db::begin_write(&self.pool)
2356 .await
2357 .map_err(|e| DurableError::storage("prune", e))?;
2358
2359 #[cfg(feature = "postgres")]
2365 zeph_db::query(sql!(
2366 "SELECT execution_id FROM durable_executions
2367 WHERE finalized_at IS NOT NULL
2368 AND ( (status = 'completed' AND finalized_at <= ?)
2369 OR (status IN ('failed', 'aborted', 'canceled') AND finalized_at <= ?) )
2370 ORDER BY finalized_at LIMIT ?
2371 FOR UPDATE"
2372 ))
2373 .bind(cutoffs.completed_before_ms)
2374 .bind(cutoffs.failed_before_ms)
2375 .bind(i64::try_from(batch).unwrap_or(i64::MAX))
2376 .execute(&mut *tx)
2377 .await
2378 .map_err(|e| DurableError::storage("prune", e))?;
2379
2380 let ids: Vec<(String,)> = zeph_db::query_as(sql!(
2381 "SELECT execution_id FROM durable_executions
2382 WHERE finalized_at IS NOT NULL
2383 AND ( (status = 'completed' AND finalized_at <= ?)
2384 OR (status IN ('failed', 'aborted', 'canceled') AND finalized_at <= ?) )
2385 ORDER BY finalized_at LIMIT ?"
2386 ))
2387 .bind(cutoffs.completed_before_ms)
2388 .bind(cutoffs.failed_before_ms)
2389 .bind(i64::try_from(batch).unwrap_or(i64::MAX))
2390 .fetch_all(&mut *tx)
2391 .await
2392 .map_err(|e| DurableError::storage("prune", e))?;
2393 if ids.is_empty() {
2394 tx.commit()
2395 .await
2396 .map_err(|e| DurableError::storage("prune", e))?;
2397 return Ok(0);
2398 }
2399 let journal = sql!("DELETE FROM durable_journal WHERE execution_id = ?");
2400 let promises = sql!("DELETE FROM durable_promises WHERE execution_id = ?");
2401 let timers = sql!("DELETE FROM durable_timers WHERE execution_id = ?");
2402 let integrity = sql!("DELETE FROM durable_execution_integrity WHERE execution_id = ?");
2411 let executions = sql!(
2414 "DELETE FROM durable_executions
2415 WHERE execution_id = ?
2416 AND finalized_at IS NOT NULL
2417 AND ( (status = 'completed' AND finalized_at <= ?)
2418 OR (status IN ('failed', 'aborted', 'canceled') AND finalized_at <= ?) )"
2419 );
2420 let mut removed = 0u64;
2421 for (id,) in &ids {
2422 for stmt in [journal, promises, timers, integrity] {
2423 zeph_db::query(stmt)
2424 .bind(id)
2425 .execute(&mut *tx)
2426 .await
2427 .map_err(|e| DurableError::storage("prune", e))?;
2428 }
2429 let result = zeph_db::query(executions)
2430 .bind(id)
2431 .bind(cutoffs.completed_before_ms)
2432 .bind(cutoffs.failed_before_ms)
2433 .execute(&mut *tx)
2434 .await
2435 .map_err(|e| DurableError::storage("prune", e))?;
2436 removed += result.rows_affected();
2437 }
2438 tx.commit()
2439 .await
2440 .map_err(|e| DurableError::storage("prune", e))?;
2441 Ok(removed)
2442 }
2443
2444 async fn sweep_orphan_batch(
2463 &self,
2464 lock_dir: &std::path::Path,
2465 cutoff_ms: i64,
2466 batch: u64,
2467 cursor: Option<crate::retention::SweepCursor>,
2468 ) -> Result<crate::retention::SweepBatchOutcome, DurableError> {
2469 let (after_updated_at, after_exec) = cursor.map_or((i64::MIN, String::new()), |c| {
2473 (c.updated_at_ms, c.execution_id)
2474 });
2475
2476 let candidates: Vec<(String, i64)> = zeph_db::query_as(sql!(
2477 "SELECT execution_id, updated_at FROM durable_executions
2478 WHERE status = 'running' AND updated_at <= ?
2479 AND (updated_at > ? OR (updated_at = ? AND execution_id > ?))
2480 ORDER BY updated_at, execution_id LIMIT ?"
2481 ))
2482 .bind(cutoff_ms)
2483 .bind(after_updated_at)
2484 .bind(after_updated_at)
2485 .bind(&after_exec)
2486 .bind(i64::try_from(batch).unwrap_or(i64::MAX))
2487 .fetch_all(&self.pool)
2488 .await
2489 .map_err(|e| DurableError::storage("sweep_orphans", e))?;
2490
2491 let scanned = u64::try_from(candidates.len()).unwrap_or(u64::MAX);
2492 let next_cursor = candidates
2493 .last()
2494 .map(|(id, updated_at)| crate::retention::SweepCursor {
2495 updated_at_ms: *updated_at,
2496 execution_id: id.clone(),
2497 });
2498
2499 let now = now_unix_millis();
2500 let abort = sql!(
2501 "UPDATE durable_executions SET status = 'aborted', finalized_at = ?, updated_at = ?
2502 WHERE execution_id = ? AND status = 'running' AND finalized_at IS NULL"
2503 );
2504 let mut aborted = 0u64;
2505 for (exec_str, _updated_at) in &candidates {
2506 let Ok(execution_id) = parse_execution_id(exec_str) else {
2507 continue;
2508 };
2509 match ExecutionLock::acquire(lock_dir, execution_id) {
2510 Ok(_lock) => {
2511 let result = zeph_db::query(abort)
2512 .bind(now)
2513 .bind(now)
2514 .bind(exec_str)
2515 .execute(&self.pool)
2516 .await
2517 .map_err(|e| DurableError::storage("sweep_orphans", e))?;
2518 aborted += result.rows_affected();
2519 }
2521 Err(DurableError::ExecutionLocked { .. }) => {
2522 }
2524 Err(e) => return Err(e),
2525 }
2526 }
2527 Ok(crate::retention::SweepBatchOutcome {
2528 scanned,
2529 aborted,
2530 next_cursor,
2531 })
2532 }
2533}
2534
2535impl Journal for LocalBackend {
2536 async fn append(&self, entry: JournalEntry) -> Result<JournalSeq, DurableError> {
2537 let span = tracing::info_span!(
2538 "durable.journal.append",
2539 execution_id = %entry.execution_id.as_uuid(),
2540 step_id = entry.step_id.value(),
2541 entry_kind = entry.entry.tag(),
2542 );
2543 async move {
2544 let row = self.prepare_row(&entry)?;
2545 let insert = sql!(
2546 "INSERT INTO durable_journal
2547 (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
2548 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
2549 RETURNING seq"
2550 );
2551 let seq: i64 = if matches!(entry.entry, EntryKind::StepResult { .. }) {
2555 let mut tx = zeph_db::begin_write(&self.pool)
2556 .await
2557 .map_err(|e| DurableError::storage("append", e))?;
2558 let (seq,): (i64,) = zeph_db::query_as(insert)
2559 .bind(row.execution_id)
2560 .bind(row.step_id)
2561 .bind(row.entry_kind)
2562 .bind(row.idem_key)
2563 .bind(row.effect_class)
2564 .bind(row.payload)
2565 .bind(row.payload_version)
2566 .bind(row.hmac)
2567 .bind(row.created_at)
2568 .fetch_one(&mut *tx)
2569 .await
2570 .map_err(|e| DurableError::storage("append", e))?;
2571 self.bump_hwm_for_step_result(&mut tx, entry.execution_id, entry.step_id)
2572 .await?;
2573 tx.commit()
2574 .await
2575 .map_err(|e| DurableError::storage("append", e))?;
2576 seq
2577 } else {
2578 let (seq,): (i64,) = zeph_db::query_as(insert)
2579 .bind(row.execution_id)
2580 .bind(row.step_id)
2581 .bind(row.entry_kind)
2582 .bind(row.idem_key)
2583 .bind(row.effect_class)
2584 .bind(row.payload)
2585 .bind(row.payload_version)
2586 .bind(row.hmac)
2587 .bind(row.created_at)
2588 .fetch_one(&self.pool)
2589 .await
2590 .map_err(|e| DurableError::storage("append", e))?;
2591 seq
2592 };
2593 Ok(JournalSeq::new(seq))
2594 }
2595 .instrument(span)
2596 .await
2597 }
2598
2599 async fn read_execution(&self, id: ExecutionId) -> Result<Vec<JournalEntry>, DurableError> {
2600 let span = tracing::info_span!(
2601 "durable.journal.read",
2602 execution_id = %id.as_uuid(),
2603 step_count = tracing::field::Empty,
2604 );
2605 async move {
2606 let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
2607 "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
2608 FROM durable_journal WHERE execution_id = ? ORDER BY seq"
2609 ))
2610 .bind(id.as_uuid().to_string())
2611 .fetch_all(&self.pool)
2612 .await
2613 .map_err(|e| DurableError::storage("read", e))?;
2614 let entries = self.rows_to_entries(id, rows).await?;
2615 tracing::Span::current().record("step_count", entries.len());
2616 Ok(entries)
2617 }
2618 .instrument(span)
2619 .await
2620 }
2621
2622 async fn read_execution_range(
2623 &self,
2624 id: ExecutionId,
2625 from_step_id: u32,
2626 limit: usize,
2627 ) -> Result<Vec<JournalEntry>, DurableError> {
2628 let span = tracing::info_span!(
2629 "durable.journal.read_segment",
2630 execution_id = %id.as_uuid(),
2631 from_step_id,
2632 count = tracing::field::Empty,
2633 );
2634 async move {
2635 let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
2636 "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
2637 FROM durable_journal WHERE execution_id = ? AND step_id >= ? ORDER BY step_id, seq LIMIT ?"
2638 ))
2639 .bind(id.as_uuid().to_string())
2640 .bind(i64::from(from_step_id))
2641 .bind(i64::try_from(limit).unwrap_or(i64::MAX))
2642 .fetch_all(&self.pool)
2643 .await
2644 .map_err(|e| DurableError::storage("read_segment", e))?;
2645 let entries = self.rows_to_entries(id, rows).await?;
2646 tracing::Span::current().record("count", entries.len());
2647 Ok(entries)
2648 }
2649 .instrument(span)
2650 .await
2651 }
2652
2653 async fn finalize(&self, id: ExecutionId, status: ExecutionStatus) -> Result<(), DurableError> {
2654 let span = tracing::info_span!(
2655 "durable.journal.finalize",
2656 execution_id = %id.as_uuid(),
2657 status = status.as_str(),
2658 );
2659 async move {
2660 let now = now_unix_millis();
2661 let finalized_at = (!status.is_running()).then_some(now);
2662 let mut tx = zeph_db::begin_write(&self.pool)
2663 .await
2664 .map_err(|e| DurableError::storage("finalize", e))?;
2665 zeph_db::query(sql!(
2670 "UPDATE durable_executions SET status = ?, updated_at = ?, finalized_at = ?
2671 WHERE execution_id = ? AND status = 'running'"
2672 ))
2673 .bind(status.as_str())
2674 .bind(now)
2675 .bind(finalized_at)
2676 .bind(id.as_uuid().to_string())
2677 .execute(&mut *tx)
2678 .await
2679 .map_err(|e| DurableError::storage("finalize", e))?;
2680 tx.commit()
2681 .await
2682 .map_err(|e| DurableError::storage("finalize", e))?;
2683 Ok(())
2684 }
2685 .instrument(span)
2686 .await
2687 }
2688
2689 async fn prune(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
2690 let now = now_unix_millis();
2691 crate::retention::prune_in_batches(policy, now, |cutoffs, batch| {
2692 self.delete_prune_batch(cutoffs, batch)
2693 })
2694 .await
2695 }
2696
2697 async fn sweep_orphans(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
2699 if policy.stale_running_after_secs == 0 {
2700 return Ok(0);
2701 }
2702 let Some(lock_dir) = self.lock_dir.clone() else {
2703 if !self
2704 .orphan_sweep_warned
2705 .swap(true, std::sync::atomic::Ordering::Relaxed)
2706 {
2707 tracing::warn!(
2708 "durable: crash-orphan sweep requires an on-disk advisory-lock dir; orphan \
2709 reclamation disabled for this backend (Postgres/:memory:/non-Unix)"
2710 );
2711 }
2712 return Ok(0);
2713 };
2714 let cutoff_ms = orphan_cutoff_ms(policy, now_unix_millis());
2715 crate::retention::sweep_orphans_in_batches(
2716 policy.prune_batch_size,
2717 cutoff_ms,
2718 |cutoff, batch, cursor| self.sweep_orphan_batch(&lock_dir, cutoff, batch, cursor),
2719 )
2720 .await
2721 }
2722}
2723
2724impl crate::sealed::Sealed for LocalBackend {}
2725
2726impl ExecutionBackend for LocalBackend {
2727 fn capabilities(&self) -> BackendCapabilities {
2728 BackendCapabilities {
2729 parallel_steps: true,
2730 cross_process: cfg!(feature = "postgres"),
2732 max_payload: usize::try_from(self.max_payload_bytes).unwrap_or(usize::MAX),
2733 }
2734 }
2735
2736 async fn lookup_committed_result(
2737 &self,
2738 id: ExecutionId,
2739 idem_key: IdempotencyKey,
2740 ) -> Result<Option<JournalEntry>, DurableError> {
2741 LocalBackend::lookup_committed_result(self, id, idem_key).await
2742 }
2743}
2744
2745struct JournalRow {
2747 execution_id: String,
2748 step_id: i64,
2749 entry_kind: &'static str,
2750 idem_key: Option<Vec<u8>>,
2751 effect_class: Option<&'static str>,
2752 payload: Option<Vec<u8>>,
2753 payload_version: Option<i32>,
2754 hmac: Option<Vec<u8>>,
2755 created_at: i64,
2756}
2757
2758type JournalRowRead = (
2766 i64,
2767 i64,
2768 String,
2769 Option<Vec<u8>>,
2770 Option<String>,
2771 Option<Vec<u8>>,
2772 Option<i32>,
2773 Option<Vec<u8>>,
2774 i64,
2775);
2776
2777type ControlHmacScanRow = (String, i64, Option<Vec<u8>>, Vec<u8>);
2780
2781type PromiseRowRead = (String, Vec<u8>, i64, Option<Vec<u8>>);
2784
2785type FoldableRowRead = (i64, Option<Vec<u8>>, Option<i32>, Option<Vec<u8>>);
2788
2789#[cfg(feature = "sqlite")]
2796fn lock_dir_for_path(path: &str) -> Option<std::path::PathBuf> {
2797 (path != ":memory:").then(|| std::path::PathBuf::from(format!("{path}.locks")))
2798}
2799
2800#[cfg(not(feature = "sqlite"))]
2801fn lock_dir_for_path(_path: &str) -> Option<std::path::PathBuf> {
2802 None
2803}
2804
2805#[cfg(all(test, not(feature = "sqlite")))]
2811mod postgres_lock_dir_tests {
2812 use super::lock_dir_for_path;
2813
2814 #[test]
2815 fn postgres_url_never_derives_a_lock_dir() {
2816 assert_eq!(
2817 lock_dir_for_path("postgres://user:secret@host/db"),
2818 None,
2819 "a Postgres connection URL (which may embed credentials) must never be used to mint \
2820 an on-disk lock directory name"
2821 );
2822 assert_eq!(lock_dir_for_path(":memory:"), None);
2823 }
2824}
2825
2826pub(crate) fn now_unix_millis() -> i64 {
2828 SystemTime::now()
2829 .duration_since(UNIX_EPOCH)
2830 .map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX))
2831}
2832
2833fn record_prior_status(outcome: CancelOutcome) {
2839 match outcome {
2840 CancelOutcome::Canceled => {
2841 tracing::Span::current().record("prior_status", "running");
2842 }
2843 CancelOutcome::AlreadyTerminal { status } => {
2844 tracing::Span::current().record("prior_status", status.as_str());
2845 }
2846 CancelOutcome::NotFound
2847 | CancelOutcome::LiveOwner { .. }
2848 | CancelOutcome::LivenessUnverifiable => {}
2849 }
2850}
2851
2852fn orphan_cutoff_ms(policy: &RetentionPolicy, now_ms: i64) -> i64 {
2855 let threshold =
2856 i64::try_from(policy.stale_running_after_secs.saturating_mul(1000)).unwrap_or(i64::MAX);
2857 now_ms.saturating_sub(threshold)
2858}
2859
2860fn slice_to_array32(bytes: &[u8], field: &'static str) -> Result<[u8; 32], DurableError> {
2862 <[u8; 32]>::try_from(bytes).map_err(|_| DurableError::Decode { context: field })
2863}
2864
2865fn parse_execution_id(text: &str) -> Result<ExecutionId, DurableError> {
2867 uuid::Uuid::parse_str(text)
2868 .map(ExecutionId::from_uuid)
2869 .map_err(|_| DurableError::Decode {
2870 context: "execution_id is not a valid UUID",
2871 })
2872}
2873
2874fn parse_timer_id(text: &str) -> Result<TimerId, DurableError> {
2876 uuid::Uuid::parse_str(text)
2877 .map(TimerId::from_uuid)
2878 .map_err(|_| DurableError::Decode {
2879 context: "timer_id is not a valid UUID",
2880 })
2881}
2882
2883fn promise_payload_aad(execution_id: ExecutionId, promise_id: PromiseId) -> PayloadAad {
2888 let binding = IdempotencyKey::derive(
2889 execution_id,
2890 StepId::new(0),
2891 promise_id.as_uuid().as_bytes(),
2892 );
2893 PayloadAad::new(
2894 execution_id,
2895 StepId::new(0),
2896 EntryKindTag::PromiseResolved,
2897 Some(binding),
2898 )
2899}
2900
2901fn static_entry_tag(tag: &str) -> &'static str {
2903 match tag {
2904 "promise_created" => "promise_created",
2905 "promise_resolved" => "promise_resolved",
2906 "timer_armed" => "timer_armed",
2907 "timer_fired" => "timer_fired",
2908 "checkpoint" => "checkpoint",
2909 _ => "unknown",
2910 }
2911}
2912
2913#[cfg(all(test, feature = "sqlite"))]
2918mod tests {
2919 use std::assert_matches;
2920
2921 use super::*;
2922 use crate::cipher::CipherError;
2923 use crate::effect::EffectClass;
2924
2925 struct XorCipher;
2928 const XOR_MASK: u8 = 0x5A;
2929
2930 impl PayloadCipher for XorCipher {
2931 fn seal(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
2932 let tag = blake3::hash(&aad.canonical_bytes());
2933 let mut out = tag.as_bytes()[..8].to_vec();
2934 out.extend(plaintext.iter().map(|b| b ^ XOR_MASK));
2935 Ok(out)
2936 }
2937
2938 fn open(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
2939 if sealed.len() < 8 {
2940 return Err(CipherError::Malformed {
2941 context: "sealed blob shorter than the aad tag",
2942 });
2943 }
2944 let expected = blake3::hash(&aad.canonical_bytes());
2945 if sealed[..8] != expected.as_bytes()[..8] {
2946 return Err(CipherError::Authentication);
2947 }
2948 Ok(sealed[8..].iter().map(|b| b ^ XOR_MASK).collect())
2949 }
2950 }
2951
2952 struct RotatingKeyedCipher {
2960 current_id: u8,
2961 previous_id: Option<u8>,
2962 }
2963
2964 impl PayloadCipher for RotatingKeyedCipher {
2965 fn seal(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
2966 let tag = blake3::hash(&aad.canonical_bytes());
2967 let mut out = vec![self.current_id];
2968 out.extend_from_slice(&tag.as_bytes()[..8]);
2969 out.extend(plaintext.iter().map(|b| b ^ XOR_MASK));
2970 Ok(out)
2971 }
2972
2973 fn open(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
2974 if sealed.len() < 9 {
2975 return Err(CipherError::Malformed {
2976 context: "sealed blob shorter than the key-id + aad tag prefix",
2977 });
2978 }
2979 let id = sealed[0];
2980 if id != self.current_id && Some(id) != self.previous_id {
2981 return Err(CipherError::UnknownKeyId { key_id: id });
2982 }
2983 let expected = blake3::hash(&aad.canonical_bytes());
2984 if sealed[1..9] != expected.as_bytes()[..8] {
2985 return Err(CipherError::Authentication);
2986 }
2987 Ok(sealed[9..].iter().map(|b| b ^ XOR_MASK).collect())
2988 }
2989 }
2990
2991 async fn mem_backend(max_payload_bytes: u64) -> LocalBackend {
2992 let backend = LocalBackend::open(":memory:", max_payload_bytes)
2993 .await
2994 .expect("open in-memory backend");
2995 backend.init().await.expect("apply migrations");
2996 backend
2997 }
2998
2999 fn step_result(exec: ExecutionId, step: u32, payload: &[u8]) -> JournalEntry {
3000 let step_id = StepId::new(step);
3001 JournalEntry {
3002 seq: None,
3003 execution_id: exec,
3004 kind: ExecutionKind::AgentTurn,
3005 step_id,
3006 entry: EntryKind::StepResult {
3007 idempotency_key: IdempotencyKey::derive(exec, step_id, b"tool:read"),
3008 payload: Bytes::copy_from_slice(payload),
3009 effect: EffectClass::Idempotent,
3010 payload_version: 1,
3011 },
3012 created_at_ms: 100,
3013 }
3014 }
3015
3016 fn effect_intent(exec: ExecutionId, step: u32) -> JournalEntry {
3017 let step_id = StepId::new(step);
3018 JournalEntry {
3019 seq: None,
3020 execution_id: exec,
3021 kind: ExecutionKind::AgentTurn,
3022 step_id,
3023 entry: EntryKind::EffectIntent {
3024 idempotency_key: IdempotencyKey::derive(exec, step_id, b"transfer"),
3025 effect: EffectClass::ExactlyOnceGuarded,
3026 hmac: None,
3027 },
3028 created_at_ms: 100,
3029 }
3030 }
3031
3032 #[tokio::test]
3039 async fn count_sealed_under_key_id_counts_matching_journal_rows_and_excludes_control_entries() {
3040 let backend = mem_backend(1_048_576).await;
3041 let exec = ExecutionId::new();
3042 backend
3043 .open_execution(exec, ExecutionKind::AgentTurn)
3044 .await
3045 .unwrap();
3046
3047 backend
3048 .append(step_result(exec, 0, &[5, 0, 0]))
3049 .await
3050 .unwrap();
3051 backend
3052 .append(step_result(exec, 1, &[6, 0, 0]))
3053 .await
3054 .unwrap();
3055 backend.append(effect_intent(exec, 2)).await.unwrap();
3057
3058 assert_eq!(backend.count_sealed_under_key_id(5).await.unwrap(), 1);
3059 assert_eq!(backend.count_sealed_under_key_id(6).await.unwrap(), 1);
3060 assert_eq!(backend.count_sealed_under_key_id(7).await.unwrap(), 0);
3061 }
3062
3063 #[tokio::test]
3067 async fn count_sealed_under_key_id_counts_matching_promise_rows() {
3068 let backend = mem_backend(1_048_576).await;
3069 let exec = ExecutionId::new();
3070 backend
3071 .open_execution(exec, ExecutionKind::AgentTurn)
3072 .await
3073 .unwrap();
3074 let promise_id = PromiseId::new();
3075 backend
3076 .insert_promise(promise_id, exec, [0u8; 32], 100)
3077 .await
3078 .unwrap();
3079 assert_eq!(backend.count_sealed_under_key_id(9).await.unwrap(), 0);
3081
3082 backend
3083 .resolve_promise(promise_id, exec, &[9, 1, 2, 3], 200)
3084 .await
3085 .unwrap();
3086
3087 assert_eq!(backend.count_sealed_under_key_id(9).await.unwrap(), 1);
3088 assert_eq!(backend.count_sealed_under_key_id(10).await.unwrap(), 0);
3089 }
3090
3091 #[tokio::test]
3092 async fn open_execution_is_fresh_then_resume() {
3093 let backend = mem_backend(1_048_576).await;
3094 let exec = ExecutionId::new();
3095 assert!(
3096 !backend
3097 .open_execution(exec, ExecutionKind::AgentTurn)
3098 .await
3099 .unwrap()
3100 );
3101 assert!(
3102 backend
3103 .open_execution(exec, ExecutionKind::AgentTurn)
3104 .await
3105 .unwrap()
3106 );
3107 }
3108
3109 #[tokio::test]
3110 async fn open_execution_exclusive_is_fresh_then_resume() {
3111 let dir = tempfile::tempdir().unwrap();
3114 let db_path = dir.path().join("durable.db");
3115 let backend = LocalBackend::open(&db_path.to_string_lossy(), 1_048_576)
3116 .await
3117 .unwrap();
3118 backend.init().await.unwrap();
3119
3120 let exec = ExecutionId::new();
3121 let (is_resume, lock) = backend
3122 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3123 .await
3124 .unwrap();
3125 assert!(!is_resume);
3126 assert!(lock.is_some(), "a file-backed backend must derive a lock");
3127 drop(lock);
3128
3129 let (is_resume, _lock) = backend
3130 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3131 .await
3132 .unwrap();
3133 assert!(is_resume);
3134 }
3135
3136 #[tokio::test]
3141 async fn open_execution_exclusive_rejects_concurrent_second_holder() {
3142 let dir = tempfile::tempdir().unwrap();
3143 let db_path = dir.path().join("durable.db");
3144 let url = db_path.to_string_lossy().into_owned();
3145
3146 let backend_a = LocalBackend::open(&url, 1_048_576).await.unwrap();
3147 backend_a.init().await.unwrap();
3148 let backend_b = LocalBackend::open(&url, 1_048_576).await.unwrap();
3149
3150 let exec = ExecutionId::new();
3151 let (_, _lock_a) = backend_a
3152 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3153 .await
3154 .unwrap();
3155
3156 let err = backend_b
3157 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3158 .await
3159 .expect_err("a second concurrent holder must be rejected");
3160 assert!(
3161 matches!(err, DurableError::ExecutionLocked { execution_id, .. } if execution_id == exec),
3162 "expected ExecutionLocked, got {err:?}"
3163 );
3164 }
3165
3166 #[tokio::test]
3167 async fn open_execution_exclusive_on_memory_backend_returns_no_lock() {
3168 let backend = mem_backend(1_048_576).await;
3171 let exec = ExecutionId::new();
3172 let (is_resume, lock) = backend
3173 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3174 .await
3175 .unwrap();
3176 assert!(!is_resume);
3177 assert!(lock.is_none());
3178 }
3179
3180 #[tokio::test]
3181 async fn list_executions_summarizes_and_filters() {
3182 let backend = mem_backend(1_048_576).await;
3183 let turn = ExecutionId::new();
3184 let dag = ExecutionId::new();
3185 backend
3186 .open_execution(turn, ExecutionKind::AgentTurn)
3187 .await
3188 .unwrap();
3189 backend
3190 .open_execution(dag, ExecutionKind::DagRun)
3191 .await
3192 .unwrap();
3193 backend.append(step_result(turn, 0, b"a")).await.unwrap();
3194 backend.append(step_result(turn, 1, b"b")).await.unwrap();
3195 backend.append(step_result(dag, 0, b"c")).await.unwrap();
3196 backend
3197 .finalize(turn, ExecutionStatus::Completed)
3198 .await
3199 .unwrap();
3200
3201 let all = backend.list_executions(None, None, 10).await.unwrap();
3203 assert_eq!(all.len(), 2);
3204
3205 let turn_row = all
3206 .iter()
3207 .find(|e| e.execution_id == turn)
3208 .expect("turn present");
3209 assert_eq!(turn_row.kind, "agent_turn");
3210 assert_eq!(turn_row.status, ExecutionStatus::Completed);
3211 assert_eq!(turn_row.step_count, 2);
3212 assert!(turn_row.finalized_at_ms.is_some());
3213
3214 let dag_row = all
3215 .iter()
3216 .find(|e| e.execution_id == dag)
3217 .expect("dag present");
3218 assert_eq!(dag_row.status, ExecutionStatus::Running);
3219 assert_eq!(dag_row.step_count, 1);
3220 assert!(dag_row.finalized_at_ms.is_none());
3221
3222 let running = backend
3224 .list_executions(Some("running"), None, 10)
3225 .await
3226 .unwrap();
3227 assert_eq!(running.len(), 1);
3228 assert_eq!(running[0].execution_id, dag);
3229
3230 let dags = backend
3232 .list_executions(None, Some("dag_run"), 10)
3233 .await
3234 .unwrap();
3235 assert_eq!(dags.len(), 1);
3236 assert_eq!(dags[0].execution_id, dag);
3237
3238 let one = backend.list_executions(None, None, 1).await.unwrap();
3240 assert_eq!(one.len(), 1);
3241 }
3242
3243 #[tokio::test]
3244 async fn append_and_read_round_trips_step_result() {
3245 let backend = mem_backend(1_048_576).await;
3246 let exec = ExecutionId::new();
3247 backend
3248 .open_execution(exec, ExecutionKind::AgentTurn)
3249 .await
3250 .unwrap();
3251
3252 let seq = backend
3253 .append(step_result(exec, 0, b"hello"))
3254 .await
3255 .unwrap();
3256 assert_eq!(seq.value(), 1, "first append takes seq 1");
3257
3258 let entries = backend.read_execution(exec).await.unwrap();
3259 assert_eq!(entries.len(), 1);
3260 match &entries[0].entry {
3261 EntryKind::StepResult {
3262 payload, effect, ..
3263 } => {
3264 assert_eq!(payload.as_ref(), b"hello");
3265 assert_eq!(*effect, EffectClass::Idempotent);
3266 }
3267 other => panic!("unexpected entry kind: {other:?}"),
3268 }
3269 assert_eq!(entries[0].seq, Some(seq));
3270 }
3271
3272 #[tokio::test]
3273 async fn cipher_seals_payload_at_rest_but_round_trips() {
3274 let backend = mem_backend(1_048_576)
3275 .await
3276 .with_cipher(Arc::new(XorCipher));
3277 let exec = ExecutionId::new();
3278 backend
3279 .open_execution(exec, ExecutionKind::AgentTurn)
3280 .await
3281 .unwrap();
3282 backend
3283 .append(step_result(exec, 0, b"secret-payload"))
3284 .await
3285 .unwrap();
3286
3287 let (stored,): (Option<Vec<u8>>,) = zeph_db::query_as(sql!(
3289 "SELECT payload FROM durable_journal WHERE execution_id = ?"
3290 ))
3291 .bind(exec.as_uuid().to_string())
3292 .fetch_one(backend.pool())
3293 .await
3294 .unwrap();
3295 let stored = stored.expect("payload present");
3296 assert_ne!(
3297 stored.as_slice(),
3298 b"secret-payload",
3299 "payload must be sealed at rest"
3300 );
3301
3302 let entries = backend.read_execution(exec).await.unwrap();
3304 match &entries[0].entry {
3305 EntryKind::StepResult { payload, .. } => {
3306 assert_eq!(payload.as_ref(), b"secret-payload");
3307 }
3308 other => panic!("unexpected entry kind: {other:?}"),
3309 }
3310 }
3311
3312 #[tokio::test]
3313 async fn control_entry_hmac_is_stamped_only_when_keyed() {
3314 let exec = ExecutionId::new();
3315
3316 let unkeyed = mem_backend(1_048_576).await;
3317 unkeyed
3318 .open_execution(exec, ExecutionKind::AgentTurn)
3319 .await
3320 .unwrap();
3321 unkeyed.append(effect_intent(exec, 0)).await.unwrap();
3322 match &unkeyed.read_execution(exec).await.unwrap()[0].entry {
3323 EntryKind::EffectIntent { hmac, .. } => assert!(hmac.is_none()),
3324 other => panic!("unexpected entry kind: {other:?}"),
3325 }
3326
3327 let keyed = mem_backend(1_048_576).await.with_hmac_key([7u8; 32]);
3328 let exec2 = ExecutionId::new();
3329 keyed
3330 .open_execution(exec2, ExecutionKind::AgentTurn)
3331 .await
3332 .unwrap();
3333 keyed.append(effect_intent(exec2, 0)).await.unwrap();
3334 match &keyed.read_execution(exec2).await.unwrap()[0].entry {
3335 EntryKind::EffectIntent { hmac, .. } => {
3336 assert!(
3337 hmac.is_some(),
3338 "keyed backend stamps a row HMAC over control entries"
3339 );
3340 }
3341 other => panic!("unexpected entry kind: {other:?}"),
3342 }
3343 }
3344
3345 #[tokio::test]
3352 async fn read_execution_rejects_control_hmac_under_wrong_key() {
3353 let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
3354 let exec = ExecutionId::new();
3355 writer
3356 .open_execution(exec, ExecutionKind::AgentTurn)
3357 .await
3358 .unwrap();
3359 writer.append(effect_intent(exec, 0)).await.unwrap();
3360
3361 let wrong_key_reader =
3362 LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([2u8; 32]);
3363 assert_matches!(
3364 wrong_key_reader.read_execution(exec).await,
3365 Err(DurableError::ControlIntegrity)
3366 );
3367
3368 let right_key_reader =
3370 LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([1u8; 32]);
3371 assert!(right_key_reader.read_execution(exec).await.is_ok());
3372 }
3373
3374 #[tokio::test]
3379 async fn read_execution_rejects_missing_hmac_on_keyed_backend() {
3380 let writer = mem_backend(1_048_576).await;
3381 let exec = ExecutionId::new();
3382 writer
3383 .open_execution(exec, ExecutionKind::AgentTurn)
3384 .await
3385 .unwrap();
3386 writer.append(effect_intent(exec, 0)).await.unwrap();
3387
3388 let keyed_reader =
3389 LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([3u8; 32]);
3390 assert_matches!(
3391 keyed_reader.read_execution(exec).await,
3392 Err(DurableError::ControlIntegrity)
3393 );
3394 }
3395
3396 #[tokio::test]
3405 async fn read_execution_rejects_stamped_hmac_on_unkeyed_backend() {
3406 let writer = mem_backend(1_048_576).await.with_hmac_key([4u8; 32]);
3407 let exec = ExecutionId::new();
3408 writer
3409 .open_execution(exec, ExecutionKind::AgentTurn)
3410 .await
3411 .unwrap();
3412 writer.append(effect_intent(exec, 0)).await.unwrap();
3413
3414 let unkeyed_reader = LocalBackend::new(writer.pool().clone(), 1_048_576);
3415 assert_matches!(
3416 unkeyed_reader.read_execution(exec).await,
3417 Err(DurableError::ControlIntegrity)
3418 );
3419 }
3420
3421 #[tokio::test]
3428 async fn verify_control_hmac_accepts_row_under_previous_key_during_window() {
3429 let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
3430 let exec = ExecutionId::new();
3431 writer
3432 .open_execution(exec, ExecutionKind::AgentTurn)
3433 .await
3434 .unwrap();
3435 writer.append(effect_intent(exec, 0)).await.unwrap();
3436
3437 let post_rotation_reader = LocalBackend::new(writer.pool().clone(), 1_048_576)
3440 .with_hmac_key([2u8; 32])
3441 .with_previous_hmac_key([1u8; 32]);
3442 assert!(
3443 post_rotation_reader.read_execution(exec).await.is_ok(),
3444 "a row stamped under the previous key must verify during the rotation window"
3445 );
3446
3447 let post_rotation_writer = LocalBackend::new(writer.pool().clone(), 1_048_576)
3450 .with_hmac_key([2u8; 32])
3451 .with_previous_hmac_key([1u8; 32]);
3452 post_rotation_writer
3453 .append(effect_intent(exec, 1))
3454 .await
3455 .unwrap();
3456 assert!(post_rotation_writer.read_execution(exec).await.is_ok());
3457 }
3458
3459 #[tokio::test]
3463 async fn verify_control_hmac_rejects_row_under_neither_current_nor_previous_key() {
3464 let writer = mem_backend(1_048_576).await.with_hmac_key([9u8; 32]);
3465 let exec = ExecutionId::new();
3466 writer
3467 .open_execution(exec, ExecutionKind::AgentTurn)
3468 .await
3469 .unwrap();
3470 writer.append(effect_intent(exec, 0)).await.unwrap();
3471
3472 let unrelated_reader = LocalBackend::new(writer.pool().clone(), 1_048_576)
3473 .with_hmac_key([2u8; 32])
3474 .with_previous_hmac_key([3u8; 32]);
3475 assert_matches!(
3476 unrelated_reader.read_execution(exec).await,
3477 Err(DurableError::ControlIntegrity)
3478 );
3479 }
3480
3481 #[tokio::test]
3486 async fn count_control_entries_under_previous_hmac_counts_previous_only_rows() {
3487 let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
3488 let exec = ExecutionId::new();
3489 writer
3490 .open_execution(exec, ExecutionKind::AgentTurn)
3491 .await
3492 .unwrap();
3493 writer.append(effect_intent(exec, 0)).await.unwrap();
3495
3496 let scanner_mid_window = LocalBackend::new(writer.pool().clone(), 1_048_576)
3497 .with_hmac_key([2u8; 32])
3498 .with_previous_hmac_key([1u8; 32]);
3499 assert_eq!(
3500 scanner_mid_window
3501 .count_control_entries_under_previous_hmac()
3502 .await
3503 .unwrap(),
3504 1,
3505 "a row stamped under the previous key only must be counted"
3506 );
3507
3508 let post_rotation_writer = LocalBackend::new(writer.pool().clone(), 1_048_576)
3510 .with_hmac_key([2u8; 32])
3511 .with_previous_hmac_key([1u8; 32]);
3512 post_rotation_writer
3513 .append(effect_intent(exec, 1))
3514 .await
3515 .unwrap();
3516 assert_eq!(
3517 post_rotation_writer
3518 .count_control_entries_under_previous_hmac()
3519 .await
3520 .unwrap(),
3521 1,
3522 "the post-rotation row (verifies under current) must not add to the count"
3523 );
3524 }
3525
3526 #[tokio::test]
3530 async fn count_control_entries_under_previous_hmac_is_zero_on_empty_journal() {
3531 let backend = mem_backend(1_048_576).await;
3532 assert_eq!(
3533 backend
3534 .count_control_entries_under_previous_hmac()
3535 .await
3536 .unwrap(),
3537 0
3538 );
3539 }
3540
3541 #[tokio::test]
3546 async fn count_control_entries_under_previous_hmac_errors_when_keys_missing() {
3547 let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
3548 let exec = ExecutionId::new();
3549 writer
3550 .open_execution(exec, ExecutionKind::AgentTurn)
3551 .await
3552 .unwrap();
3553 writer.append(effect_intent(exec, 0)).await.unwrap();
3554
3555 let unkeyed_scanner = LocalBackend::new(writer.pool().clone(), 1_048_576);
3556 assert_matches!(
3557 unkeyed_scanner
3558 .count_control_entries_under_previous_hmac()
3559 .await,
3560 Err(DurableError::ControlIntegrity)
3561 );
3562
3563 let current_only_scanner =
3564 LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([1u8; 32]);
3565 assert_matches!(
3566 current_only_scanner
3567 .count_control_entries_under_previous_hmac()
3568 .await,
3569 Err(DurableError::ControlIntegrity)
3570 );
3571 }
3572
3573 #[tokio::test]
3574 async fn promise_and_timer_entries_fail_closed() {
3575 let backend = mem_backend(1_048_576).await;
3576 let exec = ExecutionId::new();
3577 backend
3578 .open_execution(exec, ExecutionKind::AgentTurn)
3579 .await
3580 .unwrap();
3581 let timer = JournalEntry {
3582 seq: None,
3583 execution_id: exec,
3584 kind: ExecutionKind::AgentTurn,
3585 step_id: StepId::new(0),
3586 entry: EntryKind::TimerArmed {
3587 timer_id: crate::TimerId::new(),
3588 due_at_ms: 1_000,
3589 hmac: None,
3590 },
3591 created_at_ms: 0,
3592 };
3593 assert_matches!(
3594 backend.append(timer).await,
3595 Err(DurableError::UnsupportedEntryKind {
3596 kind: "timer_armed"
3597 })
3598 );
3599 }
3600
3601 #[tokio::test]
3602 async fn payload_over_limit_is_rejected_fail_closed() {
3603 let backend = mem_backend(8).await;
3604 let exec = ExecutionId::new();
3605 backend
3606 .open_execution(exec, ExecutionKind::AgentTurn)
3607 .await
3608 .unwrap();
3609 let big = vec![0u8; 64];
3610 assert_matches!(
3611 backend.append(step_result(exec, 0, &big)).await,
3612 Err(DurableError::PayloadTooLarge { .. })
3613 );
3614 }
3615
3616 #[tokio::test]
3617 async fn finalize_marks_terminal_status_and_time() {
3618 let backend = mem_backend(1_048_576).await;
3619 let exec = ExecutionId::new();
3620 backend
3621 .open_execution(exec, ExecutionKind::AgentTurn)
3622 .await
3623 .unwrap();
3624 backend
3625 .finalize(exec, ExecutionStatus::Completed)
3626 .await
3627 .unwrap();
3628
3629 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3630 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3631 ))
3632 .bind(exec.as_uuid().to_string())
3633 .fetch_one(backend.pool())
3634 .await
3635 .unwrap();
3636 assert_eq!(status, "completed");
3637 assert!(finalized.is_some(), "a terminal status stamps finalized_at");
3638 }
3639
3640 #[tokio::test]
3641 async fn finalize_is_a_noop_once_already_terminal() {
3642 let backend = mem_backend(1_048_576).await;
3645 let exec = ExecutionId::new();
3646 backend
3647 .open_execution(exec, ExecutionKind::AgentTurn)
3648 .await
3649 .unwrap();
3650 backend
3651 .finalize(exec, ExecutionStatus::Completed)
3652 .await
3653 .unwrap();
3654
3655 backend
3657 .finalize(exec, ExecutionStatus::Failed)
3658 .await
3659 .unwrap();
3660
3661 let (status,): (String,) = zeph_db::query_as(sql!(
3662 "SELECT status FROM durable_executions WHERE execution_id = ?"
3663 ))
3664 .bind(exec.as_uuid().to_string())
3665 .fetch_one(backend.pool())
3666 .await
3667 .unwrap();
3668 assert_eq!(
3669 status, "completed",
3670 "the first terminal status must stick; a later finalize call is a no-op"
3671 );
3672 }
3673
3674 #[tokio::test]
3675 async fn finalize_after_abort_is_a_noop() {
3676 let backend = mem_backend(1_048_576).await;
3680 let exec = ExecutionId::new();
3681 backend
3682 .open_execution(exec, ExecutionKind::AgentTurn)
3683 .await
3684 .unwrap();
3685 backend
3686 .finalize(exec, ExecutionStatus::Aborted)
3687 .await
3688 .unwrap();
3689
3690 backend
3691 .finalize(exec, ExecutionStatus::Completed)
3692 .await
3693 .unwrap();
3694
3695 let (status,): (String,) = zeph_db::query_as(sql!(
3696 "SELECT status FROM durable_executions WHERE execution_id = ?"
3697 ))
3698 .bind(exec.as_uuid().to_string())
3699 .fetch_one(backend.pool())
3700 .await
3701 .unwrap();
3702 assert_eq!(
3703 status, "aborted",
3704 "an aborted execution must not be overwritten by a later Completed/Failed call"
3705 );
3706 }
3707
3708 #[tokio::test]
3709 async fn reopening_a_finalized_execution_resets_it_to_running() {
3710 let backend = mem_backend(1_048_576).await;
3714 let exec = ExecutionId::new();
3715 backend
3716 .open_execution(exec, ExecutionKind::AgentTurn)
3717 .await
3718 .unwrap();
3719 backend
3720 .finalize(exec, ExecutionStatus::Completed)
3721 .await
3722 .unwrap();
3723
3724 let is_resume = backend
3725 .open_execution(exec, ExecutionKind::AgentTurn)
3726 .await
3727 .unwrap();
3728 assert!(is_resume, "the row already existed, so this is a resume");
3729
3730 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3731 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3732 ))
3733 .bind(exec.as_uuid().to_string())
3734 .fetch_one(backend.pool())
3735 .await
3736 .unwrap();
3737 assert_eq!(
3738 status, "running",
3739 "reopening a completed execution must un-finalize it"
3740 );
3741 assert!(
3742 finalized.is_none(),
3743 "reopening must clear the stale finalized_at"
3744 );
3745 }
3746
3747 #[tokio::test]
3748 async fn reopening_a_failed_execution_resets_it_to_running() {
3749 let backend = mem_backend(1_048_576).await;
3753 let exec = ExecutionId::new();
3754 backend
3755 .open_execution(exec, ExecutionKind::AgentTurn)
3756 .await
3757 .unwrap();
3758 backend
3759 .finalize(exec, ExecutionStatus::Failed)
3760 .await
3761 .unwrap();
3762
3763 let is_resume = backend
3764 .open_execution(exec, ExecutionKind::AgentTurn)
3765 .await
3766 .unwrap();
3767 assert!(is_resume, "the row already existed, so this is a resume");
3768
3769 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3770 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3771 ))
3772 .bind(exec.as_uuid().to_string())
3773 .fetch_one(backend.pool())
3774 .await
3775 .unwrap();
3776 assert_eq!(
3777 status, "running",
3778 "reopening a failed execution must un-finalize it"
3779 );
3780 assert!(
3781 finalized.is_none(),
3782 "reopening must clear the stale finalized_at"
3783 );
3784 }
3785
3786 #[tokio::test]
3787 async fn reopening_an_aborted_execution_un_finalizes_it() {
3788 let backend = mem_backend(1_048_576).await;
3795 let exec = ExecutionId::new();
3796 backend
3797 .open_execution(exec, ExecutionKind::AgentTurn)
3798 .await
3799 .unwrap();
3800 backend
3801 .finalize(exec, ExecutionStatus::Aborted)
3802 .await
3803 .unwrap();
3804
3805 let is_resume = backend
3806 .open_execution(exec, ExecutionKind::AgentTurn)
3807 .await
3808 .unwrap();
3809 assert!(is_resume, "the row already existed, so this is a resume");
3810
3811 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3812 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3813 ))
3814 .bind(exec.as_uuid().to_string())
3815 .fetch_one(backend.pool())
3816 .await
3817 .unwrap();
3818 assert_eq!(
3819 status, "running",
3820 "reopening an aborted execution must un-finalize it (INV-16)"
3821 );
3822 assert!(
3823 finalized.is_none(),
3824 "reopening must clear the stale finalized_at"
3825 );
3826 }
3827
3828 #[tokio::test]
3829 async fn cancel_execution_with_no_live_owner_cancels_immediately() {
3830 let backend = mem_backend(1_048_576).await;
3833 let exec = ExecutionId::new();
3834 backend
3835 .open_execution(exec, ExecutionKind::AgentTurn)
3836 .await
3837 .unwrap();
3838
3839 let outcome = backend.cancel_execution(exec).await.unwrap();
3840 assert_eq!(outcome, CancelOutcome::Canceled);
3841
3842 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3843 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3844 ))
3845 .bind(exec.as_uuid().to_string())
3846 .fetch_one(backend.pool())
3847 .await
3848 .unwrap();
3849 assert_eq!(status, "canceled");
3850 assert!(finalized.is_some(), "a terminal status stamps finalized_at");
3851 }
3852
3853 #[tokio::test]
3854 async fn cancel_execution_with_no_live_owner_on_file_backed_pool_cancels_immediately() {
3855 let dir = tempfile::tempdir().unwrap();
3858 let backend =
3859 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
3860 .await
3861 .unwrap();
3862 backend.init().await.unwrap();
3863
3864 let exec = ExecutionId::new();
3865 backend
3866 .open_execution(exec, ExecutionKind::AgentTurn)
3867 .await
3868 .unwrap();
3869
3870 let outcome = backend.cancel_execution(exec).await.unwrap();
3871 assert_eq!(outcome, CancelOutcome::Canceled);
3872
3873 let lock_dir = backend.lock_dir.clone().unwrap();
3875 assert!(ExecutionLock::acquire(&lock_dir, exec).is_ok());
3876 }
3877
3878 #[tokio::test]
3879 async fn cancel_execution_refuses_a_live_owner_without_touching_the_row() {
3880 let dir = tempfile::tempdir().unwrap();
3883 let db_path = dir.path().join("durable.db");
3884 let url = db_path.to_string_lossy().into_owned();
3885
3886 let owner = LocalBackend::open(&url, 1_048_576).await.unwrap();
3887 owner.init().await.unwrap();
3888 let canceler = LocalBackend::open(&url, 1_048_576).await.unwrap();
3889
3890 let exec = ExecutionId::new();
3891 let (_, _lock) = owner
3892 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3893 .await
3894 .unwrap();
3895
3896 let outcome = canceler.cancel_execution(exec).await.unwrap();
3897 assert!(
3898 matches!(outcome, CancelOutcome::LiveOwner { pid } if pid == std::process::id()),
3899 "expected LiveOwner{{pid: {}}}, got {outcome:?}",
3900 std::process::id()
3901 );
3902
3903 let (status,): (String,) = zeph_db::query_as(sql!(
3904 "SELECT status FROM durable_executions WHERE execution_id = ?"
3905 ))
3906 .bind(exec.as_uuid().to_string())
3907 .fetch_one(owner.pool())
3908 .await
3909 .unwrap();
3910 assert_eq!(status, "running", "a live-owned row must never be touched");
3911 }
3912
3913 #[tokio::test]
3914 async fn cancel_execution_is_idempotent_on_a_second_call() {
3915 let backend = mem_backend(1_048_576).await;
3917 let exec = ExecutionId::new();
3918 backend
3919 .open_execution(exec, ExecutionKind::AgentTurn)
3920 .await
3921 .unwrap();
3922
3923 assert_eq!(
3924 backend.cancel_execution(exec).await.unwrap(),
3925 CancelOutcome::Canceled
3926 );
3927 let second = backend.cancel_execution(exec).await.unwrap();
3928 assert_eq!(
3929 second,
3930 CancelOutcome::AlreadyTerminal {
3931 status: ExecutionStatus::Canceled
3932 }
3933 );
3934 }
3935
3936 #[tokio::test]
3937 async fn cancel_execution_on_each_other_terminal_status_is_already_terminal() {
3938 for status in [
3939 ExecutionStatus::Completed,
3940 ExecutionStatus::Failed,
3941 ExecutionStatus::Aborted,
3942 ] {
3943 let backend = mem_backend(1_048_576).await;
3944 let exec = ExecutionId::new();
3945 backend
3946 .open_execution(exec, ExecutionKind::AgentTurn)
3947 .await
3948 .unwrap();
3949 backend.finalize(exec, status).await.unwrap();
3950
3951 let outcome = backend.cancel_execution(exec).await.unwrap();
3952 assert_eq!(
3953 outcome,
3954 CancelOutcome::AlreadyTerminal { status },
3955 "canceling a {status:?} execution must be a no-op reporting its own status"
3956 );
3957 }
3958 }
3959
3960 #[tokio::test]
3961 async fn cancel_execution_on_unknown_id_returns_not_found() {
3962 let backend = mem_backend(1_048_576).await;
3963 let outcome = backend.cancel_execution(ExecutionId::new()).await.unwrap();
3964 assert_eq!(outcome, CancelOutcome::NotFound);
3965 }
3966
3967 #[tokio::test]
3968 async fn cancel_execution_races_finalize_exactly_one_terminal_status_wins() {
3969 let dir = tempfile::tempdir().unwrap();
3978 let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
3979 let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
3980 backend.init().await.unwrap();
3981
3982 for _ in 0..20 {
3983 let exec = ExecutionId::new();
3984 backend
3985 .open_execution(exec, ExecutionKind::AgentTurn)
3986 .await
3987 .unwrap();
3988
3989 let cancel_backend = backend.clone();
3990 let cancel = tokio::spawn(async move { cancel_backend.cancel_execution(exec).await });
3991 let finalize_backend = backend.clone();
3992 let finalize = tokio::spawn(async move {
3993 finalize_backend
3994 .finalize(exec, ExecutionStatus::Completed)
3995 .await
3996 });
3997
3998 let (cancel_result, finalize_result) = tokio::join!(cancel, finalize);
3999 let cancel_outcome = cancel_result
4000 .expect("cancel task must not panic")
4001 .expect("cancel_execution must not error under a concurrent finalize");
4002 finalize_result
4003 .expect("finalize task must not panic")
4004 .expect("finalize must not error under a concurrent cancel");
4005
4006 let (status,): (String,) = zeph_db::query_as(sql!(
4007 "SELECT status FROM durable_executions WHERE execution_id = ?"
4008 ))
4009 .bind(exec.as_uuid().to_string())
4010 .fetch_one(backend.pool())
4011 .await
4012 .unwrap();
4013
4014 match cancel_outcome {
4018 CancelOutcome::Canceled => assert_eq!(
4019 status, "canceled",
4020 "cancel_execution won the race — the row must be canceled"
4021 ),
4022 CancelOutcome::AlreadyTerminal {
4023 status: ExecutionStatus::Completed,
4024 } => assert_eq!(
4025 status, "completed",
4026 "finalize won the race — the row must be completed, and cancel's own \
4027 guarded UPDATE must have found it already non-running"
4028 ),
4029 other => panic!(
4030 "cancel_execution must only ever win or lose cleanly against a concurrent \
4031 finalize, got {other:?}"
4032 ),
4033 }
4034 }
4035 }
4036
4037 #[tokio::test]
4038 async fn cancel_execution_races_sweep_orphans_exactly_one_of_canceled_or_aborted_wins() {
4039 let dir = tempfile::tempdir().unwrap();
4048 let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
4049 let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
4050 backend.init().await.unwrap();
4051
4052 let policy = RetentionPolicy {
4053 stale_running_after_secs: 1,
4054 prune_batch_size: 10,
4055 ..RetentionPolicy::default()
4056 };
4057
4058 for _ in 0..20 {
4059 let exec = ExecutionId::new();
4060 backend
4061 .open_execution(exec, ExecutionKind::AgentTurn)
4062 .await
4063 .unwrap();
4064 backdate_updated_at(&backend, exec, 0).await;
4065
4066 let cancel_backend = backend.clone();
4067 let cancel = tokio::spawn(async move { cancel_backend.cancel_execution(exec).await });
4068 let sweep_backend = backend.clone();
4069 let policy_for_task = policy.clone();
4070 let sweep =
4071 tokio::spawn(async move { sweep_backend.sweep_orphans(&policy_for_task).await });
4072
4073 let (cancel_result, sweep_result) = tokio::join!(cancel, sweep);
4074 let cancel_outcome = cancel_result
4075 .expect("cancel task must not panic")
4076 .expect("cancel_execution must not error under a concurrent sweep");
4077 let aborted = sweep_result
4078 .expect("sweep task must not panic")
4079 .expect("sweep_orphans must not error under a concurrent cancel");
4080
4081 let (status,): (String,) = zeph_db::query_as(sql!(
4082 "SELECT status FROM durable_executions WHERE execution_id = ?"
4083 ))
4084 .bind(exec.as_uuid().to_string())
4085 .fetch_one(backend.pool())
4086 .await
4087 .unwrap();
4088
4089 match cancel_outcome {
4090 CancelOutcome::Canceled => {
4091 assert_eq!(aborted, 0, "cancel won the lock — sweep must skip this row");
4092 assert_eq!(status, "canceled");
4093 }
4094 CancelOutcome::LiveOwner { .. } => {
4095 assert_eq!(aborted, 1, "sweep won the lock — it must abort this row");
4096 assert_eq!(status, "aborted");
4097 }
4098 other => panic!(
4099 "cancel_execution must only ever win the lock (Canceled) or lose it \
4100 (LiveOwner) against a concurrent sweep, got {other:?}"
4101 ),
4102 }
4103 }
4104 }
4105
4106 #[tokio::test]
4107 async fn open_execution_on_canceled_row_fails_closed_and_never_resumes() {
4108 let backend = mem_backend(1_048_576).await;
4112 let exec = ExecutionId::new();
4113 backend
4114 .open_execution(exec, ExecutionKind::AgentTurn)
4115 .await
4116 .unwrap();
4117 let outcome = backend.cancel_execution(exec).await.unwrap();
4118 assert_eq!(outcome, CancelOutcome::Canceled);
4119
4120 let err = backend
4121 .open_execution(exec, ExecutionKind::AgentTurn)
4122 .await
4123 .expect_err("reopening a canceled execution must fail closed");
4124 assert!(
4125 matches!(err, DurableError::ExecutionCanceled { execution_id } if execution_id == exec),
4126 "expected ExecutionCanceled, got {err:?}"
4127 );
4128
4129 let (status,): (String,) = zeph_db::query_as(sql!(
4130 "SELECT status FROM durable_executions WHERE execution_id = ?"
4131 ))
4132 .bind(exec.as_uuid().to_string())
4133 .fetch_one(backend.pool())
4134 .await
4135 .unwrap();
4136 assert_eq!(
4137 status, "canceled",
4138 "the row must never be reset to running by a reopen attempt"
4139 );
4140 }
4141
4142 #[tokio::test]
4143 async fn open_execution_exclusive_on_canceled_row_fails_closed_with_lock_released() {
4144 let dir = tempfile::tempdir().unwrap();
4147 let db_path = dir.path().join("durable.db");
4148 let backend = LocalBackend::open(&db_path.to_string_lossy(), 1_048_576)
4149 .await
4150 .unwrap();
4151 backend.init().await.unwrap();
4152
4153 let exec = ExecutionId::new();
4154 backend
4155 .open_execution(exec, ExecutionKind::AgentTurn)
4156 .await
4157 .unwrap();
4158 assert_eq!(
4159 backend.cancel_execution(exec).await.unwrap(),
4160 CancelOutcome::Canceled
4161 );
4162
4163 let err = backend
4164 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
4165 .await
4166 .expect_err("reopening a canceled execution exclusively must fail closed");
4167 assert!(matches!(err, DurableError::ExecutionCanceled { .. }));
4168
4169 let dir2 = backend.lock_dir.clone().unwrap();
4171 assert!(ExecutionLock::acquire(&dir2, exec).is_ok());
4172 }
4173
4174 #[tokio::test]
4175 async fn reopen_of_a_row_deleted_out_from_under_it_starts_fresh() {
4176 let backend = mem_backend(1_048_576).await;
4182 let exec = ExecutionId::new();
4183 backend
4184 .open_execution(exec, ExecutionKind::AgentTurn)
4185 .await
4186 .unwrap();
4187 backend
4188 .finalize(exec, ExecutionStatus::Completed)
4189 .await
4190 .unwrap();
4191
4192 zeph_db::query(sql!(
4194 "DELETE FROM durable_executions WHERE execution_id = ?"
4195 ))
4196 .bind(exec.as_uuid().to_string())
4197 .execute(backend.pool())
4198 .await
4199 .unwrap();
4200
4201 let is_resume = backend
4202 .open_execution(exec, ExecutionKind::AgentTurn)
4203 .await
4204 .unwrap();
4205 assert!(
4206 !is_resume,
4207 "a row deleted by a concurrent prune must be reported as a fresh execution, not a resume"
4208 );
4209
4210 let (status,): (String,) = zeph_db::query_as(sql!(
4211 "SELECT status FROM durable_executions WHERE execution_id = ?"
4212 ))
4213 .bind(exec.as_uuid().to_string())
4214 .fetch_one(backend.pool())
4215 .await
4216 .unwrap();
4217 assert_eq!(status, "running", "the fresh row starts running");
4218 }
4219
4220 #[tokio::test]
4221 async fn prune_does_not_delete_a_row_reopened_since_it_was_finalized() {
4222 let backend = mem_backend(1_048_576).await;
4226 let exec = ExecutionId::new();
4227 backend
4228 .open_execution(exec, ExecutionKind::AgentTurn)
4229 .await
4230 .unwrap();
4231 backend.append(step_result(exec, 0, b"x")).await.unwrap();
4232 zeph_db::query(sql!(
4233 "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
4234 ))
4235 .bind(exec.as_uuid().to_string())
4236 .execute(backend.pool())
4237 .await
4238 .unwrap();
4239
4240 let is_resume = backend
4242 .open_execution(exec, ExecutionKind::AgentTurn)
4243 .await
4244 .unwrap();
4245 assert!(is_resume);
4246
4247 let policy = RetentionPolicy {
4248 ttl_completed_secs: 1,
4249 prune_batch_size: 10,
4250 ..RetentionPolicy::default()
4251 };
4252 let deleted = backend.prune(&policy).await.unwrap();
4253 assert_eq!(
4254 deleted, 0,
4255 "a reopened (un-finalized) execution must not be pruned"
4256 );
4257 assert_eq!(
4258 backend.read_execution(exec).await.unwrap().len(),
4259 1,
4260 "the execution's journal must survive"
4261 );
4262 }
4263
4264 #[tokio::test]
4265 async fn concurrent_prune_and_reopen_never_lose_or_corrupt_the_row() {
4266 let dir = tempfile::tempdir().unwrap();
4281 let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
4282 let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
4283 backend.init().await.unwrap();
4284
4285 let policy = RetentionPolicy {
4286 ttl_completed_secs: 1,
4287 prune_batch_size: 10,
4288 ..RetentionPolicy::default()
4289 };
4290
4291 for _ in 0..20 {
4292 let exec = ExecutionId::new();
4293 backend
4294 .open_execution(exec, ExecutionKind::AgentTurn)
4295 .await
4296 .unwrap();
4297 backend.append(step_result(exec, 0, b"x")).await.unwrap();
4298 zeph_db::query(sql!(
4300 "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
4301 ))
4302 .bind(exec.as_uuid().to_string())
4303 .execute(backend.pool())
4304 .await
4305 .unwrap();
4306
4307 let reopen_backend = backend.clone();
4308 let reopen = tokio::spawn(async move {
4309 reopen_backend
4310 .open_execution(exec, ExecutionKind::AgentTurn)
4311 .await
4312 });
4313 let prune_backend = backend.clone();
4314 let policy_for_task = policy.clone();
4315 let prune = tokio::spawn(async move { prune_backend.prune(&policy_for_task).await });
4316
4317 let (reopen_result, prune_result) = tokio::join!(reopen, prune);
4318 reopen_result
4319 .expect("reopen task must not panic")
4320 .expect("reopen must not error under concurrent prune");
4321 prune_result
4322 .expect("prune task must not panic")
4323 .expect("prune must not error under a concurrent reopen");
4324
4325 let (status,): (String,) = zeph_db::query_as(sql!(
4326 "SELECT status FROM durable_executions WHERE execution_id = ?"
4327 ))
4328 .bind(exec.as_uuid().to_string())
4329 .fetch_one(backend.pool())
4330 .await
4331 .expect(
4332 "the row must exist under either race outcome — reopened-running, or \
4333 deleted-then-reinserted-fresh-running by reopen's fallback",
4334 );
4335 assert_eq!(
4336 status, "running",
4337 "whichever task wins, the row must end up running — never left completed \
4338 (orphaned from a live journal) or absent"
4339 );
4340 }
4341 }
4342
4343 #[tokio::test]
4344 async fn max_seq_reflects_committed_appends() {
4345 let backend = mem_backend(1_048_576).await;
4346 assert_eq!(
4347 backend.max_seq().await.unwrap(),
4348 None,
4349 "empty journal has no max seq"
4350 );
4351
4352 let exec = ExecutionId::new();
4353 backend
4354 .open_execution(exec, ExecutionKind::AgentTurn)
4355 .await
4356 .unwrap();
4357 for step in 0..3 {
4358 backend.append(step_result(exec, step, b"x")).await.unwrap();
4359 }
4360 assert_eq!(backend.max_seq().await.unwrap(), Some(JournalSeq::new(3)));
4361 }
4362
4363 #[tokio::test]
4364 async fn append_batch_group_commits_every_entry() {
4365 let backend = mem_backend(1_048_576).await;
4366 let exec = ExecutionId::new();
4367 backend
4368 .open_execution(exec, ExecutionKind::AgentTurn)
4369 .await
4370 .unwrap();
4371 let batch = vec![
4372 step_result(exec, 0, b"a"),
4373 step_result(exec, 1, b"b"),
4374 step_result(exec, 2, b"c"),
4375 ];
4376 backend.append_batch(&batch).await.unwrap();
4377 assert_eq!(backend.read_execution(exec).await.unwrap().len(), 3);
4378 }
4379
4380 #[tokio::test]
4381 async fn read_execution_range_bounds_the_segment() {
4382 let backend = mem_backend(1_048_576).await;
4383 let exec = ExecutionId::new();
4384 backend
4385 .open_execution(exec, ExecutionKind::AgentTurn)
4386 .await
4387 .unwrap();
4388 for step in 0..5 {
4389 backend.append(step_result(exec, step, b"x")).await.unwrap();
4390 }
4391 let segment = backend.read_execution_range(exec, 2, 2).await.unwrap();
4392 assert_eq!(segment.len(), 2);
4393 assert_eq!(segment[0].step_id, StepId::new(2));
4394 assert_eq!(segment[1].step_id, StepId::new(3));
4395 }
4396
4397 #[tokio::test]
4398 async fn lookup_committed_result_finds_by_idem_key() {
4399 let backend = mem_backend(1_048_576).await;
4400 let exec = ExecutionId::new();
4401 backend
4402 .open_execution(exec, ExecutionKind::AgentTurn)
4403 .await
4404 .unwrap();
4405 let entry = step_result(exec, 0, b"committed");
4406 let idem_key = match &entry.entry {
4407 EntryKind::StepResult {
4408 idempotency_key, ..
4409 } => *idempotency_key,
4410 other => panic!("unexpected entry kind: {other:?}"),
4411 };
4412 backend.append(entry).await.unwrap();
4413
4414 let found = backend
4415 .lookup_committed_result(exec, idem_key)
4416 .await
4417 .unwrap()
4418 .expect("committed result is located by its idempotency key");
4419 match &found.entry {
4420 EntryKind::StepResult { payload, .. } => assert_eq!(payload.as_ref(), b"committed"),
4421 other => panic!("unexpected entry kind: {other:?}"),
4422 }
4423
4424 let absent = IdempotencyKey::derive(exec, StepId::new(99), b"never");
4426 assert!(
4427 backend
4428 .lookup_committed_result(exec, absent)
4429 .await
4430 .unwrap()
4431 .is_none()
4432 );
4433 }
4434
4435 #[tokio::test]
4436 async fn capabilities_describe_the_local_profile() {
4437 let backend = mem_backend(4096).await;
4438 let caps = backend.capabilities();
4439 assert!(caps.parallel_steps);
4440 assert!(
4441 !caps.cross_process,
4442 "the SQLite local backend is in-process"
4443 );
4444 assert_eq!(caps.max_payload, 4096);
4445 }
4446
4447 #[tokio::test]
4448 async fn promise_insert_state_and_resolve_round_trip() {
4449 let backend = mem_backend(1_048_576)
4450 .await
4451 .with_cipher(Arc::new(XorCipher));
4452 let exec = ExecutionId::new();
4453 backend
4454 .open_execution(exec, ExecutionKind::AgentTurn)
4455 .await
4456 .unwrap();
4457 let promise = PromiseId::derive(exec, StepId::new(0));
4458 backend
4459 .insert_promise(promise, exec, [9u8; 32], 100)
4460 .await
4461 .unwrap();
4462
4463 let pending = backend.promise_state(promise).await.unwrap().unwrap();
4464 assert!(!pending.resolved);
4465 assert_eq!(pending.execution_id, exec);
4466 assert_eq!(pending.resolver_token_hash, [9u8; 32]);
4467
4468 assert!(
4470 backend
4471 .resolve_promise(promise, exec, b"answer", 200)
4472 .await
4473 .unwrap()
4474 );
4475 assert!(
4476 !backend
4477 .resolve_promise(promise, exec, b"again", 300)
4478 .await
4479 .unwrap()
4480 );
4481
4482 let resolved = backend.promise_state(promise).await.unwrap().unwrap();
4483 assert!(resolved.resolved);
4484 let sealed = resolved.payload.expect("resolved payload present");
4485 assert_ne!(sealed.as_slice(), b"answer", "payload is sealed at rest");
4486 let opened = backend
4487 .open_promise_payload(promise, exec, &sealed)
4488 .unwrap();
4489 assert_eq!(opened.as_ref(), b"answer");
4490 }
4491
4492 #[tokio::test]
4493 async fn claim_promise_notification_is_single_winner() {
4494 let backend = mem_backend(1_048_576).await;
4495 let exec = ExecutionId::new();
4496 backend
4497 .open_execution(exec, ExecutionKind::AgentTurn)
4498 .await
4499 .unwrap();
4500 let promise = PromiseId::derive(exec, StepId::new(0));
4501 backend
4502 .insert_promise(promise, exec, [9u8; 32], 100)
4503 .await
4504 .unwrap();
4505
4506 assert!(
4508 backend
4509 .claim_promise_notification(promise, 200)
4510 .await
4511 .unwrap()
4512 );
4513 assert!(
4515 !backend
4516 .claim_promise_notification(promise, 300)
4517 .await
4518 .unwrap()
4519 );
4520 }
4521
4522 #[tokio::test]
4523 async fn timer_arm_due_and_fire() {
4524 let backend = mem_backend(1_048_576).await;
4525 let exec = ExecutionId::new();
4526 backend
4527 .open_execution(exec, ExecutionKind::AgentTurn)
4528 .await
4529 .unwrap();
4530 let past = TimerId::derive(exec, StepId::new(0));
4531 let future = TimerId::derive(exec, StepId::new(1));
4532 backend.arm_timer(past, exec, 1_000, 0).await.unwrap();
4533 backend
4534 .arm_timer(future, exec, 9_000_000_000_000, 0)
4535 .await
4536 .unwrap();
4537
4538 let due = backend.due_timers(5_000).await.unwrap();
4540 assert_eq!(due, vec![past]);
4541
4542 assert!(backend.mark_timer_fired(past).await.unwrap());
4543 assert!(
4544 !backend.mark_timer_fired(past).await.unwrap(),
4545 "second fire is a no-op"
4546 );
4547 assert_eq!(
4548 backend.timer_state(past).await.unwrap(),
4549 Some((1_000, true))
4550 );
4551 assert!(backend.due_timers(5_000).await.unwrap().is_empty());
4553 }
4554
4555 #[tokio::test]
4556 async fn prune_deletes_terminal_executions_past_ttl() {
4557 let backend = mem_backend(1_048_576).await;
4558 let old = ExecutionId::new();
4560 backend
4561 .open_execution(old, ExecutionKind::AgentTurn)
4562 .await
4563 .unwrap();
4564 backend.append(step_result(old, 0, b"x")).await.unwrap();
4565 zeph_db::query(sql!(
4567 "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
4568 ))
4569 .bind(old.as_uuid().to_string())
4570 .execute(backend.pool())
4571 .await
4572 .unwrap();
4573
4574 let live = ExecutionId::new();
4575 backend
4576 .open_execution(live, ExecutionKind::AgentTurn)
4577 .await
4578 .unwrap();
4579 backend.append(step_result(live, 0, b"y")).await.unwrap();
4580
4581 let policy = RetentionPolicy {
4582 ttl_completed_secs: 1,
4583 prune_batch_size: 10,
4584 ..RetentionPolicy::default()
4585 };
4586 let deleted = backend.prune(&policy).await.unwrap();
4587 assert_eq!(deleted, 1, "only the aged terminal execution is pruned");
4588
4589 assert!(backend.read_execution(old).await.unwrap().is_empty());
4591 assert!(
4592 backend
4593 .promise_state(PromiseId::derive(old, StepId::new(0)))
4594 .await
4595 .unwrap()
4596 .is_none()
4597 );
4598 assert_eq!(backend.read_execution(live).await.unwrap().len(), 1);
4599 }
4600
4601 #[tokio::test]
4602 async fn count_prunable_and_prune_include_canceled_executions_past_ttl() {
4603 let backend = mem_backend(1_048_576).await;
4606 let exec = ExecutionId::new();
4607 backend
4608 .open_execution(exec, ExecutionKind::AgentTurn)
4609 .await
4610 .unwrap();
4611 assert_eq!(
4612 backend.cancel_execution(exec).await.unwrap(),
4613 CancelOutcome::Canceled
4614 );
4615 zeph_db::query(sql!(
4617 "UPDATE durable_executions SET finalized_at = 1000 WHERE execution_id = ?"
4618 ))
4619 .bind(exec.as_uuid().to_string())
4620 .execute(backend.pool())
4621 .await
4622 .unwrap();
4623
4624 let policy = RetentionPolicy {
4625 ttl_failed_secs: 1,
4626 prune_batch_size: 10,
4627 ..RetentionPolicy::default()
4628 };
4629 let prunable = backend.count_prunable(&policy).await.unwrap();
4630 assert_eq!(
4631 prunable, 1,
4632 "an aged canceled row must be counted as prunable"
4633 );
4634
4635 let deleted = backend.prune(&policy).await.unwrap();
4636 assert_eq!(deleted, 1, "an aged canceled row must actually be pruned");
4637 assert!(backend.read_execution(exec).await.unwrap().is_empty());
4638 }
4639
4640 #[tokio::test]
4652 async fn prune_deletes_a_keyed_execution_and_its_integrity_row() {
4653 let backend = mem_backend(1_048_576).await.with_hwm_key(0, [42u8; 32]);
4654 let old = ExecutionId::new();
4655 backend
4656 .open_execution(old, ExecutionKind::AgentTurn)
4657 .await
4658 .unwrap();
4659 backend.append(step_result(old, 0, b"x")).await.unwrap();
4660
4661 let before: (i64,) = zeph_db::query_as(sql!(
4663 "SELECT COUNT(*) FROM durable_execution_integrity WHERE execution_id = ?"
4664 ))
4665 .bind(old.as_uuid().to_string())
4666 .fetch_one(backend.pool())
4667 .await
4668 .unwrap();
4669 assert_eq!(
4670 before.0, 1,
4671 "a committed StepResult must create an integrity row"
4672 );
4673
4674 zeph_db::query(sql!(
4675 "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
4676 ))
4677 .bind(old.as_uuid().to_string())
4678 .execute(backend.pool())
4679 .await
4680 .unwrap();
4681
4682 let policy = RetentionPolicy {
4683 ttl_completed_secs: 1,
4684 prune_batch_size: 10,
4685 ..RetentionPolicy::default()
4686 };
4687 let deleted = backend
4688 .prune(&policy)
4689 .await
4690 .expect("prune must not fail closed on a keyed execution's FK");
4691 assert_eq!(deleted, 1, "the keyed execution is pruned like any other");
4692
4693 assert!(backend.read_execution(old).await.unwrap().is_empty());
4694 let after: (i64,) = zeph_db::query_as(sql!(
4695 "SELECT COUNT(*) FROM durable_execution_integrity WHERE execution_id = ?"
4696 ))
4697 .bind(old.as_uuid().to_string())
4698 .fetch_one(backend.pool())
4699 .await
4700 .unwrap();
4701 assert_eq!(
4702 after.0, 0,
4703 "the integrity row must be pruned alongside its execution"
4704 );
4705 }
4706
4707 async fn backdate_updated_at(backend: &LocalBackend, id: ExecutionId, updated_at_ms: i64) {
4709 zeph_db::query(sql!(
4710 "UPDATE durable_executions SET updated_at = ? WHERE execution_id = ?"
4711 ))
4712 .bind(updated_at_ms)
4713 .bind(id.as_uuid().to_string())
4714 .execute(backend.pool())
4715 .await
4716 .unwrap();
4717 }
4718
4719 #[tokio::test]
4720 async fn sweep_orphans_disabled_when_threshold_is_zero() {
4721 let dir = tempfile::tempdir().unwrap();
4724 let backend =
4725 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4726 .await
4727 .unwrap();
4728 backend.init().await.unwrap();
4729
4730 let exec = ExecutionId::new();
4731 backend
4732 .open_execution(exec, ExecutionKind::AgentTurn)
4733 .await
4734 .unwrap();
4735 backdate_updated_at(&backend, exec, 0).await;
4736
4737 let policy = RetentionPolicy {
4738 stale_running_after_secs: 0,
4739 ..RetentionPolicy::default()
4740 };
4741 let aborted = backend.sweep_orphans(&policy).await.unwrap();
4742 assert_eq!(
4743 aborted, 0,
4744 "stale_running_after_secs = 0 disables the sweep"
4745 );
4746
4747 let (status,): (String,) = zeph_db::query_as(sql!(
4748 "SELECT status FROM durable_executions WHERE execution_id = ?"
4749 ))
4750 .bind(exec.as_uuid().to_string())
4751 .fetch_one(backend.pool())
4752 .await
4753 .unwrap();
4754 assert_eq!(status, "running");
4755 }
4756
4757 #[tokio::test]
4758 async fn sweep_orphans_is_a_documented_no_op_on_memory_backend() {
4759 let backend = mem_backend(1_048_576).await;
4762 let exec = ExecutionId::new();
4763 backend
4764 .open_execution(exec, ExecutionKind::AgentTurn)
4765 .await
4766 .unwrap();
4767 backdate_updated_at(&backend, exec, 0).await;
4768
4769 let policy = RetentionPolicy {
4770 stale_running_after_secs: 1,
4771 ..RetentionPolicy::default()
4772 };
4773 let aborted = backend.sweep_orphans(&policy).await.unwrap();
4774 assert_eq!(
4775 aborted, 0,
4776 "a lock_dir=None backend must never abort on staleness alone"
4777 );
4778
4779 let (status,): (String,) = zeph_db::query_as(sql!(
4780 "SELECT status FROM durable_executions WHERE execution_id = ?"
4781 ))
4782 .bind(exec.as_uuid().to_string())
4783 .fetch_one(backend.pool())
4784 .await
4785 .unwrap();
4786 assert_eq!(status, "running");
4787 }
4788
4789 #[tokio::test]
4790 async fn sweep_orphans_aborts_a_stale_running_execution_with_no_live_owner() {
4791 let dir = tempfile::tempdir().unwrap();
4793 let backend =
4794 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4795 .await
4796 .unwrap();
4797 backend.init().await.unwrap();
4798
4799 let exec = ExecutionId::new();
4800 backend
4801 .open_execution(exec, ExecutionKind::AgentTurn)
4802 .await
4803 .unwrap();
4804 backdate_updated_at(&backend, exec, 0).await;
4807
4808 let policy = RetentionPolicy {
4809 stale_running_after_secs: 1,
4810 ..RetentionPolicy::default()
4811 };
4812 let aborted = backend.sweep_orphans(&policy).await.unwrap();
4813 assert_eq!(aborted, 1);
4814
4815 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
4816 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
4817 ))
4818 .bind(exec.as_uuid().to_string())
4819 .fetch_one(backend.pool())
4820 .await
4821 .unwrap();
4822 assert_eq!(status, "aborted");
4823 assert!(finalized.is_some());
4824 }
4825
4826 #[tokio::test]
4827 async fn sweep_orphans_skips_an_execution_whose_lock_is_held_by_a_live_owner() {
4828 let dir = tempfile::tempdir().unwrap();
4832 let db_path = dir.path().join("durable.db");
4833 let url = db_path.to_string_lossy().into_owned();
4834
4835 let owner = LocalBackend::open(&url, 1_048_576).await.unwrap();
4836 owner.init().await.unwrap();
4837 let sweeper = LocalBackend::open(&url, 1_048_576).await.unwrap();
4838
4839 let exec = ExecutionId::new();
4840 let (_, _lock) = owner
4841 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
4842 .await
4843 .unwrap();
4844 backdate_updated_at(&owner, exec, 0).await;
4845
4846 let policy = RetentionPolicy {
4847 stale_running_after_secs: 1,
4848 ..RetentionPolicy::default()
4849 };
4850 let aborted = sweeper.sweep_orphans(&policy).await.unwrap();
4851 assert_eq!(aborted, 0, "a live-held lock must never be swept");
4852
4853 let (status,): (String,) = zeph_db::query_as(sql!(
4854 "SELECT status FROM durable_executions WHERE execution_id = ?"
4855 ))
4856 .bind(exec.as_uuid().to_string())
4857 .fetch_one(owner.pool())
4858 .await
4859 .unwrap();
4860 assert_eq!(status, "running");
4861 }
4862
4863 #[tokio::test]
4864 async fn sweep_orphans_leaves_a_fresh_running_execution_untouched() {
4865 let dir = tempfile::tempdir().unwrap();
4867 let backend =
4868 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4869 .await
4870 .unwrap();
4871 backend.init().await.unwrap();
4872
4873 let exec = ExecutionId::new();
4874 backend
4875 .open_execution(exec, ExecutionKind::AgentTurn)
4876 .await
4877 .unwrap();
4878
4879 let policy = RetentionPolicy {
4880 stale_running_after_secs: 3600,
4881 ..RetentionPolicy::default()
4882 };
4883 let aborted = backend.sweep_orphans(&policy).await.unwrap();
4884 assert_eq!(aborted, 0);
4885 }
4886
4887 #[tokio::test]
4888 async fn sweep_orphans_never_touches_a_stale_canceled_row() {
4889 let dir = tempfile::tempdir().unwrap();
4893 let backend =
4894 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4895 .await
4896 .unwrap();
4897 backend.init().await.unwrap();
4898
4899 let exec = ExecutionId::new();
4900 backend
4901 .open_execution(exec, ExecutionKind::AgentTurn)
4902 .await
4903 .unwrap();
4904 assert_eq!(
4905 backend.cancel_execution(exec).await.unwrap(),
4906 CancelOutcome::Canceled
4907 );
4908 backdate_updated_at(&backend, exec, 0).await;
4909
4910 let policy = RetentionPolicy {
4911 stale_running_after_secs: 1,
4912 ..RetentionPolicy::default()
4913 };
4914 for _ in 0..3 {
4915 let aborted = backend.sweep_orphans(&policy).await.unwrap();
4916 assert_eq!(aborted, 0, "a canceled row must never be swept");
4917 }
4918
4919 let (status,): (String,) = zeph_db::query_as(sql!(
4920 "SELECT status FROM durable_executions WHERE execution_id = ?"
4921 ))
4922 .bind(exec.as_uuid().to_string())
4923 .fetch_one(backend.pool())
4924 .await
4925 .unwrap();
4926 assert_eq!(
4927 status, "canceled",
4928 "sweep must never resurrect a canceled row"
4929 );
4930 }
4931
4932 #[tokio::test]
4933 async fn count_orphans_matches_sweep_without_mutating() {
4934 let dir = tempfile::tempdir().unwrap();
4935 let backend =
4936 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4937 .await
4938 .unwrap();
4939 backend.init().await.unwrap();
4940
4941 let exec = ExecutionId::new();
4942 backend
4943 .open_execution(exec, ExecutionKind::AgentTurn)
4944 .await
4945 .unwrap();
4946 backdate_updated_at(&backend, exec, 0).await;
4947
4948 let policy = RetentionPolicy {
4949 stale_running_after_secs: 1,
4950 ..RetentionPolicy::default()
4951 };
4952 let counted = backend.count_orphans(&policy).await.unwrap();
4953 assert_eq!(counted, 1);
4954
4955 let (status,): (String,) = zeph_db::query_as(sql!(
4957 "SELECT status FROM durable_executions WHERE execution_id = ?"
4958 ))
4959 .bind(exec.as_uuid().to_string())
4960 .fetch_one(backend.pool())
4961 .await
4962 .unwrap();
4963 assert_eq!(status, "running");
4964
4965 let aborted = backend.sweep_orphans(&policy).await.unwrap();
4966 assert_eq!(
4967 aborted, counted,
4968 "sweep must abort exactly what count_orphans counted"
4969 );
4970 }
4971
4972 #[tokio::test]
4978 async fn sweep_orphans_processes_every_batch_when_candidates_straddle_the_batch_size() {
4979 let dir = tempfile::tempdir().unwrap();
4980 let backend =
4981 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4982 .await
4983 .unwrap();
4984 backend.init().await.unwrap();
4985
4986 let batch_size = 2u64;
4987 let candidate_count = batch_size + 1; let mut execs = Vec::new();
4989 for _ in 0..candidate_count {
4990 let exec = ExecutionId::new();
4991 backend
4992 .open_execution(exec, ExecutionKind::AgentTurn)
4993 .await
4994 .unwrap();
4995 backdate_updated_at(&backend, exec, 0).await;
4996 execs.push(exec);
4997 }
4998
4999 let policy = RetentionPolicy {
5000 stale_running_after_secs: 1,
5001 prune_batch_size: batch_size,
5002 ..RetentionPolicy::default()
5003 };
5004 let aborted = backend.sweep_orphans(&policy).await.unwrap();
5005 assert_eq!(
5006 aborted, candidate_count,
5007 "every candidate must be aborted, including the one past the first batch"
5008 );
5009
5010 for exec in execs {
5011 let (status,): (String,) = zeph_db::query_as(sql!(
5012 "SELECT status FROM durable_executions WHERE execution_id = ?"
5013 ))
5014 .bind(exec.as_uuid().to_string())
5015 .fetch_one(backend.pool())
5016 .await
5017 .unwrap();
5018 assert_eq!(status, "aborted");
5019 }
5020 }
5021
5022 #[tokio::test]
5035 async fn sweep_orphans_terminates_when_lock_held_candidates_exceed_batch_size() {
5036 let dir = tempfile::tempdir().unwrap();
5037 let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
5038
5039 let owner = LocalBackend::open(&db_url, 1_048_576).await.unwrap();
5040 owner.init().await.unwrap();
5041 let sweeper = LocalBackend::open(&db_url, 1_048_576).await.unwrap();
5042
5043 let batch_size = 2u64;
5044 let candidate_count = batch_size * 2 + 1; let mut locks = Vec::new();
5046 for _ in 0..candidate_count {
5047 let exec = ExecutionId::new();
5048 let (_, lock) = owner
5049 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
5050 .await
5051 .unwrap();
5052 backdate_updated_at(&owner, exec, 0).await;
5053 locks.push(lock); }
5055
5056 let policy = RetentionPolicy {
5057 stale_running_after_secs: 1,
5058 prune_batch_size: batch_size,
5059 ..RetentionPolicy::default()
5060 };
5061
5062 let aborted = tokio::time::timeout(
5063 std::time::Duration::from_secs(10),
5064 sweeper.sweep_orphans(&policy),
5065 )
5066 .await
5067 .expect(
5068 "sweep_orphans must terminate even when lock-held candidates exceed prune_batch_size \
5069 (#6254 C1) — it hung instead of returning",
5070 )
5071 .unwrap();
5072
5073 assert_eq!(aborted, 0, "every candidate's lock is held by a live owner");
5074 drop(locks);
5075 }
5076
5077 #[tokio::test]
5086 async fn concurrent_sweep_and_reopen_race_never_corrupts_the_row() {
5087 let dir = tempfile::tempdir().unwrap();
5088 let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
5089 let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
5090 backend.init().await.unwrap();
5091
5092 let policy = RetentionPolicy {
5093 stale_running_after_secs: 1,
5094 prune_batch_size: 10,
5095 ..RetentionPolicy::default()
5096 };
5097
5098 for _ in 0..20 {
5099 let exec = ExecutionId::new();
5100 backend
5101 .open_execution(exec, ExecutionKind::AgentTurn)
5102 .await
5103 .unwrap();
5104 backdate_updated_at(&backend, exec, 0).await;
5105
5106 let sweep_backend = backend.clone();
5107 let policy_for_task = policy.clone();
5108 let sweep =
5109 tokio::spawn(async move { sweep_backend.sweep_orphans(&policy_for_task).await });
5110
5111 let reopen_backend = backend.clone();
5112 let reopen = tokio::spawn(async move {
5113 reopen_backend
5114 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
5115 .await
5116 });
5117
5118 let (sweep_result, reopen_result) = tokio::join!(sweep, reopen);
5119 let aborted = sweep_result
5120 .expect("sweep task must not panic")
5121 .expect("sweep must not error under a concurrent reopen");
5122 assert!(aborted <= 1, "at most one candidate row exists per trial");
5123
5124 match reopen_result.expect("reopen task must not panic") {
5125 Ok((_is_resume, _lock)) => {
5126 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
5130 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
5131 ))
5132 .bind(exec.as_uuid().to_string())
5133 .fetch_one(backend.pool())
5134 .await
5135 .unwrap();
5136 assert_eq!(status, "running");
5137 assert!(finalized.is_none());
5138 backend
5146 .finalize(exec, ExecutionStatus::Completed)
5147 .await
5148 .unwrap();
5149 }
5150 Err(DurableError::ExecutionLocked { .. }) => {
5151 }
5153 Err(e) => panic!(
5154 "reopen must only ever fail with ExecutionLocked under this race, got {e:?}"
5155 ),
5156 }
5157 }
5158 }
5159
5160 #[tokio::test]
5161 async fn checkpoint_fold_compacts_idempotent_prefix_and_replays() {
5162 let backend = mem_backend(1_048_576)
5163 .await
5164 .with_cipher(Arc::new(XorCipher));
5165 let exec = ExecutionId::new();
5166 backend
5167 .open_execution(exec, ExecutionKind::AgentTurn)
5168 .await
5169 .unwrap();
5170 for step in 0..5 {
5171 backend
5172 .append(step_result(exec, step, format!("v{step}").as_bytes()))
5173 .await
5174 .unwrap();
5175 }
5176
5177 let folded = backend.checkpoint_fold(exec, 3).await.unwrap();
5179 assert_eq!(folded, 3);
5180
5181 let remaining = backend.read_execution(exec).await.unwrap();
5183 let step_results: Vec<u32> = remaining
5184 .iter()
5185 .filter(|e| matches!(e.entry, EntryKind::StepResult { .. }))
5186 .map(|e| e.step_id.value())
5187 .collect();
5188 assert_eq!(step_results, vec![3, 4], "folded step rows are deleted");
5189 assert!(
5190 remaining
5191 .iter()
5192 .any(|e| matches!(e.entry, EntryKind::Checkpoint { .. })),
5193 "a checkpoint entry replaces the folded prefix"
5194 );
5195
5196 let preloaded = backend.read_checkpoints(exec).await.unwrap();
5198 assert_eq!(preloaded.len(), 3);
5199 for (i, entry) in preloaded.iter().enumerate() {
5200 let step = u32::try_from(i).unwrap();
5201 assert_eq!(entry.step_id, StepId::new(step));
5202 match &entry.entry {
5203 EntryKind::StepResult {
5204 payload,
5205 idempotency_key,
5206 ..
5207 } => {
5208 assert_eq!(payload.as_ref(), format!("v{step}").as_bytes());
5209 assert_eq!(
5210 *idempotency_key,
5211 IdempotencyKey::derive(exec, StepId::new(step), b"tool:read")
5212 );
5213 }
5214 other => panic!("unexpected folded entry: {other:?}"),
5215 }
5216 }
5217 }
5218
5219 #[tokio::test]
5225 async fn hwm_is_a_no_op_when_unkeyed() {
5226 let backend = mem_backend(1_048_576).await;
5227 let exec = ExecutionId::new();
5228 assert!(
5229 !backend
5230 .open_execution(exec, ExecutionKind::AgentTurn)
5231 .await
5232 .unwrap()
5233 );
5234 backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5235 assert!(
5237 backend
5238 .open_execution(exec, ExecutionKind::AgentTurn)
5239 .await
5240 .unwrap()
5241 );
5242 }
5243
5244 #[tokio::test]
5245 async fn hwm_verifies_on_resume_after_single_append_and_batch_append() {
5246 let backend = mem_backend(1_048_576).await.with_hwm_key(0, [1u8; 32]);
5247 let exec = ExecutionId::new();
5248 assert!(
5249 !backend
5250 .open_execution(exec, ExecutionKind::AgentTurn)
5251 .await
5252 .unwrap()
5253 );
5254 backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5255 backend
5256 .append_batch(&[step_result(exec, 1, b"v1"), step_result(exec, 2, b"v2")])
5257 .await
5258 .unwrap();
5259
5260 assert!(
5261 backend
5262 .open_execution(exec, ExecutionKind::AgentTurn)
5263 .await
5264 .unwrap(),
5265 "resume must succeed when the recomputed count matches the signed HWM"
5266 );
5267 }
5268
5269 #[tokio::test]
5270 async fn hwm_detects_deletion_of_a_committed_step_result() {
5271 let backend = mem_backend(1_048_576).await.with_hwm_key(0, [2u8; 32]);
5272 let exec = ExecutionId::new();
5273 backend
5274 .open_execution(exec, ExecutionKind::AgentTurn)
5275 .await
5276 .unwrap();
5277 backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5278 backend.append(step_result(exec, 1, b"v1")).await.unwrap();
5279
5280 zeph_db::query(sql!(
5283 "DELETE FROM durable_journal WHERE execution_id = ? AND step_id = 1"
5284 ))
5285 .bind(exec.as_uuid().to_string())
5286 .execute(backend.pool())
5287 .await
5288 .unwrap();
5289
5290 let err = backend
5291 .open_execution(exec, ExecutionKind::AgentTurn)
5292 .await
5293 .unwrap_err();
5294 assert_matches!(
5295 err,
5296 DurableError::HighWaterMarkIntegrity {
5297 reason: "count_mismatch",
5298 ..
5299 }
5300 );
5301
5302 let summaries = backend.list_executions(None, None, 10).await.unwrap();
5305 let summary = summaries.iter().find(|s| s.execution_id == exec).unwrap();
5306 assert_eq!(summary.status, ExecutionStatus::Aborted);
5307 }
5308
5309 #[tokio::test]
5310 async fn hwm_survives_a_legitimate_checkpoint_fold() {
5311 let backend = mem_backend(1_048_576)
5312 .await
5313 .with_cipher(Arc::new(XorCipher))
5314 .with_hwm_key(0, [3u8; 32]);
5315 let exec = ExecutionId::new();
5316 backend
5317 .open_execution(exec, ExecutionKind::AgentTurn)
5318 .await
5319 .unwrap();
5320 for step in 0..5 {
5321 backend
5322 .append(step_result(exec, step, format!("v{step}").as_bytes()))
5323 .await
5324 .unwrap();
5325 }
5326
5327 let folded = backend.checkpoint_fold(exec, 3).await.unwrap();
5328 assert_eq!(folded, 3);
5329
5330 assert!(
5331 backend
5332 .open_execution(exec, ExecutionKind::AgentTurn)
5333 .await
5334 .unwrap(),
5335 "a legitimate fold must not trip the HWM check: committed_result_count is invariant \
5336 across it (folded_count restores what the DELETE removed)"
5337 );
5338 }
5339
5340 #[tokio::test]
5350 async fn count_integrity_rows_under_epoch_catches_a_checkpoint_folded_pre_rotation_execution() {
5351 let pre_rotation = mem_backend(1_048_576)
5352 .await
5353 .with_cipher(Arc::new(RotatingKeyedCipher {
5354 current_id: 0,
5355 previous_id: None,
5356 }))
5357 .with_hwm_key(0, [20u8; 32]);
5358 let exec = ExecutionId::new();
5359 pre_rotation
5360 .open_execution(exec, ExecutionKind::AgentTurn)
5361 .await
5362 .unwrap();
5363 for step in 0..3 {
5364 pre_rotation
5365 .append(step_result(exec, step, format!("v{step}").as_bytes()))
5366 .await
5367 .unwrap();
5368 }
5369
5370 let post_rotation = LocalBackend::new(pre_rotation.pool().clone(), 1_048_576)
5375 .with_cipher(Arc::new(RotatingKeyedCipher {
5376 current_id: 1,
5377 previous_id: Some(0),
5378 }))
5379 .with_hwm_key(1, [21u8; 32])
5380 .with_previous_hwm_key(0, [20u8; 32]);
5381
5382 let folded = post_rotation.checkpoint_fold(exec, 3).await.unwrap();
5383 assert_eq!(
5384 folded, 3,
5385 "fold must compact every committed StepResult, leaving none live"
5386 );
5387
5388 assert_eq!(
5389 post_rotation.count_sealed_under_key_id(0).await.unwrap(),
5390 0,
5391 "every pre-rotation payload was folded away and resealed under the new key_id; the \
5392 AEAD scan sees nothing left sealed under the previous key_id"
5393 );
5394 assert_eq!(
5395 post_rotation
5396 .count_integrity_rows_under_epoch(0)
5397 .await
5398 .unwrap(),
5399 1,
5400 "the folded execution's HWM row still carries the previous epoch -- checkpoint_fold \
5401 never re-signs it (S1)"
5402 );
5403 assert_eq!(
5404 post_rotation
5405 .count_integrity_rows_under_epoch(1)
5406 .await
5407 .unwrap(),
5408 0,
5409 "the row has not migrated to the current epoch -- only a fresh StepResult commit \
5410 after resume would bump it"
5411 );
5412
5413 assert!(
5417 post_rotation
5418 .open_execution(exec, ExecutionKind::AgentTurn)
5419 .await
5420 .unwrap(),
5421 "a folded pre-rotation execution must still resume through the open rotation window"
5422 );
5423 }
5424
5425 #[tokio::test]
5426 async fn hwm_detects_deletion_that_a_fold_does_not_cover() {
5427 let backend = mem_backend(1_048_576)
5428 .await
5429 .with_cipher(Arc::new(XorCipher))
5430 .with_hwm_key(0, [4u8; 32]);
5431 let exec = ExecutionId::new();
5432 backend
5433 .open_execution(exec, ExecutionKind::AgentTurn)
5434 .await
5435 .unwrap();
5436 for step in 0..5 {
5437 backend
5438 .append(step_result(exec, step, format!("v{step}").as_bytes()))
5439 .await
5440 .unwrap();
5441 }
5442 backend.checkpoint_fold(exec, 3).await.unwrap();
5443
5444 zeph_db::query(sql!(
5446 "DELETE FROM durable_journal WHERE execution_id = ? AND step_id = 4 AND entry_kind = 'step_result'"
5447 ))
5448 .bind(exec.as_uuid().to_string())
5449 .execute(backend.pool())
5450 .await
5451 .unwrap();
5452
5453 assert_matches!(
5454 backend
5455 .open_execution(exec, ExecutionKind::AgentTurn)
5456 .await
5457 .unwrap_err(),
5458 DurableError::HighWaterMarkIntegrity {
5459 reason: "count_mismatch",
5460 ..
5461 }
5462 );
5463 }
5464
5465 #[tokio::test]
5466 async fn hwm_unresolvable_key_epoch_fails_closed_not_legacy() {
5467 let writer = mem_backend(1_048_576).await.with_hwm_key(0, [5u8; 32]);
5468 let exec = ExecutionId::new();
5469 writer
5470 .open_execution(exec, ExecutionKind::AgentTurn)
5471 .await
5472 .unwrap();
5473 writer.append(step_result(exec, 0, b"v0")).await.unwrap();
5474
5475 let reader = LocalBackend::new(writer.pool().clone(), 1_048_576).with_hwm_key(9, [6u8; 32]);
5479 assert_matches!(
5480 reader
5481 .open_execution(exec, ExecutionKind::AgentTurn)
5482 .await
5483 .unwrap_err(),
5484 DurableError::HighWaterMarkIntegrity {
5485 reason: "key_epoch_unresolvable",
5486 ..
5487 }
5488 );
5489 }
5490
5491 #[tokio::test]
5492 async fn hwm_previous_epoch_key_resolves_as_rekeyed_not_tampered() {
5493 let writer = mem_backend(1_048_576).await.with_hwm_key(0, [7u8; 32]);
5494 let exec = ExecutionId::new();
5495 writer
5496 .open_execution(exec, ExecutionKind::AgentTurn)
5497 .await
5498 .unwrap();
5499 writer.append(step_result(exec, 0, b"v0")).await.unwrap();
5500
5501 let reader = LocalBackend::new(writer.pool().clone(), 1_048_576)
5505 .with_hwm_key(1, [8u8; 32])
5506 .with_previous_hwm_key(0, [7u8; 32]);
5507 assert!(
5508 reader
5509 .open_execution(exec, ExecutionKind::AgentTurn)
5510 .await
5511 .unwrap(),
5512 "a row signed under a registered previous epoch must verify, not fail as tampered"
5513 );
5514 }
5515
5516 #[tokio::test]
5517 async fn hwm_wrong_key_under_the_same_epoch_is_tamper() {
5518 let writer = mem_backend(1_048_576).await.with_hwm_key(0, [9u8; 32]);
5519 let exec = ExecutionId::new();
5520 writer
5521 .open_execution(exec, ExecutionKind::AgentTurn)
5522 .await
5523 .unwrap();
5524 writer.append(step_result(exec, 0, b"v0")).await.unwrap();
5525
5526 let reader =
5527 LocalBackend::new(writer.pool().clone(), 1_048_576).with_hwm_key(0, [10u8; 32]);
5528 assert_matches!(
5529 reader
5530 .open_execution(exec, ExecutionKind::AgentTurn)
5531 .await
5532 .unwrap_err(),
5533 DurableError::HighWaterMarkIntegrity {
5534 reason: "hmac_mismatch",
5535 ..
5536 }
5537 );
5538 }
5539
5540 #[tokio::test]
5541 async fn hwm_accepts_a_legacy_execution_with_no_integrity_row() {
5542 let unkeyed_writer = mem_backend(1_048_576).await;
5546 let exec = ExecutionId::new();
5547 unkeyed_writer
5548 .open_execution(exec, ExecutionKind::AgentTurn)
5549 .await
5550 .unwrap();
5551 unkeyed_writer
5552 .append(step_result(exec, 0, b"v0"))
5553 .await
5554 .unwrap();
5555
5556 let keyed_reader =
5557 LocalBackend::new(unkeyed_writer.pool().clone(), 1_048_576).with_hwm_key(0, [11u8; 32]);
5558 assert!(
5559 keyed_reader
5560 .open_execution(exec, ExecutionKind::AgentTurn)
5561 .await
5562 .unwrap(),
5563 "an execution with no integrity row at all is legacy, not tampered"
5564 );
5565 }
5566
5567 #[tokio::test]
5570 async fn hwm_unsealed_absent_row_after_deletion_is_still_ok() {
5571 let backend = mem_backend(1_048_576).await.with_hwm_key(0, [30u8; 32]);
5575 let exec = ExecutionId::new();
5576 backend
5577 .open_execution(exec, ExecutionKind::AgentTurn)
5578 .await
5579 .unwrap();
5580 backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5581
5582 zeph_db::query(sql!(
5583 "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5584 ))
5585 .bind(exec.as_uuid().to_string())
5586 .execute(backend.pool())
5587 .await
5588 .unwrap();
5589
5590 assert!(
5591 backend
5592 .open_execution(exec, ExecutionKind::AgentTurn)
5593 .await
5594 .unwrap(),
5595 "unsealed backend must not treat an absent integrity row as tamper"
5596 );
5597 }
5598
5599 #[tokio::test]
5600 async fn hwm_post_seal_absent_row_with_committed_results_is_tamper() {
5601 let backend = mem_backend(1_048_576)
5602 .await
5603 .with_hwm_key(0, [31u8; 32])
5604 .with_integrity_sealed(true);
5605 let exec = ExecutionId::new();
5606 backend
5607 .open_execution(exec, ExecutionKind::AgentTurn)
5608 .await
5609 .unwrap();
5610 backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5611
5612 zeph_db::query(sql!(
5615 "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5616 ))
5617 .bind(exec.as_uuid().to_string())
5618 .execute(backend.pool())
5619 .await
5620 .unwrap();
5621
5622 let err = backend
5623 .open_execution(exec, ExecutionKind::AgentTurn)
5624 .await
5625 .unwrap_err();
5626 assert_matches!(
5627 err,
5628 DurableError::HighWaterMarkIntegrity {
5629 reason: "integrity_row_absent_post_seal",
5630 ..
5631 }
5632 );
5633 }
5634
5635 #[tokio::test]
5636 async fn hwm_post_seal_forged_created_at_does_not_evade_the_seal() {
5637 let backend = mem_backend(1_048_576)
5640 .await
5641 .with_hwm_key(0, [32u8; 32])
5642 .with_integrity_sealed(true);
5643 let exec = ExecutionId::new();
5644 backend
5645 .open_execution(exec, ExecutionKind::AgentTurn)
5646 .await
5647 .unwrap();
5648 backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5649
5650 zeph_db::query(sql!(
5651 "UPDATE durable_executions SET created_at = 0 WHERE execution_id = ?"
5652 ))
5653 .bind(exec.as_uuid().to_string())
5654 .execute(backend.pool())
5655 .await
5656 .unwrap();
5657 zeph_db::query(sql!(
5658 "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5659 ))
5660 .bind(exec.as_uuid().to_string())
5661 .execute(backend.pool())
5662 .await
5663 .unwrap();
5664
5665 let err = backend
5666 .open_execution(exec, ExecutionKind::AgentTurn)
5667 .await
5668 .unwrap_err();
5669 assert_matches!(
5670 err,
5671 DurableError::HighWaterMarkIntegrity {
5672 reason: "integrity_row_absent_post_seal",
5673 ..
5674 },
5675 "forging created_at must not evade the seal — it is never consulted"
5676 );
5677 }
5678
5679 #[tokio::test]
5680 async fn hwm_grandfathered_execution_absent_row_is_ok() {
5681 let exec = ExecutionId::new();
5682 let writer = mem_backend(1_048_576).await.with_hwm_key(0, [33u8; 32]);
5683 writer
5684 .open_execution(exec, ExecutionKind::AgentTurn)
5685 .await
5686 .unwrap();
5687 writer.append(step_result(exec, 0, b"v0")).await.unwrap();
5688 zeph_db::query(sql!(
5689 "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5690 ))
5691 .bind(exec.as_uuid().to_string())
5692 .execute(writer.pool())
5693 .await
5694 .unwrap();
5695
5696 let sealed_but_grandfathered = LocalBackend::new(writer.pool().clone(), 1_048_576)
5697 .with_hwm_key(0, [33u8; 32])
5698 .with_integrity_sealed(true)
5699 .with_grandfather(std::collections::HashSet::from([exec]));
5700
5701 assert!(
5702 sealed_but_grandfathered
5703 .open_execution(exec, ExecutionKind::AgentTurn)
5704 .await
5705 .unwrap(),
5706 "a grandfathered execution_id must resume despite the seal"
5707 );
5708 }
5709
5710 #[tokio::test]
5711 async fn find_unsealed_resumable_executions_finds_only_the_offending_set() {
5712 let backend = mem_backend(1_048_576).await.with_hwm_key(0, [35u8; 32]);
5713
5714 let offending = ExecutionId::new();
5716 backend
5717 .open_execution(offending, ExecutionKind::AgentTurn)
5718 .await
5719 .unwrap();
5720 backend
5721 .append(step_result(offending, 0, b"v0"))
5722 .await
5723 .unwrap();
5724 zeph_db::query(sql!(
5725 "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5726 ))
5727 .bind(offending.as_uuid().to_string())
5728 .execute(backend.pool())
5729 .await
5730 .unwrap();
5731
5732 let intact = ExecutionId::new();
5734 backend
5735 .open_execution(intact, ExecutionKind::AgentTurn)
5736 .await
5737 .unwrap();
5738 backend.append(step_result(intact, 0, b"v0")).await.unwrap();
5739
5740 let empty = ExecutionId::new();
5742 backend
5743 .open_execution(empty, ExecutionKind::AgentTurn)
5744 .await
5745 .unwrap();
5746
5747 let terminal = ExecutionId::new();
5749 backend
5750 .open_execution(terminal, ExecutionKind::AgentTurn)
5751 .await
5752 .unwrap();
5753 backend
5754 .append(step_result(terminal, 0, b"v0"))
5755 .await
5756 .unwrap();
5757 zeph_db::query(sql!(
5758 "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5759 ))
5760 .bind(terminal.as_uuid().to_string())
5761 .execute(backend.pool())
5762 .await
5763 .unwrap();
5764 backend
5765 .finalize(terminal, ExecutionStatus::Completed)
5766 .await
5767 .unwrap();
5768
5769 let found = backend.find_unsealed_resumable_executions().await.unwrap();
5770 assert_eq!(
5771 found,
5772 vec![offending],
5773 "only the truly offending execution must be returned"
5774 );
5775 }
5776
5777 #[tokio::test]
5778 async fn hwm_post_seal_absent_row_with_zero_committed_results_is_ok() {
5779 let backend = mem_backend(1_048_576)
5782 .await
5783 .with_hwm_key(0, [34u8; 32])
5784 .with_integrity_sealed(true);
5785 let exec = ExecutionId::new();
5786 backend
5787 .open_execution(exec, ExecutionKind::AgentTurn)
5788 .await
5789 .unwrap();
5790
5791 assert!(
5792 backend
5793 .open_execution(exec, ExecutionKind::AgentTurn)
5794 .await
5795 .unwrap(),
5796 "zero committed results, post-seal, must not be treated as tamper"
5797 );
5798 }
5799
5800 #[tokio::test]
5801 async fn hwm_ignores_effect_intent_and_control_entries() {
5802 let backend = mem_backend(1_048_576).await.with_hwm_key(0, [12u8; 32]);
5805 let exec = ExecutionId::new();
5806 backend
5807 .open_execution(exec, ExecutionKind::AgentTurn)
5808 .await
5809 .unwrap();
5810 backend.append(effect_intent(exec, 0)).await.unwrap();
5811 backend.append(step_result(exec, 1, b"v1")).await.unwrap();
5812
5813 let stored: (i64,) = zeph_db::query_as(sql!(
5814 "SELECT committed_result_count FROM durable_execution_integrity WHERE execution_id = ?"
5815 ))
5816 .bind(exec.as_uuid().to_string())
5817 .fetch_one(backend.pool())
5818 .await
5819 .unwrap();
5820 assert_eq!(
5821 stored.0, 1,
5822 "only the StepResult row counts, not the EffectIntent"
5823 );
5824
5825 assert!(
5826 backend
5827 .open_execution(exec, ExecutionKind::AgentTurn)
5828 .await
5829 .unwrap()
5830 );
5831 }
5832}