1use std::fmt;
38use std::fmt::Write as _;
39use std::path::PathBuf;
40use std::sync::Arc;
41use std::time::{SystemTime, UNIX_EPOCH};
42
43use bytes::Bytes;
44use zeph_db::{DbPool, sql};
45
46use crate::backend::execution_lock::ExecutionLock;
47use crate::backend::{BackendCapabilities, ExecutionBackend, ExecutionSummary, RedactedEntry};
48use crate::cipher::{EntryKindTag, PayloadAad, PayloadCipher, ensure_payload_within_limit};
49use crate::config::RetentionPolicy;
50use crate::error::DurableError;
51use crate::ids::{
52 ExecutionId, ExecutionKind, IdempotencyKey, JournalSeq, PromiseId, StepId, TimerId,
53};
54use crate::journal::{EntryKind, ExecutionStatus, Journal, JournalEntry};
55use crate::promise::PromiseRecord;
56use crate::retention::{CheckpointSnapshot, FoldedStep, decode_checkpoint, encode_checkpoint};
57use crate::waiters::NotifyRegistry;
58use tracing::Instrument as _;
59
60const SEAL_OVERHEAD_SLACK: u64 = 128;
68
69type ExecutionRow = (String, String, String, i64, i64, Option<i64>, i64);
71
72type RedactedRow = (
74 i64,
75 i64,
76 String,
77 Option<Vec<u8>>,
78 Option<String>,
79 Option<i64>,
80 i64,
81);
82
83fn idem_key_prefix(bytes: &[u8]) -> String {
85 bytes.iter().take(8).fold(String::new(), |mut acc, b| {
86 let _ = write!(acc, "{b:02x}");
87 acc
88 })
89}
90
91#[derive(Debug, Clone, Copy, PartialEq, Eq)]
96pub enum CancelOutcome {
97 Canceled,
100 AlreadyTerminal {
103 status: ExecutionStatus,
105 },
106 NotFound,
108 LiveOwner {
114 pid: u32,
116 },
117 LivenessUnverifiable,
121}
122
123pub struct LocalBackend {
141 pool: DbPool,
142 cipher: Option<Arc<dyn PayloadCipher>>,
143 hmac_key: Option<[u8; 32]>,
144 previous_hmac_key: Option<[u8; 32]>,
149 hwm_key: Option<HwmKeySlot>,
157 hwm_key_previous: Option<HwmKeySlot>,
160 max_payload_bytes: u64,
161 promise_waiters: NotifyRegistry,
163 timer_waiters: NotifyRegistry,
165 lock_dir: Option<PathBuf>,
171 orphan_sweep_warned: std::sync::atomic::AtomicBool,
175 integrity_sealed: bool,
182 integrity_grandfather: std::collections::HashSet<ExecutionId>,
190}
191
192#[derive(Clone, Copy)]
198struct HwmKeySlot {
199 epoch: u32,
200 key: [u8; 32],
201}
202
203impl fmt::Debug for LocalBackend {
204 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
206 f.debug_struct("LocalBackend")
207 .field("cipher", &self.cipher.as_ref().map(|_| "<cipher>"))
208 .field("hmac_key", &self.hmac_key.as_ref().map(|_| "<redacted>"))
209 .field(
210 "previous_hmac_key",
211 &self.previous_hmac_key.as_ref().map(|_| "<redacted>"),
212 )
213 .field("hwm_key_epoch", &self.hwm_key.as_ref().map(|s| s.epoch))
214 .field("max_payload_bytes", &self.max_payload_bytes)
215 .finish_non_exhaustive()
216 }
217}
218
219impl LocalBackend {
220 #[must_use]
226 pub fn new(pool: DbPool, max_payload_bytes: u64) -> Self {
227 Self {
228 pool,
229 cipher: None,
230 hmac_key: None,
231 previous_hmac_key: None,
232 hwm_key: None,
233 hwm_key_previous: None,
234 max_payload_bytes,
235 promise_waiters: NotifyRegistry::default(),
236 timer_waiters: NotifyRegistry::default(),
237 lock_dir: None,
238 orphan_sweep_warned: std::sync::atomic::AtomicBool::new(false),
239 integrity_sealed: false,
240 integrity_grandfather: std::collections::HashSet::new(),
241 }
242 }
243
244 pub async fn open(path: &str, max_payload_bytes: u64) -> Result<Self, DurableError> {
258 let pool = zeph_db::DbConfig {
259 url: path.to_string(),
260 pool_size: 5,
261 }
262 .connect()
263 .await
264 .map_err(|e| DurableError::storage("open", e))?;
265 let mut backend = Self::new(pool, max_payload_bytes);
266 backend.lock_dir = lock_dir_for_path(path);
267 Ok(backend)
268 }
269
270 #[must_use]
272 pub fn with_cipher(mut self, cipher: Arc<dyn PayloadCipher>) -> Self {
273 self.cipher = Some(cipher);
274 self
275 }
276
277 #[must_use]
280 pub fn with_hmac_key(mut self, key: [u8; 32]) -> Self {
281 self.hmac_key = Some(key);
282 self
283 }
284
285 #[must_use]
296 pub fn with_previous_hmac_key(mut self, key: [u8; 32]) -> Self {
297 self.previous_hmac_key = Some(key);
298 self
299 }
300
301 #[must_use]
310 pub fn with_hwm_key(mut self, epoch: u32, key: [u8; 32]) -> Self {
311 self.hwm_key = Some(HwmKeySlot { epoch, key });
312 self
313 }
314
315 #[must_use]
320 pub fn with_previous_hwm_key(mut self, epoch: u32, key: [u8; 32]) -> Self {
321 self.hwm_key_previous = Some(HwmKeySlot { epoch, key });
322 self
323 }
324
325 #[must_use]
331 pub fn with_integrity_sealed(mut self, sealed: bool) -> Self {
332 self.integrity_sealed = sealed;
333 self
334 }
335
336 #[must_use]
342 pub fn with_grandfather(mut self, ids: std::collections::HashSet<ExecutionId>) -> Self {
343 self.integrity_grandfather = ids;
344 self
345 }
346
347 #[must_use]
349 pub fn pool(&self) -> &DbPool {
350 &self.pool
351 }
352
353 pub async fn init(&self) -> Result<(), DurableError> {
361 zeph_db::run_migrations(&self.pool)
362 .await
363 .map_err(|e| DurableError::storage("init", e))?;
364 Ok(())
365 }
366
367 pub async fn list_executions(
382 &self,
383 status: Option<&str>,
384 kind: Option<&str>,
385 limit: i64,
386 ) -> Result<Vec<ExecutionSummary>, DurableError> {
387 let span = tracing::info_span!(
388 "durable.backend.list",
389 status = status.unwrap_or("*"),
390 kind = kind.unwrap_or("*"),
391 count = tracing::field::Empty,
392 );
393 async move {
394 let rows: Vec<ExecutionRow> =
398 zeph_db::query_as(sql!(
399 "SELECT
400 e.execution_id,
401 e.kind,
402 e.status,
403 e.created_at,
404 e.updated_at,
405 e.finalized_at,
406 (SELECT COUNT(*) FROM durable_journal j WHERE j.execution_id = e.execution_id)
407 FROM durable_executions e
408 WHERE e.status = COALESCE(?, e.status)
409 AND e.kind = COALESCE(?, e.kind)
410 ORDER BY e.created_at DESC
411 LIMIT ?"
412 ))
413 .bind(status)
414 .bind(kind)
415 .bind(limit)
416 .fetch_all(&self.pool)
417 .await
418 .map_err(|e| DurableError::storage("list", e))?;
419 tracing::Span::current().record("count", rows.len());
420 rows.into_iter()
421 .map(|(id, kind, status, created, updated, finalized, steps)| {
422 Ok(ExecutionSummary {
423 execution_id: parse_execution_id(&id)?,
424 kind,
425 status: ExecutionStatus::from_tag(&status).ok_or(DurableError::Decode {
426 context: "execution status is not a recognized CHECK-constrained value",
427 })?,
428 created_at_ms: created,
429 updated_at_ms: updated,
430 finalized_at_ms: finalized,
431 step_count: steps.max(0).cast_unsigned(),
432 })
433 })
434 .collect()
435 }
436 .instrument(span)
437 .await
438 }
439
440 pub async fn execution_status(
451 &self,
452 id: ExecutionId,
453 ) -> Result<Option<ExecutionStatus>, DurableError> {
454 let row: Option<(String,)> = zeph_db::query_as(sql!(
455 "SELECT status FROM durable_executions WHERE execution_id = ?"
456 ))
457 .bind(id.as_uuid().to_string())
458 .fetch_optional(&self.pool)
459 .await
460 .map_err(|e| DurableError::storage("execution_status", e))?;
461 row.map(|(status,)| {
462 ExecutionStatus::from_tag(&status).ok_or(DurableError::Decode {
463 context: "execution status is not a recognized CHECK-constrained value",
464 })
465 })
466 .transpose()
467 }
468
469 pub async fn read_execution_redacted(
482 &self,
483 id: ExecutionId,
484 ) -> Result<Vec<RedactedEntry>, DurableError> {
485 let exec = id.as_uuid().to_string();
486 let rows: Vec<RedactedRow> = zeph_db::query_as(sql!(
487 "SELECT seq, step_id, entry_kind, idem_key, effect_class, LENGTH(payload), created_at
488 FROM durable_journal WHERE execution_id = ? ORDER BY seq"
489 ))
490 .bind(&exec)
491 .fetch_all(&self.pool)
492 .await
493 .map_err(|e| DurableError::storage("read_redacted", e))?;
494 Ok(rows
495 .into_iter()
496 .map(
497 |(seq, step, entry_kind, idem, effect_class, payload_len, created)| RedactedEntry {
498 seq,
499 step_id: StepId::new(u32::try_from(step).unwrap_or(0)),
500 entry_kind,
501 effect_class,
502 idem_key_prefix: idem.as_deref().map(idem_key_prefix),
503 payload_len: payload_len.unwrap_or(0).max(0).cast_unsigned(),
504 created_at_ms: created,
505 },
506 )
507 .collect())
508 }
509
510 pub async fn count_prunable(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
519 let cutoffs = crate::retention::PruneCutoffs::from_policy(policy, now_unix_millis());
520 let (count,): (i64,) = zeph_db::query_as(sql!(
521 "SELECT COUNT(*) FROM durable_executions
522 WHERE finalized_at IS NOT NULL
523 AND ( (status = 'completed' AND finalized_at <= ?)
524 OR (status IN ('failed', 'aborted', 'canceled') AND finalized_at <= ?) )"
525 ))
526 .bind(cutoffs.completed_before_ms)
527 .bind(cutoffs.failed_before_ms)
528 .fetch_one(&self.pool)
529 .await
530 .map_err(|e| DurableError::storage("count_prunable", e))?;
531 Ok(count.max(0).cast_unsigned())
532 }
533
534 pub async fn count_orphans(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
547 if policy.stale_running_after_secs == 0 {
548 return Ok(0);
549 }
550 let Some(lock_dir) = self.lock_dir.clone() else {
551 return Ok(0);
552 };
553 let cutoff_ms = orphan_cutoff_ms(policy, now_unix_millis());
554 let candidates: Vec<(String,)> = zeph_db::query_as(sql!(
555 "SELECT execution_id FROM durable_executions WHERE status = 'running' AND updated_at <= ?"
556 ))
557 .bind(cutoff_ms)
558 .fetch_all(&self.pool)
559 .await
560 .map_err(|e| DurableError::storage("count_orphans", e))?;
561 let mut count = 0u64;
562 for (exec_str,) in &candidates {
563 let Ok(execution_id) = parse_execution_id(exec_str) else {
564 continue;
565 };
566 if ExecutionLock::acquire(&lock_dir, execution_id).is_ok() {
567 count += 1;
568 }
569 }
570 Ok(count)
571 }
572
573 pub async fn count_sealed_under_key_id(&self, key_id: u8) -> Result<u64, DurableError> {
598 #[cfg(feature = "postgres")]
599 {
600 let key_id_param = i32::from(key_id);
601 let (journal_count,): (i64,) = zeph_db::query_as(sql!(
602 "SELECT COUNT(*) FROM durable_journal
603 WHERE payload IS NOT NULL AND get_byte(payload, 0) = ?"
604 ))
605 .bind(key_id_param)
606 .fetch_one(&self.pool)
607 .await
608 .map_err(|e| DurableError::storage("count_sealed_under_key_id", e))?;
609 let (promises_count,): (i64,) = zeph_db::query_as(sql!(
610 "SELECT COUNT(*) FROM durable_promises
611 WHERE payload IS NOT NULL AND get_byte(payload, 0) = ?"
612 ))
613 .bind(key_id_param)
614 .fetch_one(&self.pool)
615 .await
616 .map_err(|e| DurableError::storage("count_sealed_under_key_id", e))?;
617 Ok((journal_count.max(0) + promises_count.max(0)).cast_unsigned())
618 }
619 #[cfg(not(feature = "postgres"))]
620 {
621 let key_byte = vec![key_id];
622 let (journal_count,): (i64,) = zeph_db::query_as(sql!(
623 "SELECT COUNT(*) FROM durable_journal
624 WHERE payload IS NOT NULL AND substr(payload, 1, 1) = ?"
625 ))
626 .bind(key_byte.clone())
627 .fetch_one(&self.pool)
628 .await
629 .map_err(|e| DurableError::storage("count_sealed_under_key_id", e))?;
630 let (promises_count,): (i64,) = zeph_db::query_as(sql!(
631 "SELECT COUNT(*) FROM durable_promises
632 WHERE payload IS NOT NULL AND substr(payload, 1, 1) = ?"
633 ))
634 .bind(key_byte)
635 .fetch_one(&self.pool)
636 .await
637 .map_err(|e| DurableError::storage("count_sealed_under_key_id", e))?;
638 Ok((journal_count.max(0) + promises_count.max(0)).cast_unsigned())
639 }
640 }
641
642 pub async fn count_control_entries_under_previous_hmac(&self) -> Result<u64, DurableError> {
679 let rows: Vec<ControlHmacScanRow> = zeph_db::query_as(sql!(
680 "SELECT execution_id, step_id, idem_key, hmac
681 FROM durable_journal
682 WHERE entry_kind = 'effect_intent' AND hmac IS NOT NULL"
683 ))
684 .fetch_all(&self.pool)
685 .await
686 .map_err(|e| DurableError::storage("count_control_entries_under_previous_hmac", e))?;
687
688 if rows.is_empty() {
689 return Ok(0);
690 }
691
692 let (Some(current_key), Some(previous_key)) =
693 (self.hmac_key.as_ref(), self.previous_hmac_key.as_ref())
694 else {
695 return Err(DurableError::ControlIntegrity);
696 };
697
698 let mut count = 0u64;
699 for (execution_id_raw, step_id_raw, idem_key_raw, hmac_raw) in rows {
700 let Ok(execution_id) = parse_execution_id(&execution_id_raw) else {
701 continue;
702 };
703 let Ok(step_id_value) = u32::try_from(step_id_raw) else {
704 continue;
705 };
706 let step_id = StepId::new(step_id_value);
707 let idem_key = idem_key_raw
708 .as_deref()
709 .and_then(|b| slice_to_array32(b, "effect_intent idem_key").ok())
710 .map(IdempotencyKey::from_bytes);
711 let Ok(stored) = slice_to_array32(&hmac_raw, "effect_intent hmac") else {
712 continue;
713 };
714
715 let tag = EntryKindTag::EffectIntent.as_str();
716 let expected_current = Self::keyed_control_hmac(
717 current_key,
718 execution_id,
719 step_id,
720 tag,
721 idem_key.as_ref(),
722 );
723 if blake3::Hash::from(expected_current) == blake3::Hash::from(stored) {
724 continue;
725 }
726 let expected_previous = Self::keyed_control_hmac(
727 previous_key,
728 execution_id,
729 step_id,
730 tag,
731 idem_key.as_ref(),
732 );
733 if blake3::Hash::from(expected_previous) == blake3::Hash::from(stored) {
734 count += 1;
735 }
736 }
737 Ok(count)
738 }
739
740 pub async fn count_integrity_rows_under_epoch(&self, epoch: u32) -> Result<u64, DurableError> {
763 let count: i64 = zeph_db::query_scalar(sql!(
764 "SELECT COUNT(*) FROM durable_execution_integrity WHERE key_epoch = ?"
765 ))
766 .bind(i64::from(epoch))
767 .fetch_one(&self.pool)
768 .await
769 .map_err(|e| DurableError::storage("count_integrity_rows_under_epoch", e))?;
770 Ok(count.max(0).cast_unsigned())
771 }
772
773 pub async fn open_execution(
817 &self,
818 id: ExecutionId,
819 kind: ExecutionKind,
820 ) -> Result<bool, DurableError> {
821 let span = tracing::info_span!(
822 "durable.backend.open",
823 execution_id = %id.as_uuid(),
824 kind = kind.as_str(),
825 is_resume = tracing::field::Empty,
826 );
827 async move {
828 let exec = id.as_uuid().to_string();
829
830 let reopened = zeph_db::query(sql!(
834 "UPDATE durable_executions SET status = 'running', updated_at = ?, finalized_at = NULL
835 WHERE execution_id = ? AND status IN ('completed', 'failed', 'aborted')"
836 ))
837 .bind(now_unix_millis())
838 .bind(&exec)
839 .execute(&self.pool)
840 .await
841 .map_err(|e| DurableError::storage("open", e))?;
842 if reopened.rows_affected() > 0 {
843 self.verify_high_water_mark(id).await?;
844 tracing::Span::current().record("is_resume", true);
845 return Ok(true);
846 }
847
848 let existing: Option<(String,)> = zeph_db::query_as(sql!(
855 "SELECT status FROM durable_executions WHERE execution_id = ?"
856 ))
857 .bind(&exec)
858 .fetch_optional(&self.pool)
859 .await
860 .map_err(|e| DurableError::storage("open", e))?;
861 if let Some((status,)) = existing {
862 if status == "canceled" {
863 return Err(DurableError::ExecutionCanceled { execution_id: id });
864 }
865 self.verify_high_water_mark(id).await?;
866 tracing::Span::current().record("is_resume", true);
867 return Ok(true);
868 }
869 let now = now_unix_millis();
870 zeph_db::query(sql!(
871 "INSERT INTO durable_executions
872 (execution_id, kind, status, created_at, updated_at, finalized_at)
873 VALUES (?, ?, 'running', ?, ?, NULL)"
874 ))
875 .bind(&exec)
876 .bind(kind.as_str())
877 .bind(now)
878 .bind(now)
879 .execute(&self.pool)
880 .await
881 .map_err(|e| DurableError::storage("open", e))?;
882 tracing::Span::current().record("is_resume", false);
883 Ok(false)
884 }
885 .instrument(span)
886 .await
887 }
888
889 pub async fn open_execution_exclusive(
911 &self,
912 id: ExecutionId,
913 kind: ExecutionKind,
914 ) -> Result<(bool, Option<ExecutionLock>), DurableError> {
915 let lock = self
916 .lock_dir
917 .as_deref()
918 .map(|dir| ExecutionLock::acquire(dir, id))
919 .transpose()?;
920 let is_resume = self.open_execution(id, kind).await?;
921 Ok((is_resume, lock))
922 }
923
924 pub async fn cancel_execution(&self, id: ExecutionId) -> Result<CancelOutcome, DurableError> {
965 let span = tracing::info_span!(
966 "durable.backend.cancel",
967 execution_id = %id.as_uuid(),
968 prior_status = tracing::field::Empty,
969 path = tracing::field::Empty,
970 );
971 async move {
972 let exec = id.as_uuid().to_string();
973
974 if let Some(lock_dir) = self.lock_dir.clone() {
975 let _lock = match ExecutionLock::acquire(&lock_dir, id) {
976 Ok(lock) => lock,
977 Err(DurableError::ExecutionLocked { holder_pid, .. }) => {
978 tracing::Span::current().record("path", "live_owner_refused");
979 return Ok(CancelOutcome::LiveOwner { pid: holder_pid });
980 }
981 Err(e) => return Err(e),
982 };
983 let outcome = self.cancel_write(&exec).await?;
984 tracing::Span::current().record("path", "immediate");
985 record_prior_status(outcome);
986 return Ok(outcome);
987 }
989
990 if self.capabilities().cross_process {
991 tracing::Span::current().record("path", "unverifiable");
992 return Ok(CancelOutcome::LivenessUnverifiable);
993 }
994
995 tracing::Span::current().record("path", "no_lock_dir_single_process");
996 let outcome = self.cancel_write(&exec).await?;
997 record_prior_status(outcome);
998 Ok(outcome)
999 }
1000 .instrument(span)
1001 .await
1002 }
1003
1004 async fn cancel_write(&self, exec: &str) -> Result<CancelOutcome, DurableError> {
1008 let now = now_unix_millis();
1009 let mut tx = zeph_db::begin_write(&self.pool)
1010 .await
1011 .map_err(|e| DurableError::storage("cancel", e))?;
1012 let result = zeph_db::query(sql!(
1013 "UPDATE durable_executions SET status = 'canceled', finalized_at = ?, updated_at = ?
1014 WHERE execution_id = ? AND status = 'running'"
1015 ))
1016 .bind(now)
1017 .bind(now)
1018 .bind(exec)
1019 .execute(&mut *tx)
1020 .await
1021 .map_err(|e| DurableError::storage("cancel", e))?;
1022 if result.rows_affected() > 0 {
1023 tx.commit()
1024 .await
1025 .map_err(|e| DurableError::storage("cancel", e))?;
1026 return Ok(CancelOutcome::Canceled);
1027 }
1028
1029 let existing: Option<(String,)> = zeph_db::query_as(sql!(
1033 "SELECT status FROM durable_executions WHERE execution_id = ?"
1034 ))
1035 .bind(exec)
1036 .fetch_optional(&mut *tx)
1037 .await
1038 .map_err(|e| DurableError::storage("cancel", e))?;
1039 tx.commit()
1040 .await
1041 .map_err(|e| DurableError::storage("cancel", e))?;
1042 match existing {
1043 None => Ok(CancelOutcome::NotFound),
1044 Some((status,)) => {
1045 let status = ExecutionStatus::from_tag(&status).ok_or(DurableError::Decode {
1046 context: "unrecognized durable_executions.status value",
1047 })?;
1048 Ok(CancelOutcome::AlreadyTerminal { status })
1049 }
1050 }
1051 }
1052
1053 pub(crate) async fn append_batch(&self, entries: &[JournalEntry]) -> Result<(), DurableError> {
1066 if entries.is_empty() {
1067 return Ok(());
1068 }
1069 let mut rows = Vec::with_capacity(entries.len());
1070 for entry in entries {
1071 rows.push(self.prepare_row(entry)?);
1072 }
1073 let insert = sql!(
1077 "INSERT INTO durable_journal
1078 (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
1079 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
1080 );
1081 let mut tx = zeph_db::begin_write(&self.pool)
1082 .await
1083 .map_err(|e| DurableError::storage("append_batch", e))?;
1084 for (entry, row) in entries.iter().zip(rows) {
1085 zeph_db::query(insert)
1086 .bind(row.execution_id)
1087 .bind(row.step_id)
1088 .bind(row.entry_kind)
1089 .bind(row.idem_key)
1090 .bind(row.effect_class)
1091 .bind(row.payload)
1092 .bind(row.payload_version)
1093 .bind(row.hmac)
1094 .bind(row.created_at)
1095 .execute(&mut *tx)
1096 .await
1097 .map_err(|e| DurableError::storage("append_batch", e))?;
1098 if matches!(entry.entry, EntryKind::StepResult { .. }) {
1099 self.bump_hwm_for_step_result(&mut tx, entry.execution_id, entry.step_id)
1100 .await?;
1101 }
1102 }
1103 tx.commit()
1104 .await
1105 .map_err(|e| DurableError::storage("append_batch", e))?;
1106 Ok(())
1107 }
1108
1109 pub(crate) async fn lookup_committed_result(
1123 &self,
1124 id: ExecutionId,
1125 idem_key: IdempotencyKey,
1126 ) -> Result<Option<JournalEntry>, DurableError> {
1127 let span = tracing::info_span!(
1128 "durable.journal.lookup_idem",
1129 execution_id = %id.as_uuid(),
1130 found = tracing::field::Empty,
1131 );
1132 async move {
1133 let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
1134 "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
1135 FROM durable_journal
1136 WHERE execution_id = ? AND idem_key = ? AND entry_kind = 'step_result'
1137 ORDER BY seq LIMIT 1"
1138 ))
1139 .bind(id.as_uuid().to_string())
1140 .bind(idem_key.as_bytes().to_vec())
1141 .fetch_all(&self.pool)
1142 .await
1143 .map_err(|e| DurableError::storage("lookup_idem", e))?;
1144 let entry = self.rows_to_entries(id, rows).await?.into_iter().next();
1145 tracing::Span::current().record("found", entry.is_some());
1146 Ok(entry)
1147 }
1148 .instrument(span)
1149 .await
1150 }
1151
1152 pub(crate) async fn max_seq(&self) -> Result<Option<JournalSeq>, DurableError> {
1162 let max: Option<i64> = zeph_db::query_scalar(sql!("SELECT MAX(seq) FROM durable_journal"))
1163 .fetch_one(&self.pool)
1164 .await
1165 .map_err(|e| DurableError::storage("max_seq", e))?;
1166 Ok(max.map(JournalSeq::new))
1167 }
1168
1169 pub(crate) fn promise_waiters(&self) -> &NotifyRegistry {
1171 &self.promise_waiters
1172 }
1173
1174 pub(crate) fn timer_waiters(&self) -> &NotifyRegistry {
1176 &self.timer_waiters
1177 }
1178
1179 pub(crate) async fn insert_promise(
1188 &self,
1189 id: PromiseId,
1190 execution_id: ExecutionId,
1191 resolver_token_hash: [u8; 32],
1192 created_at_ms: i64,
1193 ) -> Result<(), DurableError> {
1194 let span = tracing::info_span!("durable.promise.create", promise_id = %id.as_uuid());
1195 async move {
1196 zeph_db::query(sql!(
1197 "INSERT INTO durable_promises
1198 (promise_id, execution_id, resolver_token_hash, resolved, payload, created_at, resolved_at)
1199 VALUES (?, ?, ?, 0, NULL, ?, NULL)"
1200 ))
1201 .bind(id.as_uuid().to_string())
1202 .bind(execution_id.as_uuid().to_string())
1203 .bind(resolver_token_hash.to_vec())
1204 .bind(created_at_ms)
1205 .execute(&self.pool)
1206 .await
1207 .map_err(|e| DurableError::storage("insert_promise", e))?;
1208 Ok(())
1209 }
1210 .instrument(span)
1211 .await
1212 }
1213
1214 pub(crate) async fn promise_state(
1221 &self,
1222 id: PromiseId,
1223 ) -> Result<Option<PromiseRecord>, DurableError> {
1224 let row: Option<PromiseRowRead> = zeph_db::query_as(sql!(
1225 "SELECT execution_id, resolver_token_hash, resolved, payload
1226 FROM durable_promises WHERE promise_id = ?"
1227 ))
1228 .bind(id.as_uuid().to_string())
1229 .fetch_optional(&self.pool)
1230 .await
1231 .map_err(|e| DurableError::storage("promise_state", e))?;
1232 let Some((exec, hash, resolved, payload)) = row else {
1233 return Ok(None);
1234 };
1235 Ok(Some(PromiseRecord {
1236 execution_id: parse_execution_id(&exec)?,
1237 resolver_token_hash: slice_to_array32(&hash, "promise resolver_token_hash")?,
1238 resolved: resolved != 0,
1239 payload,
1240 }))
1241 }
1242
1243 pub(crate) async fn resolve_promise(
1254 &self,
1255 id: PromiseId,
1256 execution_id: ExecutionId,
1257 value_plaintext: &[u8],
1258 resolved_at_ms: i64,
1259 ) -> Result<bool, DurableError> {
1260 let span = tracing::info_span!("durable.promise.resolve", promise_id = %id.as_uuid());
1261 async move {
1262 ensure_payload_within_limit(value_plaintext.len(), self.max_payload_bytes)?;
1263 let aad = promise_payload_aad(execution_id, id);
1264 let sealed = self.seal_payload(value_plaintext, &aad)?;
1265 let affected = zeph_db::query(sql!(
1266 "UPDATE durable_promises SET resolved = 1, payload = ?, resolved_at = ?
1267 WHERE promise_id = ? AND resolved = 0"
1268 ))
1269 .bind(sealed)
1270 .bind(resolved_at_ms)
1271 .bind(id.as_uuid().to_string())
1272 .execute(&self.pool)
1273 .await
1274 .map_err(|e| DurableError::storage("resolve_promise", e))?
1275 .rows_affected();
1276 if affected > 0 {
1277 self.promise_waiters.wake(id.as_uuid());
1278 }
1279 Ok(affected > 0)
1280 }
1281 .instrument(span)
1282 .await
1283 }
1284
1285 pub(crate) async fn claim_promise_notification(
1298 &self,
1299 id: PromiseId,
1300 notified_at_ms: i64,
1301 ) -> Result<bool, DurableError> {
1302 let span = tracing::info_span!("durable.promise.claim_notify", promise_id = %id.as_uuid());
1303 async move {
1304 let affected = zeph_db::query(sql!(
1305 "UPDATE durable_promises SET notified_at = ?
1306 WHERE promise_id = ? AND notified_at IS NULL"
1307 ))
1308 .bind(notified_at_ms)
1309 .bind(id.as_uuid().to_string())
1310 .execute(&self.pool)
1311 .await
1312 .map_err(|e| DurableError::storage("claim_promise_notification", e))?
1313 .rows_affected();
1314 Ok(affected > 0)
1315 }
1316 .instrument(span)
1317 .await
1318 }
1319
1320 pub(crate) fn open_promise_payload(
1327 &self,
1328 id: PromiseId,
1329 execution_id: ExecutionId,
1330 sealed: &[u8],
1331 ) -> Result<Bytes, DurableError> {
1332 ensure_payload_within_limit(
1333 sealed.len(),
1334 self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1335 )?;
1336 let aad = promise_payload_aad(execution_id, id);
1337 self.open_payload(sealed, &aad)
1338 }
1339
1340 pub(crate) async fn arm_timer(
1348 &self,
1349 id: TimerId,
1350 execution_id: ExecutionId,
1351 due_at_ms: i64,
1352 created_at_ms: i64,
1353 ) -> Result<(), DurableError> {
1354 let span = tracing::info_span!("durable.timer.arm", timer_id = %id.as_uuid(), due_at_ms);
1355 async move {
1356 zeph_db::query(sql!(
1357 "INSERT INTO durable_timers (timer_id, execution_id, due_at, fired, created_at)
1358 VALUES (?, ?, ?, 0, ?)"
1359 ))
1360 .bind(id.as_uuid().to_string())
1361 .bind(execution_id.as_uuid().to_string())
1362 .bind(due_at_ms)
1363 .bind(created_at_ms)
1364 .execute(&self.pool)
1365 .await
1366 .map_err(|e| DurableError::storage("arm_timer", e))?;
1367 Ok(())
1368 }
1369 .instrument(span)
1370 .await
1371 }
1372
1373 pub(crate) async fn timer_state(
1379 &self,
1380 id: TimerId,
1381 ) -> Result<Option<(i64, bool)>, DurableError> {
1382 let row: Option<(i64, i64)> = zeph_db::query_as(sql!(
1383 "SELECT due_at, fired FROM durable_timers WHERE timer_id = ?"
1384 ))
1385 .bind(id.as_uuid().to_string())
1386 .fetch_optional(&self.pool)
1387 .await
1388 .map_err(|e| DurableError::storage("timer_state", e))?;
1389 Ok(row.map(|(due_at, fired)| (due_at, fired != 0)))
1390 }
1391
1392 pub(crate) async fn due_timers(&self, now_ms: i64) -> Result<Vec<TimerId>, DurableError> {
1402 let rows: Vec<(String,)> = zeph_db::query_as(sql!(
1403 "SELECT timer_id FROM durable_timers WHERE fired = 0 AND due_at <= ? ORDER BY due_at"
1404 ))
1405 .bind(now_ms)
1406 .fetch_all(&self.pool)
1407 .await
1408 .map_err(|e| DurableError::storage("due_timers", e))?;
1409 rows.into_iter().map(|(id,)| parse_timer_id(&id)).collect()
1410 }
1411
1412 pub(crate) async fn mark_timer_fired(&self, id: TimerId) -> Result<bool, DurableError> {
1420 let span = tracing::info_span!("durable.timer.fire", timer_id = %id.as_uuid());
1421 async move {
1422 let affected = zeph_db::query(sql!(
1423 "UPDATE durable_timers SET fired = 1 WHERE timer_id = ? AND fired = 0"
1424 ))
1425 .bind(id.as_uuid().to_string())
1426 .execute(&self.pool)
1427 .await
1428 .map_err(|e| DurableError::storage("mark_timer_fired", e))?
1429 .rows_affected();
1430 if affected > 0 {
1431 self.timer_waiters.wake(id.as_uuid());
1432 }
1433 Ok(affected > 0)
1434 }
1435 .instrument(span)
1436 .await
1437 }
1438
1439 fn open_foldable_steps(
1445 &self,
1446 execution_id: ExecutionId,
1447 rows: Vec<FoldableRowRead>,
1448 ) -> Result<Vec<FoldedStep>, DurableError> {
1449 let mut folded = Vec::with_capacity(rows.len());
1450 for (step_raw, idem, version, payload) in rows {
1451 let step = u32::try_from(step_raw).map_err(|_| DurableError::Decode {
1452 context: "checkpoint step_id out of u32 range",
1453 })?;
1454 let idem_bytes = idem.ok_or(DurableError::Decode {
1455 context: "checkpoint step result missing idem_key",
1456 })?;
1457 let idem_key =
1458 IdempotencyKey::from_bytes(slice_to_array32(&idem_bytes, "checkpoint idem_key")?);
1459 let sealed = payload.ok_or(DurableError::Decode {
1460 context: "checkpoint step result missing payload",
1461 })?;
1462 let aad = PayloadAad::new(
1463 execution_id,
1464 StepId::new(step),
1465 EntryKindTag::StepResult,
1466 Some(idem_key),
1467 );
1468 let plaintext = self.open_payload(&sealed, &aad)?;
1469 let payload_version =
1470 u8::try_from(version.unwrap_or(1)).map_err(|_| DurableError::Decode {
1471 context: "checkpoint payload_version out of u8 range",
1472 })?;
1473 folded.push(FoldedStep {
1474 step_id: step,
1475 idem_key: *idem_key.as_bytes(),
1476 payload_version,
1477 payload: plaintext,
1478 });
1479 }
1480 Ok(folded)
1481 }
1482
1483 pub(crate) async fn checkpoint_fold(
1504 &self,
1505 execution_id: ExecutionId,
1506 up_to_step: u32,
1507 ) -> Result<u64, DurableError> {
1508 let span = tracing::info_span!(
1509 "durable.journal.checkpoint",
1510 execution_id = %execution_id.as_uuid(),
1511 folded_count = tracing::field::Empty,
1512 );
1513 async move {
1514 let exec = execution_id.as_uuid().to_string();
1515 let rows: Vec<FoldableRowRead> = zeph_db::query_as(sql!(
1516 "SELECT step_id, idem_key, payload_version, payload FROM durable_journal
1517 WHERE execution_id = ? AND entry_kind = 'step_result'
1518 AND effect_class = 'idempotent' AND step_id < ?
1519 ORDER BY step_id"
1520 ))
1521 .bind(&exec)
1522 .bind(i64::from(up_to_step))
1523 .fetch_all(&self.pool)
1524 .await
1525 .map_err(|e| DurableError::storage("checkpoint", e))?;
1526 if rows.is_empty() {
1527 return Ok(0);
1528 }
1529
1530 let mut folded = self.open_foldable_steps(execution_id, rows)?;
1532 let lens: Vec<usize> = folded.iter().map(|s| s.payload.len()).collect();
1533 let take = crate::retention::fold_prefix_len(
1534 &lens,
1535 crate::retention::checkpoint_budget(self.max_payload_bytes),
1536 );
1537 if take == 0 {
1538 return Ok(0);
1541 }
1542 folded.truncate(take);
1543 let fold_end = folded.last().map_or(up_to_step, |s| s.step_id.saturating_add(1));
1544
1545 let snapshot = encode_checkpoint(&folded);
1546 let snap_aad =
1547 PayloadAad::new(execution_id, StepId::new(fold_end), EntryKindTag::Checkpoint, None);
1548 let sealed_snapshot = self.seal_payload(&snapshot, &snap_aad)?;
1549
1550 let count = folded.len() as u64;
1555
1556 let mut tx = zeph_db::begin_write(&self.pool)
1557 .await
1558 .map_err(|e| DurableError::storage("checkpoint", e))?;
1559 zeph_db::query(sql!(
1560 "INSERT INTO durable_journal
1561 (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at, folded_count)
1562 VALUES (?, ?, 'checkpoint', NULL, NULL, ?, ?, NULL, ?, ?)"
1563 ))
1564 .bind(&exec)
1565 .bind(i64::from(fold_end))
1566 .bind(sealed_snapshot)
1567 .bind(i32::from(crate::step::PAYLOAD_VERSION))
1568 .bind(now_unix_millis())
1569 .bind(i64::try_from(count).unwrap_or(i64::MAX))
1570 .execute(&mut *tx)
1571 .await
1572 .map_err(|e| DurableError::storage("checkpoint", e))?;
1573 zeph_db::query(sql!(
1574 "DELETE FROM durable_journal
1575 WHERE execution_id = ? AND entry_kind = 'step_result'
1576 AND effect_class = 'idempotent' AND step_id < ?"
1577 ))
1578 .bind(&exec)
1579 .bind(i64::from(fold_end))
1580 .execute(&mut *tx)
1581 .await
1582 .map_err(|e| DurableError::storage("checkpoint", e))?;
1583 tx.commit()
1584 .await
1585 .map_err(|e| DurableError::storage("checkpoint", e))?;
1586
1587 tracing::Span::current().record("folded_count", count);
1588 Ok(count)
1589 }
1590 .instrument(span)
1591 .await
1592 }
1593
1594 pub(crate) async fn read_checkpoints(
1607 &self,
1608 execution_id: ExecutionId,
1609 ) -> Result<Vec<JournalEntry>, DurableError> {
1610 let rows: Vec<(i64, Option<Vec<u8>>)> = zeph_db::query_as(sql!(
1611 "SELECT step_id, payload FROM durable_journal
1612 WHERE execution_id = ? AND entry_kind = 'checkpoint' ORDER BY step_id"
1613 ))
1614 .bind(execution_id.as_uuid().to_string())
1615 .fetch_all(&self.pool)
1616 .await
1617 .map_err(|e| DurableError::storage("read_checkpoints", e))?;
1618 if rows.is_empty() {
1619 return Ok(Vec::new());
1620 }
1621 let mut folded: CheckpointSnapshot = Vec::new();
1622 for (up_to, payload) in rows {
1623 let up_to = u32::try_from(up_to).map_err(|_| DurableError::Decode {
1624 context: "checkpoint up_to_step out of u32 range",
1625 })?;
1626 let sealed = payload.ok_or(DurableError::Decode {
1627 context: "checkpoint entry missing snapshot payload",
1628 })?;
1629 ensure_payload_within_limit(
1630 sealed.len(),
1631 self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1632 )?;
1633 let aad = PayloadAad::new(
1634 execution_id,
1635 StepId::new(up_to),
1636 EntryKindTag::Checkpoint,
1637 None,
1638 );
1639 let plaintext = self.open_payload(&sealed, &aad)?;
1640 folded.extend(decode_checkpoint(&plaintext)?);
1641 }
1642 let kind = self.lookup_kind(execution_id).await?;
1645 let entries = folded
1646 .into_iter()
1647 .map(|step| JournalEntry {
1648 seq: None,
1649 execution_id,
1650 kind,
1651 step_id: StepId::new(step.step_id),
1652 entry: EntryKind::StepResult {
1653 idempotency_key: IdempotencyKey::from_bytes(step.idem_key),
1654 payload: step.payload,
1655 effect: crate::EffectClass::Idempotent,
1656 payload_version: step.payload_version,
1657 },
1658 created_at_ms: 0,
1659 })
1660 .collect();
1661 Ok(entries)
1662 }
1663
1664 async fn delete_prune_batch(
1681 &self,
1682 cutoffs: crate::retention::PruneCutoffs,
1683 batch: u64,
1684 ) -> Result<u64, DurableError> {
1685 let mut tx = zeph_db::begin_write(&self.pool)
1686 .await
1687 .map_err(|e| DurableError::storage("prune", e))?;
1688
1689 #[cfg(feature = "postgres")]
1695 zeph_db::query(sql!(
1696 "SELECT execution_id FROM durable_executions
1697 WHERE finalized_at IS NOT NULL
1698 AND ( (status = 'completed' AND finalized_at <= ?)
1699 OR (status IN ('failed', 'aborted', 'canceled') AND finalized_at <= ?) )
1700 ORDER BY finalized_at LIMIT ?
1701 FOR UPDATE"
1702 ))
1703 .bind(cutoffs.completed_before_ms)
1704 .bind(cutoffs.failed_before_ms)
1705 .bind(i64::try_from(batch).unwrap_or(i64::MAX))
1706 .execute(&mut *tx)
1707 .await
1708 .map_err(|e| DurableError::storage("prune", e))?;
1709
1710 let ids: Vec<(String,)> = zeph_db::query_as(sql!(
1711 "SELECT execution_id FROM durable_executions
1712 WHERE finalized_at IS NOT NULL
1713 AND ( (status = 'completed' AND finalized_at <= ?)
1714 OR (status IN ('failed', 'aborted', 'canceled') AND finalized_at <= ?) )
1715 ORDER BY finalized_at LIMIT ?"
1716 ))
1717 .bind(cutoffs.completed_before_ms)
1718 .bind(cutoffs.failed_before_ms)
1719 .bind(i64::try_from(batch).unwrap_or(i64::MAX))
1720 .fetch_all(&mut *tx)
1721 .await
1722 .map_err(|e| DurableError::storage("prune", e))?;
1723 if ids.is_empty() {
1724 tx.commit()
1725 .await
1726 .map_err(|e| DurableError::storage("prune", e))?;
1727 return Ok(0);
1728 }
1729 let journal = sql!("DELETE FROM durable_journal WHERE execution_id = ?");
1730 let promises = sql!("DELETE FROM durable_promises WHERE execution_id = ?");
1731 let timers = sql!("DELETE FROM durable_timers WHERE execution_id = ?");
1732 let integrity = sql!("DELETE FROM durable_execution_integrity WHERE execution_id = ?");
1741 let executions = sql!(
1744 "DELETE FROM durable_executions
1745 WHERE execution_id = ?
1746 AND finalized_at IS NOT NULL
1747 AND ( (status = 'completed' AND finalized_at <= ?)
1748 OR (status IN ('failed', 'aborted', 'canceled') AND finalized_at <= ?) )"
1749 );
1750 let mut removed = 0u64;
1751 for (id,) in &ids {
1752 for stmt in [journal, promises, timers, integrity] {
1753 zeph_db::query(stmt)
1754 .bind(id)
1755 .execute(&mut *tx)
1756 .await
1757 .map_err(|e| DurableError::storage("prune", e))?;
1758 }
1759 let result = zeph_db::query(executions)
1760 .bind(id)
1761 .bind(cutoffs.completed_before_ms)
1762 .bind(cutoffs.failed_before_ms)
1763 .execute(&mut *tx)
1764 .await
1765 .map_err(|e| DurableError::storage("prune", e))?;
1766 removed += result.rows_affected();
1767 }
1768 tx.commit()
1769 .await
1770 .map_err(|e| DurableError::storage("prune", e))?;
1771 Ok(removed)
1772 }
1773
1774 async fn sweep_orphan_batch(
1793 &self,
1794 lock_dir: &std::path::Path,
1795 cutoff_ms: i64,
1796 batch: u64,
1797 cursor: Option<crate::retention::SweepCursor>,
1798 ) -> Result<crate::retention::SweepBatchOutcome, DurableError> {
1799 let (after_updated_at, after_exec) = cursor.map_or((i64::MIN, String::new()), |c| {
1803 (c.updated_at_ms, c.execution_id)
1804 });
1805
1806 let candidates: Vec<(String, i64)> = zeph_db::query_as(sql!(
1807 "SELECT execution_id, updated_at FROM durable_executions
1808 WHERE status = 'running' AND updated_at <= ?
1809 AND (updated_at > ? OR (updated_at = ? AND execution_id > ?))
1810 ORDER BY updated_at, execution_id LIMIT ?"
1811 ))
1812 .bind(cutoff_ms)
1813 .bind(after_updated_at)
1814 .bind(after_updated_at)
1815 .bind(&after_exec)
1816 .bind(i64::try_from(batch).unwrap_or(i64::MAX))
1817 .fetch_all(&self.pool)
1818 .await
1819 .map_err(|e| DurableError::storage("sweep_orphans", e))?;
1820
1821 let scanned = u64::try_from(candidates.len()).unwrap_or(u64::MAX);
1822 let next_cursor = candidates
1823 .last()
1824 .map(|(id, updated_at)| crate::retention::SweepCursor {
1825 updated_at_ms: *updated_at,
1826 execution_id: id.clone(),
1827 });
1828
1829 let now = now_unix_millis();
1830 let abort = sql!(
1831 "UPDATE durable_executions SET status = 'aborted', finalized_at = ?, updated_at = ?
1832 WHERE execution_id = ? AND status = 'running' AND finalized_at IS NULL"
1833 );
1834 let mut aborted = 0u64;
1835 for (exec_str, _updated_at) in &candidates {
1836 let Ok(execution_id) = parse_execution_id(exec_str) else {
1837 continue;
1838 };
1839 match ExecutionLock::acquire(lock_dir, execution_id) {
1840 Ok(_lock) => {
1841 let result = zeph_db::query(abort)
1842 .bind(now)
1843 .bind(now)
1844 .bind(exec_str)
1845 .execute(&self.pool)
1846 .await
1847 .map_err(|e| DurableError::storage("sweep_orphans", e))?;
1848 aborted += result.rows_affected();
1849 }
1851 Err(DurableError::ExecutionLocked { .. }) => {
1852 }
1854 Err(e) => return Err(e),
1855 }
1856 }
1857 Ok(crate::retention::SweepBatchOutcome {
1858 scanned,
1859 aborted,
1860 next_cursor,
1861 })
1862 }
1863
1864 fn seal_payload(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, DurableError> {
1866 match &self.cipher {
1867 Some(cipher) => Ok(cipher.seal(plaintext, aad)?),
1868 None => Ok(plaintext.to_vec()),
1869 }
1870 }
1871
1872 fn open_payload(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Bytes, DurableError> {
1874 match &self.cipher {
1875 Some(cipher) => Ok(Bytes::from(cipher.open(sealed, aad)?)),
1876 None => Ok(Bytes::copy_from_slice(sealed)),
1877 }
1878 }
1879
1880 fn control_hmac(
1885 &self,
1886 entry: &JournalEntry,
1887 idem_key: Option<&IdempotencyKey>,
1888 ) -> Option<Vec<u8>> {
1889 self.compute_control_hmac(
1890 entry.execution_id,
1891 entry.step_id,
1892 entry.entry.tag(),
1893 idem_key,
1894 )
1895 .map(|h| h.to_vec())
1896 }
1897
1898 fn compute_control_hmac(
1903 &self,
1904 execution_id: ExecutionId,
1905 step_id: StepId,
1906 tag: &'static str,
1907 idem_key: Option<&IdempotencyKey>,
1908 ) -> Option<[u8; 32]> {
1909 let key = self.hmac_key.as_ref()?;
1910 Some(Self::keyed_control_hmac(
1911 key,
1912 execution_id,
1913 step_id,
1914 tag,
1915 idem_key,
1916 ))
1917 }
1918
1919 fn keyed_control_hmac(
1922 key: &[u8; 32],
1923 execution_id: ExecutionId,
1924 step_id: StepId,
1925 tag: &'static str,
1926 idem_key: Option<&IdempotencyKey>,
1927 ) -> [u8; 32] {
1928 let mut input = Vec::with_capacity(16 + 4 + 16 + 32);
1929 input.extend_from_slice(execution_id.as_bytes());
1930 input.extend_from_slice(&step_id.value().to_le_bytes());
1931 input.extend_from_slice(tag.as_bytes());
1932 if let Some(k) = idem_key {
1933 input.extend_from_slice(k.as_bytes());
1934 }
1935 *blake3::keyed_hash(key, &input).as_bytes()
1936 }
1937
1938 fn verify_control_hmac(
1959 &self,
1960 execution_id: ExecutionId,
1961 step_id: StepId,
1962 tag: &'static str,
1963 idem_key: Option<&IdempotencyKey>,
1964 stored: Option<[u8; 32]>,
1965 ) -> Result<(), DurableError> {
1966 let Some(current_key) = self.hmac_key.as_ref() else {
1967 return if stored.is_some() {
1968 Err(DurableError::ControlIntegrity)
1969 } else {
1970 Ok(())
1971 };
1972 };
1973 let Some(stored) = stored else {
1974 return Err(DurableError::ControlIntegrity);
1975 };
1976 let expected_current =
1977 Self::keyed_control_hmac(current_key, execution_id, step_id, tag, idem_key);
1978 if blake3::Hash::from(expected_current) == blake3::Hash::from(stored) {
1979 return Ok(());
1980 }
1981 if let Some(previous_key) = self.previous_hmac_key.as_ref() {
1982 let expected_previous =
1983 Self::keyed_control_hmac(previous_key, execution_id, step_id, tag, idem_key);
1984 if blake3::Hash::from(expected_previous) == blake3::Hash::from(stored) {
1985 return Ok(());
1986 }
1987 }
1988 Err(DurableError::ControlIntegrity)
1989 }
1990
1991 fn compute_hwm_hmac(
1999 execution_id: ExecutionId,
2000 max_committed_step_id: u32,
2001 committed_result_count: u64,
2002 key_epoch: u32,
2003 key: &[u8; 32],
2004 ) -> [u8; 32] {
2005 let mut input = Vec::with_capacity(16 + 4 + 8 + 4);
2006 input.extend_from_slice(execution_id.as_bytes());
2007 input.extend_from_slice(&max_committed_step_id.to_le_bytes());
2008 input.extend_from_slice(&committed_result_count.to_le_bytes());
2009 input.extend_from_slice(&key_epoch.to_le_bytes());
2010 *blake3::keyed_hash(key, &input).as_bytes()
2011 }
2012
2013 fn resolve_hwm_key(&self, epoch: u32) -> Option<[u8; 32]> {
2022 if let Some(slot) = &self.hwm_key
2023 && slot.epoch == epoch
2024 {
2025 return Some(slot.key);
2026 }
2027 if let Some(slot) = &self.hwm_key_previous
2028 && slot.epoch == epoch
2029 {
2030 return Some(slot.key);
2031 }
2032 None
2033 }
2034
2035 async fn bump_hwm_for_step_result(
2046 &self,
2047 tx: &mut zeph_db::DbTransaction<'_>,
2048 execution_id: ExecutionId,
2049 step_id: StepId,
2050 ) -> Result<(), DurableError> {
2051 let Some(slot) = &self.hwm_key else {
2052 return Ok(());
2053 };
2054 let exec = execution_id.as_uuid().to_string();
2055 let existing: Option<(i64, i64)> = zeph_db::query_as(sql!(
2056 "SELECT max_committed_step_id, committed_result_count
2057 FROM durable_execution_integrity WHERE execution_id = ?"
2058 ))
2059 .bind(&exec)
2060 .fetch_optional(&mut **tx)
2061 .await
2062 .map_err(|e| DurableError::storage("hwm_bump", e))?;
2063 let (prev_max, prev_count) = existing.unwrap_or((0, 0));
2064 let new_max = prev_max.max(i64::from(step_id.value()));
2065 let new_count = prev_count.saturating_add(1);
2066 let hmac = Self::compute_hwm_hmac(
2067 execution_id,
2068 u32::try_from(new_max).unwrap_or(u32::MAX),
2069 u64::try_from(new_count).unwrap_or(u64::MAX),
2070 slot.epoch,
2071 &slot.key,
2072 );
2073 zeph_db::query(sql!(
2074 "INSERT INTO durable_execution_integrity
2075 (execution_id, key_epoch, max_committed_step_id, committed_result_count, hwm_hmac, updated_at)
2076 VALUES (?, ?, ?, ?, ?, ?)
2077 ON CONFLICT(execution_id) DO UPDATE SET
2078 key_epoch = excluded.key_epoch,
2079 max_committed_step_id = excluded.max_committed_step_id,
2080 committed_result_count = excluded.committed_result_count,
2081 hwm_hmac = excluded.hwm_hmac,
2082 updated_at = excluded.updated_at"
2083 ))
2084 .bind(&exec)
2085 .bind(i64::from(slot.epoch))
2086 .bind(new_max)
2087 .bind(new_count)
2088 .bind(hmac.to_vec())
2089 .bind(now_unix_millis())
2090 .execute(&mut **tx)
2091 .await
2092 .map_err(|e| DurableError::storage("hwm_bump", e))?;
2093 Ok(())
2094 }
2095
2096 async fn verify_high_water_mark(&self, execution_id: ExecutionId) -> Result<(), DurableError> {
2104 if self.hwm_key.is_none() {
2105 return Ok(());
2106 }
2107 if let Err(error) = self.check_high_water_mark(execution_id).await {
2108 if let Err(finalize_error) = self.finalize(execution_id, ExecutionStatus::Aborted).await
2109 {
2110 tracing::warn!(
2111 error = %finalize_error,
2112 "failed to mark HWM-integrity-failed execution aborted"
2113 );
2114 }
2115 return Err(error);
2116 }
2117 Ok(())
2118 }
2119
2120 pub async fn find_unsealed_resumable_executions(
2133 &self,
2134 ) -> Result<Vec<ExecutionId>, DurableError> {
2135 let rows: Vec<(String,)> = zeph_db::query_as(sql!(
2136 "SELECT e.execution_id FROM durable_executions e
2137 WHERE e.status = 'running'
2138 AND NOT EXISTS (
2139 SELECT 1 FROM durable_execution_integrity i WHERE i.execution_id = e.execution_id
2140 )
2141 AND (
2142 EXISTS (
2143 SELECT 1 FROM durable_journal j
2144 WHERE j.execution_id = e.execution_id AND j.entry_kind = 'step_result'
2145 )
2146 OR EXISTS (
2147 SELECT 1 FROM durable_journal j
2148 WHERE j.execution_id = e.execution_id AND j.entry_kind = 'checkpoint'
2149 AND j.folded_count > 0
2150 )
2151 )"
2152 ))
2153 .fetch_all(&self.pool)
2154 .await
2155 .map_err(|e| DurableError::storage("seal_integrity_scan", e))?;
2156
2157 rows.into_iter()
2158 .map(|(id,)| {
2159 ExecutionId::parse_str(&id).map_err(|_| DurableError::Decode {
2160 context: "malformed execution_id in durable_executions",
2161 })
2162 })
2163 .collect()
2164 }
2165
2166 async fn committed_step_result_count(
2172 &self,
2173 execution_id: ExecutionId,
2174 ) -> Result<u64, DurableError> {
2175 let exec = execution_id.as_uuid().to_string();
2176 let live_count: i64 = zeph_db::query_scalar(sql!(
2177 "SELECT COUNT(*) FROM durable_journal
2178 WHERE execution_id = ? AND entry_kind = 'step_result'"
2179 ))
2180 .bind(&exec)
2181 .fetch_one(&self.pool)
2182 .await
2183 .map_err(|e| DurableError::storage("hwm_verify", e))?;
2184 let folded_sum: i64 = zeph_db::query_scalar(sql!(
2185 "SELECT COALESCE(SUM(folded_count), 0) FROM durable_journal
2186 WHERE execution_id = ? AND entry_kind = 'checkpoint'"
2187 ))
2188 .bind(&exec)
2189 .fetch_one(&self.pool)
2190 .await
2191 .map_err(|e| DurableError::storage("hwm_verify", e))?;
2192 Ok(u64::try_from(live_count.saturating_add(folded_sum)).unwrap_or(0))
2193 }
2194
2195 async fn check_high_water_mark(&self, execution_id: ExecutionId) -> Result<(), DurableError> {
2212 let exec = execution_id.as_uuid().to_string();
2213 let stored: Option<(i64, i64, i64, Vec<u8>)> = zeph_db::query_as(sql!(
2214 "SELECT key_epoch, max_committed_step_id, committed_result_count, hwm_hmac
2215 FROM durable_execution_integrity WHERE execution_id = ?"
2216 ))
2217 .bind(&exec)
2218 .fetch_optional(&self.pool)
2219 .await
2220 .map_err(|e| DurableError::storage("hwm_verify", e))?;
2221 let Some((epoch_raw, max_step_raw, count_raw, hmac)) = stored else {
2222 if self.hwm_key.is_some()
2223 && self.integrity_sealed
2224 && !self.integrity_grandfather.contains(&execution_id)
2225 && self.committed_step_result_count(execution_id).await? >= 1
2226 {
2227 return Err(DurableError::HighWaterMarkIntegrity {
2228 execution_id,
2229 reason: "integrity_row_absent_post_seal",
2230 hint: "TAMPER: this backend is sealed against pre-feature integrity-row \
2231 absence, this execution is keyed and not grandfathered, and it has \
2232 committed StepResults — a legitimate keyed execution can never reach \
2233 this state (the integrity row is written atomically with its first \
2234 committed StepResult), so an absent row here means the row was \
2235 deleted outside the write path",
2236 });
2237 }
2238 return Ok(());
2239 };
2240
2241 let fail =
2246 |reason: &'static str, hint: &'static str| DurableError::HighWaterMarkIntegrity {
2247 execution_id,
2248 reason,
2249 hint,
2250 };
2251 let tamper = |reason: &'static str| {
2252 fail(
2253 reason,
2254 "TAMPER: the signed high-water-mark did not authenticate under any key this \
2255 backend holds for the recorded epoch",
2256 )
2257 };
2258
2259 let epoch = u32::try_from(epoch_raw).map_err(|_| tamper("hmac_mismatch"))?;
2260 let Some(key) = self.resolve_hwm_key(epoch) else {
2261 return Err(fail(
2262 "key_epoch_unresolvable",
2263 "possibly re-keyed: this execution's signed key_epoch is neither the current key \
2264 nor a registered previous rotation key — if ZEPH_DURABLE_KEY was recently \
2265 rotated, ensure the rotation window is still open (ZEPH_DURABLE_KEY_PREVIOUS \
2266 present and [durable] previous_key_id set); the window is closed permanently by \
2267 `zeph durable rotate-key --drop-previous`. The durable resume path cannot \
2268 proceed without it (no interactive override)",
2269 ));
2270 };
2271 let stored_hmac =
2272 <[u8; 32]>::try_from(hmac.as_slice()).map_err(|_| tamper("hmac_mismatch"))?;
2273 let max_step = u32::try_from(max_step_raw).unwrap_or(u32::MAX);
2274 let count = u64::try_from(count_raw).unwrap_or(u64::MAX);
2275 let expected = Self::compute_hwm_hmac(execution_id, max_step, count, epoch, &key);
2276 if blake3::Hash::from(expected) != blake3::Hash::from(stored_hmac) {
2277 return Err(tamper("hmac_mismatch"));
2278 }
2279
2280 let recomputed = self.committed_step_result_count(execution_id).await?;
2281 if recomputed != count {
2282 return Err(fail(
2283 "count_mismatch",
2284 "TAMPER: the recomputed committed-result count (surviving StepResult rows plus \
2285 every checkpoint's folded_count) disagrees with the signed value — a committed \
2286 result was likely deleted outside the write path",
2287 ));
2288 }
2289 Ok(())
2290 }
2291
2292 fn prepare_row(&self, entry: &JournalEntry) -> Result<JournalRow, DurableError> {
2294 let execution_id = entry.execution_id.as_uuid().to_string();
2295 let step_id = i64::from(entry.step_id.value());
2296 let created_at = entry.created_at_ms;
2297 let entry_kind = entry.entry.tag();
2298 match &entry.entry {
2299 EntryKind::StepResult {
2300 idempotency_key,
2301 payload,
2302 effect,
2303 payload_version,
2304 } => {
2305 ensure_payload_within_limit(payload.len(), self.max_payload_bytes)?;
2306 let aad = PayloadAad::new(
2307 entry.execution_id,
2308 entry.step_id,
2309 EntryKindTag::StepResult,
2310 Some(*idempotency_key),
2311 );
2312 let sealed = self.seal_payload(payload.as_ref(), &aad)?;
2313 Ok(JournalRow {
2314 execution_id,
2315 step_id,
2316 entry_kind,
2317 idem_key: Some(idempotency_key.as_bytes().to_vec()),
2318 effect_class: Some(effect.as_str()),
2319 payload: Some(sealed),
2320 payload_version: Some(i32::from(*payload_version)),
2321 hmac: None,
2322 created_at,
2323 })
2324 }
2325 EntryKind::EffectIntent {
2326 idempotency_key,
2327 effect,
2328 hmac: _,
2329 } => {
2330 let hmac = self.control_hmac(entry, Some(idempotency_key));
2333 Ok(JournalRow {
2334 execution_id,
2335 step_id,
2336 entry_kind,
2337 idem_key: Some(idempotency_key.as_bytes().to_vec()),
2338 effect_class: Some(effect.as_str()),
2339 payload: None,
2340 payload_version: None,
2341 hmac,
2342 created_at,
2343 })
2344 }
2345 EntryKind::PromiseCreated { .. }
2346 | EntryKind::PromiseResolved { .. }
2347 | EntryKind::TimerArmed { .. }
2348 | EntryKind::TimerFired { .. }
2349 | EntryKind::Checkpoint { .. } => {
2350 Err(DurableError::UnsupportedEntryKind { kind: entry_kind })
2351 }
2352 }
2353 }
2354
2355 async fn lookup_kind(&self, id: ExecutionId) -> Result<ExecutionKind, DurableError> {
2357 let kind: Option<String> = zeph_db::query_scalar(sql!(
2358 "SELECT kind FROM durable_executions WHERE execution_id = ?"
2359 ))
2360 .bind(id.as_uuid().to_string())
2361 .fetch_optional(&self.pool)
2362 .await
2363 .map_err(|e| DurableError::storage("read", e))?;
2364 let kind = kind.ok_or(DurableError::Decode {
2365 context: "journaled entries reference a missing execution row",
2366 })?;
2367 ExecutionKind::from_tag(&kind).ok_or(DurableError::Decode {
2368 context: "execution kind is not reconstructible (custom kind read-back unsupported)",
2369 })
2370 }
2371
2372 fn row_to_entry(
2374 &self,
2375 id: ExecutionId,
2376 kind: ExecutionKind,
2377 row: JournalRowRead,
2378 ) -> Result<JournalEntry, DurableError> {
2379 let (
2380 seq,
2381 step_id_raw,
2382 entry_kind,
2383 idem_key,
2384 effect_class,
2385 payload,
2386 payload_version,
2387 hmac,
2388 created_at,
2389 ) = row;
2390 let step_id =
2391 StepId::new(
2392 u32::try_from(step_id_raw).map_err(|_| DurableError::Decode {
2393 context: "step_id out of u32 range",
2394 })?,
2395 );
2396 let entry = match entry_kind.as_str() {
2397 "step_result" => {
2398 let idem_bytes = idem_key.ok_or(DurableError::Decode {
2399 context: "step_result idem_key missing",
2400 })?;
2401 let idem_key = IdempotencyKey::from_bytes(slice_to_array32(
2402 &idem_bytes,
2403 "step_result idem_key",
2404 )?);
2405 let effect = effect_class
2406 .as_deref()
2407 .and_then(crate::EffectClass::from_tag)
2408 .ok_or(DurableError::Decode {
2409 context: "step_result effect_class missing or invalid",
2410 })?;
2411 let sealed = payload.ok_or(DurableError::Decode {
2412 context: "step_result payload missing",
2413 })?;
2414 ensure_payload_within_limit(
2415 sealed.len(),
2416 self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
2417 )?;
2418 let aad = PayloadAad::new(id, step_id, EntryKindTag::StepResult, Some(idem_key));
2419 let opened = self.open_payload(&sealed, &aad)?;
2420 let version = u8::try_from(payload_version.unwrap_or(1)).map_err(|_| {
2421 DurableError::Decode {
2422 context: "payload_version out of u8 range",
2423 }
2424 })?;
2425 EntryKind::StepResult {
2426 idempotency_key: idem_key,
2427 payload: opened,
2428 effect,
2429 payload_version: version,
2430 }
2431 }
2432 "effect_intent" => {
2433 let idem_bytes = idem_key.ok_or(DurableError::Decode {
2434 context: "effect_intent idem_key missing",
2435 })?;
2436 let idem_key = IdempotencyKey::from_bytes(slice_to_array32(
2437 &idem_bytes,
2438 "effect_intent idem_key",
2439 )?);
2440 let effect = effect_class
2441 .as_deref()
2442 .and_then(crate::EffectClass::from_tag)
2443 .ok_or(DurableError::Decode {
2444 context: "effect_intent effect_class missing or invalid",
2445 })?;
2446 let hmac = hmac
2447 .map(|bytes| slice_to_array32(&bytes, "effect_intent hmac"))
2448 .transpose()?;
2449 self.verify_control_hmac(
2450 id,
2451 step_id,
2452 EntryKindTag::EffectIntent.as_str(),
2453 Some(&idem_key),
2454 hmac,
2455 )?;
2456 EntryKind::EffectIntent {
2457 idempotency_key: idem_key,
2458 effect,
2459 hmac,
2460 }
2461 }
2462 "checkpoint" => self.checkpoint_entry(id, step_id, payload)?,
2463 other => {
2464 return Err(DurableError::UnsupportedEntryKind {
2465 kind: static_entry_tag(other),
2466 });
2467 }
2468 };
2469 Ok(JournalEntry {
2470 seq: Some(JournalSeq::new(seq)),
2471 execution_id: id,
2472 kind,
2473 step_id,
2474 entry,
2475 created_at_ms: created_at,
2476 })
2477 }
2478
2479 fn checkpoint_entry(
2484 &self,
2485 id: ExecutionId,
2486 step_id: StepId,
2487 payload: Option<Vec<u8>>,
2488 ) -> Result<EntryKind, DurableError> {
2489 let sealed = payload.ok_or(DurableError::Decode {
2490 context: "checkpoint entry missing snapshot payload",
2491 })?;
2492 ensure_payload_within_limit(
2493 sealed.len(),
2494 self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
2495 )?;
2496 let aad = PayloadAad::new(id, step_id, EntryKindTag::Checkpoint, None);
2497 let snapshot = self.open_payload(&sealed, &aad)?;
2498 Ok(EntryKind::Checkpoint {
2499 up_to_step: step_id.value(),
2500 snapshot,
2501 })
2502 }
2503
2504 async fn rows_to_entries(
2506 &self,
2507 id: ExecutionId,
2508 rows: Vec<JournalRowRead>,
2509 ) -> Result<Vec<JournalEntry>, DurableError> {
2510 if rows.is_empty() {
2511 return Ok(Vec::new());
2512 }
2513 let kind = self.lookup_kind(id).await?;
2514 let mut entries = Vec::with_capacity(rows.len());
2515 for row in rows {
2516 entries.push(self.row_to_entry(id, kind, row)?);
2517 }
2518 Ok(entries)
2519 }
2520}
2521
2522impl Journal for LocalBackend {
2523 async fn append(&self, entry: JournalEntry) -> Result<JournalSeq, DurableError> {
2524 let span = tracing::info_span!(
2525 "durable.journal.append",
2526 execution_id = %entry.execution_id.as_uuid(),
2527 step_id = entry.step_id.value(),
2528 entry_kind = entry.entry.tag(),
2529 );
2530 async move {
2531 let row = self.prepare_row(&entry)?;
2532 let insert = sql!(
2533 "INSERT INTO durable_journal
2534 (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
2535 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
2536 RETURNING seq"
2537 );
2538 let seq: i64 = if matches!(entry.entry, EntryKind::StepResult { .. }) {
2542 let mut tx = zeph_db::begin_write(&self.pool)
2543 .await
2544 .map_err(|e| DurableError::storage("append", e))?;
2545 let (seq,): (i64,) = zeph_db::query_as(insert)
2546 .bind(row.execution_id)
2547 .bind(row.step_id)
2548 .bind(row.entry_kind)
2549 .bind(row.idem_key)
2550 .bind(row.effect_class)
2551 .bind(row.payload)
2552 .bind(row.payload_version)
2553 .bind(row.hmac)
2554 .bind(row.created_at)
2555 .fetch_one(&mut *tx)
2556 .await
2557 .map_err(|e| DurableError::storage("append", e))?;
2558 self.bump_hwm_for_step_result(&mut tx, entry.execution_id, entry.step_id)
2559 .await?;
2560 tx.commit()
2561 .await
2562 .map_err(|e| DurableError::storage("append", e))?;
2563 seq
2564 } else {
2565 let (seq,): (i64,) = zeph_db::query_as(insert)
2566 .bind(row.execution_id)
2567 .bind(row.step_id)
2568 .bind(row.entry_kind)
2569 .bind(row.idem_key)
2570 .bind(row.effect_class)
2571 .bind(row.payload)
2572 .bind(row.payload_version)
2573 .bind(row.hmac)
2574 .bind(row.created_at)
2575 .fetch_one(&self.pool)
2576 .await
2577 .map_err(|e| DurableError::storage("append", e))?;
2578 seq
2579 };
2580 Ok(JournalSeq::new(seq))
2581 }
2582 .instrument(span)
2583 .await
2584 }
2585
2586 async fn read_execution(&self, id: ExecutionId) -> Result<Vec<JournalEntry>, DurableError> {
2587 let span = tracing::info_span!(
2588 "durable.journal.read",
2589 execution_id = %id.as_uuid(),
2590 step_count = tracing::field::Empty,
2591 );
2592 async move {
2593 let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
2594 "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
2595 FROM durable_journal WHERE execution_id = ? ORDER BY seq"
2596 ))
2597 .bind(id.as_uuid().to_string())
2598 .fetch_all(&self.pool)
2599 .await
2600 .map_err(|e| DurableError::storage("read", e))?;
2601 let entries = self.rows_to_entries(id, rows).await?;
2602 tracing::Span::current().record("step_count", entries.len());
2603 Ok(entries)
2604 }
2605 .instrument(span)
2606 .await
2607 }
2608
2609 async fn read_execution_range(
2610 &self,
2611 id: ExecutionId,
2612 from_step_id: u32,
2613 limit: usize,
2614 ) -> Result<Vec<JournalEntry>, DurableError> {
2615 let span = tracing::info_span!(
2616 "durable.journal.read_segment",
2617 execution_id = %id.as_uuid(),
2618 from_step_id,
2619 count = tracing::field::Empty,
2620 );
2621 async move {
2622 let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
2623 "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
2624 FROM durable_journal WHERE execution_id = ? AND step_id >= ? ORDER BY step_id, seq LIMIT ?"
2625 ))
2626 .bind(id.as_uuid().to_string())
2627 .bind(i64::from(from_step_id))
2628 .bind(i64::try_from(limit).unwrap_or(i64::MAX))
2629 .fetch_all(&self.pool)
2630 .await
2631 .map_err(|e| DurableError::storage("read_segment", e))?;
2632 let entries = self.rows_to_entries(id, rows).await?;
2633 tracing::Span::current().record("count", entries.len());
2634 Ok(entries)
2635 }
2636 .instrument(span)
2637 .await
2638 }
2639
2640 async fn finalize(&self, id: ExecutionId, status: ExecutionStatus) -> Result<(), DurableError> {
2641 let span = tracing::info_span!(
2642 "durable.journal.finalize",
2643 execution_id = %id.as_uuid(),
2644 status = status.as_str(),
2645 );
2646 async move {
2647 let now = now_unix_millis();
2648 let finalized_at = (!status.is_running()).then_some(now);
2649 let mut tx = zeph_db::begin_write(&self.pool)
2650 .await
2651 .map_err(|e| DurableError::storage("finalize", e))?;
2652 zeph_db::query(sql!(
2657 "UPDATE durable_executions SET status = ?, updated_at = ?, finalized_at = ?
2658 WHERE execution_id = ? AND status = 'running'"
2659 ))
2660 .bind(status.as_str())
2661 .bind(now)
2662 .bind(finalized_at)
2663 .bind(id.as_uuid().to_string())
2664 .execute(&mut *tx)
2665 .await
2666 .map_err(|e| DurableError::storage("finalize", e))?;
2667 tx.commit()
2668 .await
2669 .map_err(|e| DurableError::storage("finalize", e))?;
2670 Ok(())
2671 }
2672 .instrument(span)
2673 .await
2674 }
2675
2676 async fn prune(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
2677 let now = now_unix_millis();
2678 crate::retention::prune_in_batches(policy, now, |cutoffs, batch| {
2679 self.delete_prune_batch(cutoffs, batch)
2680 })
2681 .await
2682 }
2683
2684 async fn sweep_orphans(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
2686 if policy.stale_running_after_secs == 0 {
2687 return Ok(0);
2688 }
2689 let Some(lock_dir) = self.lock_dir.clone() else {
2690 if !self
2691 .orphan_sweep_warned
2692 .swap(true, std::sync::atomic::Ordering::Relaxed)
2693 {
2694 tracing::warn!(
2695 "durable: crash-orphan sweep requires an on-disk advisory-lock dir; orphan \
2696 reclamation disabled for this backend (Postgres/:memory:/non-Unix)"
2697 );
2698 }
2699 return Ok(0);
2700 };
2701 let cutoff_ms = orphan_cutoff_ms(policy, now_unix_millis());
2702 crate::retention::sweep_orphans_in_batches(
2703 policy.prune_batch_size,
2704 cutoff_ms,
2705 |cutoff, batch, cursor| self.sweep_orphan_batch(&lock_dir, cutoff, batch, cursor),
2706 )
2707 .await
2708 }
2709}
2710
2711impl crate::sealed::Sealed for LocalBackend {}
2712
2713impl ExecutionBackend for LocalBackend {
2714 fn capabilities(&self) -> BackendCapabilities {
2715 BackendCapabilities {
2716 parallel_steps: true,
2717 cross_process: cfg!(feature = "postgres"),
2719 max_payload: usize::try_from(self.max_payload_bytes).unwrap_or(usize::MAX),
2720 }
2721 }
2722
2723 async fn lookup_committed_result(
2724 &self,
2725 id: ExecutionId,
2726 idem_key: IdempotencyKey,
2727 ) -> Result<Option<JournalEntry>, DurableError> {
2728 LocalBackend::lookup_committed_result(self, id, idem_key).await
2729 }
2730}
2731
2732struct JournalRow {
2734 execution_id: String,
2735 step_id: i64,
2736 entry_kind: &'static str,
2737 idem_key: Option<Vec<u8>>,
2738 effect_class: Option<&'static str>,
2739 payload: Option<Vec<u8>>,
2740 payload_version: Option<i32>,
2741 hmac: Option<Vec<u8>>,
2742 created_at: i64,
2743}
2744
2745type JournalRowRead = (
2753 i64,
2754 i64,
2755 String,
2756 Option<Vec<u8>>,
2757 Option<String>,
2758 Option<Vec<u8>>,
2759 Option<i32>,
2760 Option<Vec<u8>>,
2761 i64,
2762);
2763
2764type ControlHmacScanRow = (String, i64, Option<Vec<u8>>, Vec<u8>);
2767
2768type PromiseRowRead = (String, Vec<u8>, i64, Option<Vec<u8>>);
2771
2772type FoldableRowRead = (i64, Option<Vec<u8>>, Option<i32>, Option<Vec<u8>>);
2775
2776#[cfg(feature = "sqlite")]
2783fn lock_dir_for_path(path: &str) -> Option<std::path::PathBuf> {
2784 (path != ":memory:").then(|| std::path::PathBuf::from(format!("{path}.locks")))
2785}
2786
2787#[cfg(not(feature = "sqlite"))]
2788fn lock_dir_for_path(_path: &str) -> Option<std::path::PathBuf> {
2789 None
2790}
2791
2792#[cfg(all(test, not(feature = "sqlite")))]
2798mod postgres_lock_dir_tests {
2799 use super::lock_dir_for_path;
2800
2801 #[test]
2802 fn postgres_url_never_derives_a_lock_dir() {
2803 assert_eq!(
2804 lock_dir_for_path("postgres://user:secret@host/db"),
2805 None,
2806 "a Postgres connection URL (which may embed credentials) must never be used to mint \
2807 an on-disk lock directory name"
2808 );
2809 assert_eq!(lock_dir_for_path(":memory:"), None);
2810 }
2811}
2812
2813pub(crate) fn now_unix_millis() -> i64 {
2815 SystemTime::now()
2816 .duration_since(UNIX_EPOCH)
2817 .map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX))
2818}
2819
2820fn record_prior_status(outcome: CancelOutcome) {
2826 match outcome {
2827 CancelOutcome::Canceled => {
2828 tracing::Span::current().record("prior_status", "running");
2829 }
2830 CancelOutcome::AlreadyTerminal { status } => {
2831 tracing::Span::current().record("prior_status", status.as_str());
2832 }
2833 CancelOutcome::NotFound
2834 | CancelOutcome::LiveOwner { .. }
2835 | CancelOutcome::LivenessUnverifiable => {}
2836 }
2837}
2838
2839fn orphan_cutoff_ms(policy: &RetentionPolicy, now_ms: i64) -> i64 {
2842 let threshold =
2843 i64::try_from(policy.stale_running_after_secs.saturating_mul(1000)).unwrap_or(i64::MAX);
2844 now_ms.saturating_sub(threshold)
2845}
2846
2847fn slice_to_array32(bytes: &[u8], field: &'static str) -> Result<[u8; 32], DurableError> {
2849 <[u8; 32]>::try_from(bytes).map_err(|_| DurableError::Decode { context: field })
2850}
2851
2852fn parse_execution_id(text: &str) -> Result<ExecutionId, DurableError> {
2854 uuid::Uuid::parse_str(text)
2855 .map(ExecutionId::from_uuid)
2856 .map_err(|_| DurableError::Decode {
2857 context: "execution_id is not a valid UUID",
2858 })
2859}
2860
2861fn parse_timer_id(text: &str) -> Result<TimerId, DurableError> {
2863 uuid::Uuid::parse_str(text)
2864 .map(TimerId::from_uuid)
2865 .map_err(|_| DurableError::Decode {
2866 context: "timer_id is not a valid UUID",
2867 })
2868}
2869
2870fn promise_payload_aad(execution_id: ExecutionId, promise_id: PromiseId) -> PayloadAad {
2875 let binding = IdempotencyKey::derive(
2876 execution_id,
2877 StepId::new(0),
2878 promise_id.as_uuid().as_bytes(),
2879 );
2880 PayloadAad::new(
2881 execution_id,
2882 StepId::new(0),
2883 EntryKindTag::PromiseResolved,
2884 Some(binding),
2885 )
2886}
2887
2888fn static_entry_tag(tag: &str) -> &'static str {
2890 match tag {
2891 "promise_created" => "promise_created",
2892 "promise_resolved" => "promise_resolved",
2893 "timer_armed" => "timer_armed",
2894 "timer_fired" => "timer_fired",
2895 "checkpoint" => "checkpoint",
2896 _ => "unknown",
2897 }
2898}
2899
2900#[cfg(all(test, feature = "sqlite"))]
2905mod tests {
2906 use std::assert_matches;
2907
2908 use super::*;
2909 use crate::cipher::CipherError;
2910 use crate::effect::EffectClass;
2911
2912 struct XorCipher;
2915 const XOR_MASK: u8 = 0x5A;
2916
2917 impl PayloadCipher for XorCipher {
2918 fn seal(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
2919 let tag = blake3::hash(&aad.canonical_bytes());
2920 let mut out = tag.as_bytes()[..8].to_vec();
2921 out.extend(plaintext.iter().map(|b| b ^ XOR_MASK));
2922 Ok(out)
2923 }
2924
2925 fn open(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
2926 if sealed.len() < 8 {
2927 return Err(CipherError::Malformed {
2928 context: "sealed blob shorter than the aad tag",
2929 });
2930 }
2931 let expected = blake3::hash(&aad.canonical_bytes());
2932 if sealed[..8] != expected.as_bytes()[..8] {
2933 return Err(CipherError::Authentication);
2934 }
2935 Ok(sealed[8..].iter().map(|b| b ^ XOR_MASK).collect())
2936 }
2937 }
2938
2939 struct RotatingKeyedCipher {
2947 current_id: u8,
2948 previous_id: Option<u8>,
2949 }
2950
2951 impl PayloadCipher for RotatingKeyedCipher {
2952 fn seal(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
2953 let tag = blake3::hash(&aad.canonical_bytes());
2954 let mut out = vec![self.current_id];
2955 out.extend_from_slice(&tag.as_bytes()[..8]);
2956 out.extend(plaintext.iter().map(|b| b ^ XOR_MASK));
2957 Ok(out)
2958 }
2959
2960 fn open(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
2961 if sealed.len() < 9 {
2962 return Err(CipherError::Malformed {
2963 context: "sealed blob shorter than the key-id + aad tag prefix",
2964 });
2965 }
2966 let id = sealed[0];
2967 if id != self.current_id && Some(id) != self.previous_id {
2968 return Err(CipherError::UnknownKeyId { key_id: id });
2969 }
2970 let expected = blake3::hash(&aad.canonical_bytes());
2971 if sealed[1..9] != expected.as_bytes()[..8] {
2972 return Err(CipherError::Authentication);
2973 }
2974 Ok(sealed[9..].iter().map(|b| b ^ XOR_MASK).collect())
2975 }
2976 }
2977
2978 async fn mem_backend(max_payload_bytes: u64) -> LocalBackend {
2979 let backend = LocalBackend::open(":memory:", max_payload_bytes)
2980 .await
2981 .expect("open in-memory backend");
2982 backend.init().await.expect("apply migrations");
2983 backend
2984 }
2985
2986 fn step_result(exec: ExecutionId, step: u32, payload: &[u8]) -> JournalEntry {
2987 let step_id = StepId::new(step);
2988 JournalEntry {
2989 seq: None,
2990 execution_id: exec,
2991 kind: ExecutionKind::AgentTurn,
2992 step_id,
2993 entry: EntryKind::StepResult {
2994 idempotency_key: IdempotencyKey::derive(exec, step_id, b"tool:read"),
2995 payload: Bytes::copy_from_slice(payload),
2996 effect: EffectClass::Idempotent,
2997 payload_version: 1,
2998 },
2999 created_at_ms: 100,
3000 }
3001 }
3002
3003 fn effect_intent(exec: ExecutionId, step: u32) -> JournalEntry {
3004 let step_id = StepId::new(step);
3005 JournalEntry {
3006 seq: None,
3007 execution_id: exec,
3008 kind: ExecutionKind::AgentTurn,
3009 step_id,
3010 entry: EntryKind::EffectIntent {
3011 idempotency_key: IdempotencyKey::derive(exec, step_id, b"transfer"),
3012 effect: EffectClass::ExactlyOnceGuarded,
3013 hmac: None,
3014 },
3015 created_at_ms: 100,
3016 }
3017 }
3018
3019 #[tokio::test]
3026 async fn count_sealed_under_key_id_counts_matching_journal_rows_and_excludes_control_entries() {
3027 let backend = mem_backend(1_048_576).await;
3028 let exec = ExecutionId::new();
3029 backend
3030 .open_execution(exec, ExecutionKind::AgentTurn)
3031 .await
3032 .unwrap();
3033
3034 backend
3035 .append(step_result(exec, 0, &[5, 0, 0]))
3036 .await
3037 .unwrap();
3038 backend
3039 .append(step_result(exec, 1, &[6, 0, 0]))
3040 .await
3041 .unwrap();
3042 backend.append(effect_intent(exec, 2)).await.unwrap();
3044
3045 assert_eq!(backend.count_sealed_under_key_id(5).await.unwrap(), 1);
3046 assert_eq!(backend.count_sealed_under_key_id(6).await.unwrap(), 1);
3047 assert_eq!(backend.count_sealed_under_key_id(7).await.unwrap(), 0);
3048 }
3049
3050 #[tokio::test]
3054 async fn count_sealed_under_key_id_counts_matching_promise_rows() {
3055 let backend = mem_backend(1_048_576).await;
3056 let exec = ExecutionId::new();
3057 backend
3058 .open_execution(exec, ExecutionKind::AgentTurn)
3059 .await
3060 .unwrap();
3061 let promise_id = PromiseId::new();
3062 backend
3063 .insert_promise(promise_id, exec, [0u8; 32], 100)
3064 .await
3065 .unwrap();
3066 assert_eq!(backend.count_sealed_under_key_id(9).await.unwrap(), 0);
3068
3069 backend
3070 .resolve_promise(promise_id, exec, &[9, 1, 2, 3], 200)
3071 .await
3072 .unwrap();
3073
3074 assert_eq!(backend.count_sealed_under_key_id(9).await.unwrap(), 1);
3075 assert_eq!(backend.count_sealed_under_key_id(10).await.unwrap(), 0);
3076 }
3077
3078 #[tokio::test]
3079 async fn open_execution_is_fresh_then_resume() {
3080 let backend = mem_backend(1_048_576).await;
3081 let exec = ExecutionId::new();
3082 assert!(
3083 !backend
3084 .open_execution(exec, ExecutionKind::AgentTurn)
3085 .await
3086 .unwrap()
3087 );
3088 assert!(
3089 backend
3090 .open_execution(exec, ExecutionKind::AgentTurn)
3091 .await
3092 .unwrap()
3093 );
3094 }
3095
3096 #[tokio::test]
3097 async fn open_execution_exclusive_is_fresh_then_resume() {
3098 let dir = tempfile::tempdir().unwrap();
3101 let db_path = dir.path().join("durable.db");
3102 let backend = LocalBackend::open(&db_path.to_string_lossy(), 1_048_576)
3103 .await
3104 .unwrap();
3105 backend.init().await.unwrap();
3106
3107 let exec = ExecutionId::new();
3108 let (is_resume, lock) = backend
3109 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3110 .await
3111 .unwrap();
3112 assert!(!is_resume);
3113 assert!(lock.is_some(), "a file-backed backend must derive a lock");
3114 drop(lock);
3115
3116 let (is_resume, _lock) = backend
3117 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3118 .await
3119 .unwrap();
3120 assert!(is_resume);
3121 }
3122
3123 #[tokio::test]
3128 async fn open_execution_exclusive_rejects_concurrent_second_holder() {
3129 let dir = tempfile::tempdir().unwrap();
3130 let db_path = dir.path().join("durable.db");
3131 let url = db_path.to_string_lossy().into_owned();
3132
3133 let backend_a = LocalBackend::open(&url, 1_048_576).await.unwrap();
3134 backend_a.init().await.unwrap();
3135 let backend_b = LocalBackend::open(&url, 1_048_576).await.unwrap();
3136
3137 let exec = ExecutionId::new();
3138 let (_, _lock_a) = backend_a
3139 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3140 .await
3141 .unwrap();
3142
3143 let err = backend_b
3144 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3145 .await
3146 .expect_err("a second concurrent holder must be rejected");
3147 assert!(
3148 matches!(err, DurableError::ExecutionLocked { execution_id, .. } if execution_id == exec),
3149 "expected ExecutionLocked, got {err:?}"
3150 );
3151 }
3152
3153 #[tokio::test]
3154 async fn open_execution_exclusive_on_memory_backend_returns_no_lock() {
3155 let backend = mem_backend(1_048_576).await;
3158 let exec = ExecutionId::new();
3159 let (is_resume, lock) = backend
3160 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3161 .await
3162 .unwrap();
3163 assert!(!is_resume);
3164 assert!(lock.is_none());
3165 }
3166
3167 #[tokio::test]
3168 async fn list_executions_summarizes_and_filters() {
3169 let backend = mem_backend(1_048_576).await;
3170 let turn = ExecutionId::new();
3171 let dag = ExecutionId::new();
3172 backend
3173 .open_execution(turn, ExecutionKind::AgentTurn)
3174 .await
3175 .unwrap();
3176 backend
3177 .open_execution(dag, ExecutionKind::DagRun)
3178 .await
3179 .unwrap();
3180 backend.append(step_result(turn, 0, b"a")).await.unwrap();
3181 backend.append(step_result(turn, 1, b"b")).await.unwrap();
3182 backend.append(step_result(dag, 0, b"c")).await.unwrap();
3183 backend
3184 .finalize(turn, ExecutionStatus::Completed)
3185 .await
3186 .unwrap();
3187
3188 let all = backend.list_executions(None, None, 10).await.unwrap();
3190 assert_eq!(all.len(), 2);
3191
3192 let turn_row = all
3193 .iter()
3194 .find(|e| e.execution_id == turn)
3195 .expect("turn present");
3196 assert_eq!(turn_row.kind, "agent_turn");
3197 assert_eq!(turn_row.status, ExecutionStatus::Completed);
3198 assert_eq!(turn_row.step_count, 2);
3199 assert!(turn_row.finalized_at_ms.is_some());
3200
3201 let dag_row = all
3202 .iter()
3203 .find(|e| e.execution_id == dag)
3204 .expect("dag present");
3205 assert_eq!(dag_row.status, ExecutionStatus::Running);
3206 assert_eq!(dag_row.step_count, 1);
3207 assert!(dag_row.finalized_at_ms.is_none());
3208
3209 let running = backend
3211 .list_executions(Some("running"), None, 10)
3212 .await
3213 .unwrap();
3214 assert_eq!(running.len(), 1);
3215 assert_eq!(running[0].execution_id, dag);
3216
3217 let dags = backend
3219 .list_executions(None, Some("dag_run"), 10)
3220 .await
3221 .unwrap();
3222 assert_eq!(dags.len(), 1);
3223 assert_eq!(dags[0].execution_id, dag);
3224
3225 let one = backend.list_executions(None, None, 1).await.unwrap();
3227 assert_eq!(one.len(), 1);
3228 }
3229
3230 #[tokio::test]
3231 async fn append_and_read_round_trips_step_result() {
3232 let backend = mem_backend(1_048_576).await;
3233 let exec = ExecutionId::new();
3234 backend
3235 .open_execution(exec, ExecutionKind::AgentTurn)
3236 .await
3237 .unwrap();
3238
3239 let seq = backend
3240 .append(step_result(exec, 0, b"hello"))
3241 .await
3242 .unwrap();
3243 assert_eq!(seq.value(), 1, "first append takes seq 1");
3244
3245 let entries = backend.read_execution(exec).await.unwrap();
3246 assert_eq!(entries.len(), 1);
3247 match &entries[0].entry {
3248 EntryKind::StepResult {
3249 payload, effect, ..
3250 } => {
3251 assert_eq!(payload.as_ref(), b"hello");
3252 assert_eq!(*effect, EffectClass::Idempotent);
3253 }
3254 other => panic!("unexpected entry kind: {other:?}"),
3255 }
3256 assert_eq!(entries[0].seq, Some(seq));
3257 }
3258
3259 #[tokio::test]
3260 async fn cipher_seals_payload_at_rest_but_round_trips() {
3261 let backend = mem_backend(1_048_576)
3262 .await
3263 .with_cipher(Arc::new(XorCipher));
3264 let exec = ExecutionId::new();
3265 backend
3266 .open_execution(exec, ExecutionKind::AgentTurn)
3267 .await
3268 .unwrap();
3269 backend
3270 .append(step_result(exec, 0, b"secret-payload"))
3271 .await
3272 .unwrap();
3273
3274 let (stored,): (Option<Vec<u8>>,) = zeph_db::query_as(sql!(
3276 "SELECT payload FROM durable_journal WHERE execution_id = ?"
3277 ))
3278 .bind(exec.as_uuid().to_string())
3279 .fetch_one(backend.pool())
3280 .await
3281 .unwrap();
3282 let stored = stored.expect("payload present");
3283 assert_ne!(
3284 stored.as_slice(),
3285 b"secret-payload",
3286 "payload must be sealed at rest"
3287 );
3288
3289 let entries = backend.read_execution(exec).await.unwrap();
3291 match &entries[0].entry {
3292 EntryKind::StepResult { payload, .. } => {
3293 assert_eq!(payload.as_ref(), b"secret-payload");
3294 }
3295 other => panic!("unexpected entry kind: {other:?}"),
3296 }
3297 }
3298
3299 #[tokio::test]
3300 async fn control_entry_hmac_is_stamped_only_when_keyed() {
3301 let exec = ExecutionId::new();
3302
3303 let unkeyed = mem_backend(1_048_576).await;
3304 unkeyed
3305 .open_execution(exec, ExecutionKind::AgentTurn)
3306 .await
3307 .unwrap();
3308 unkeyed.append(effect_intent(exec, 0)).await.unwrap();
3309 match &unkeyed.read_execution(exec).await.unwrap()[0].entry {
3310 EntryKind::EffectIntent { hmac, .. } => assert!(hmac.is_none()),
3311 other => panic!("unexpected entry kind: {other:?}"),
3312 }
3313
3314 let keyed = mem_backend(1_048_576).await.with_hmac_key([7u8; 32]);
3315 let exec2 = ExecutionId::new();
3316 keyed
3317 .open_execution(exec2, ExecutionKind::AgentTurn)
3318 .await
3319 .unwrap();
3320 keyed.append(effect_intent(exec2, 0)).await.unwrap();
3321 match &keyed.read_execution(exec2).await.unwrap()[0].entry {
3322 EntryKind::EffectIntent { hmac, .. } => {
3323 assert!(
3324 hmac.is_some(),
3325 "keyed backend stamps a row HMAC over control entries"
3326 );
3327 }
3328 other => panic!("unexpected entry kind: {other:?}"),
3329 }
3330 }
3331
3332 #[tokio::test]
3339 async fn read_execution_rejects_control_hmac_under_wrong_key() {
3340 let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
3341 let exec = ExecutionId::new();
3342 writer
3343 .open_execution(exec, ExecutionKind::AgentTurn)
3344 .await
3345 .unwrap();
3346 writer.append(effect_intent(exec, 0)).await.unwrap();
3347
3348 let wrong_key_reader =
3349 LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([2u8; 32]);
3350 assert_matches!(
3351 wrong_key_reader.read_execution(exec).await,
3352 Err(DurableError::ControlIntegrity)
3353 );
3354
3355 let right_key_reader =
3357 LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([1u8; 32]);
3358 assert!(right_key_reader.read_execution(exec).await.is_ok());
3359 }
3360
3361 #[tokio::test]
3366 async fn read_execution_rejects_missing_hmac_on_keyed_backend() {
3367 let writer = mem_backend(1_048_576).await;
3368 let exec = ExecutionId::new();
3369 writer
3370 .open_execution(exec, ExecutionKind::AgentTurn)
3371 .await
3372 .unwrap();
3373 writer.append(effect_intent(exec, 0)).await.unwrap();
3374
3375 let keyed_reader =
3376 LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([3u8; 32]);
3377 assert_matches!(
3378 keyed_reader.read_execution(exec).await,
3379 Err(DurableError::ControlIntegrity)
3380 );
3381 }
3382
3383 #[tokio::test]
3392 async fn read_execution_rejects_stamped_hmac_on_unkeyed_backend() {
3393 let writer = mem_backend(1_048_576).await.with_hmac_key([4u8; 32]);
3394 let exec = ExecutionId::new();
3395 writer
3396 .open_execution(exec, ExecutionKind::AgentTurn)
3397 .await
3398 .unwrap();
3399 writer.append(effect_intent(exec, 0)).await.unwrap();
3400
3401 let unkeyed_reader = LocalBackend::new(writer.pool().clone(), 1_048_576);
3402 assert_matches!(
3403 unkeyed_reader.read_execution(exec).await,
3404 Err(DurableError::ControlIntegrity)
3405 );
3406 }
3407
3408 #[tokio::test]
3415 async fn verify_control_hmac_accepts_row_under_previous_key_during_window() {
3416 let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
3417 let exec = ExecutionId::new();
3418 writer
3419 .open_execution(exec, ExecutionKind::AgentTurn)
3420 .await
3421 .unwrap();
3422 writer.append(effect_intent(exec, 0)).await.unwrap();
3423
3424 let post_rotation_reader = LocalBackend::new(writer.pool().clone(), 1_048_576)
3427 .with_hmac_key([2u8; 32])
3428 .with_previous_hmac_key([1u8; 32]);
3429 assert!(
3430 post_rotation_reader.read_execution(exec).await.is_ok(),
3431 "a row stamped under the previous key must verify during the rotation window"
3432 );
3433
3434 let post_rotation_writer = LocalBackend::new(writer.pool().clone(), 1_048_576)
3437 .with_hmac_key([2u8; 32])
3438 .with_previous_hmac_key([1u8; 32]);
3439 post_rotation_writer
3440 .append(effect_intent(exec, 1))
3441 .await
3442 .unwrap();
3443 assert!(post_rotation_writer.read_execution(exec).await.is_ok());
3444 }
3445
3446 #[tokio::test]
3450 async fn verify_control_hmac_rejects_row_under_neither_current_nor_previous_key() {
3451 let writer = mem_backend(1_048_576).await.with_hmac_key([9u8; 32]);
3452 let exec = ExecutionId::new();
3453 writer
3454 .open_execution(exec, ExecutionKind::AgentTurn)
3455 .await
3456 .unwrap();
3457 writer.append(effect_intent(exec, 0)).await.unwrap();
3458
3459 let unrelated_reader = LocalBackend::new(writer.pool().clone(), 1_048_576)
3460 .with_hmac_key([2u8; 32])
3461 .with_previous_hmac_key([3u8; 32]);
3462 assert_matches!(
3463 unrelated_reader.read_execution(exec).await,
3464 Err(DurableError::ControlIntegrity)
3465 );
3466 }
3467
3468 #[tokio::test]
3473 async fn count_control_entries_under_previous_hmac_counts_previous_only_rows() {
3474 let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
3475 let exec = ExecutionId::new();
3476 writer
3477 .open_execution(exec, ExecutionKind::AgentTurn)
3478 .await
3479 .unwrap();
3480 writer.append(effect_intent(exec, 0)).await.unwrap();
3482
3483 let scanner_mid_window = LocalBackend::new(writer.pool().clone(), 1_048_576)
3484 .with_hmac_key([2u8; 32])
3485 .with_previous_hmac_key([1u8; 32]);
3486 assert_eq!(
3487 scanner_mid_window
3488 .count_control_entries_under_previous_hmac()
3489 .await
3490 .unwrap(),
3491 1,
3492 "a row stamped under the previous key only must be counted"
3493 );
3494
3495 let post_rotation_writer = LocalBackend::new(writer.pool().clone(), 1_048_576)
3497 .with_hmac_key([2u8; 32])
3498 .with_previous_hmac_key([1u8; 32]);
3499 post_rotation_writer
3500 .append(effect_intent(exec, 1))
3501 .await
3502 .unwrap();
3503 assert_eq!(
3504 post_rotation_writer
3505 .count_control_entries_under_previous_hmac()
3506 .await
3507 .unwrap(),
3508 1,
3509 "the post-rotation row (verifies under current) must not add to the count"
3510 );
3511 }
3512
3513 #[tokio::test]
3517 async fn count_control_entries_under_previous_hmac_is_zero_on_empty_journal() {
3518 let backend = mem_backend(1_048_576).await;
3519 assert_eq!(
3520 backend
3521 .count_control_entries_under_previous_hmac()
3522 .await
3523 .unwrap(),
3524 0
3525 );
3526 }
3527
3528 #[tokio::test]
3533 async fn count_control_entries_under_previous_hmac_errors_when_keys_missing() {
3534 let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
3535 let exec = ExecutionId::new();
3536 writer
3537 .open_execution(exec, ExecutionKind::AgentTurn)
3538 .await
3539 .unwrap();
3540 writer.append(effect_intent(exec, 0)).await.unwrap();
3541
3542 let unkeyed_scanner = LocalBackend::new(writer.pool().clone(), 1_048_576);
3543 assert_matches!(
3544 unkeyed_scanner
3545 .count_control_entries_under_previous_hmac()
3546 .await,
3547 Err(DurableError::ControlIntegrity)
3548 );
3549
3550 let current_only_scanner =
3551 LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([1u8; 32]);
3552 assert_matches!(
3553 current_only_scanner
3554 .count_control_entries_under_previous_hmac()
3555 .await,
3556 Err(DurableError::ControlIntegrity)
3557 );
3558 }
3559
3560 #[tokio::test]
3561 async fn promise_and_timer_entries_fail_closed() {
3562 let backend = mem_backend(1_048_576).await;
3563 let exec = ExecutionId::new();
3564 backend
3565 .open_execution(exec, ExecutionKind::AgentTurn)
3566 .await
3567 .unwrap();
3568 let timer = JournalEntry {
3569 seq: None,
3570 execution_id: exec,
3571 kind: ExecutionKind::AgentTurn,
3572 step_id: StepId::new(0),
3573 entry: EntryKind::TimerArmed {
3574 timer_id: crate::TimerId::new(),
3575 due_at_ms: 1_000,
3576 hmac: None,
3577 },
3578 created_at_ms: 0,
3579 };
3580 assert_matches!(
3581 backend.append(timer).await,
3582 Err(DurableError::UnsupportedEntryKind {
3583 kind: "timer_armed"
3584 })
3585 );
3586 }
3587
3588 #[tokio::test]
3589 async fn payload_over_limit_is_rejected_fail_closed() {
3590 let backend = mem_backend(8).await;
3591 let exec = ExecutionId::new();
3592 backend
3593 .open_execution(exec, ExecutionKind::AgentTurn)
3594 .await
3595 .unwrap();
3596 let big = vec![0u8; 64];
3597 assert_matches!(
3598 backend.append(step_result(exec, 0, &big)).await,
3599 Err(DurableError::PayloadTooLarge { .. })
3600 );
3601 }
3602
3603 #[tokio::test]
3604 async fn finalize_marks_terminal_status_and_time() {
3605 let backend = mem_backend(1_048_576).await;
3606 let exec = ExecutionId::new();
3607 backend
3608 .open_execution(exec, ExecutionKind::AgentTurn)
3609 .await
3610 .unwrap();
3611 backend
3612 .finalize(exec, ExecutionStatus::Completed)
3613 .await
3614 .unwrap();
3615
3616 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3617 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3618 ))
3619 .bind(exec.as_uuid().to_string())
3620 .fetch_one(backend.pool())
3621 .await
3622 .unwrap();
3623 assert_eq!(status, "completed");
3624 assert!(finalized.is_some(), "a terminal status stamps finalized_at");
3625 }
3626
3627 #[tokio::test]
3628 async fn finalize_is_a_noop_once_already_terminal() {
3629 let backend = mem_backend(1_048_576).await;
3632 let exec = ExecutionId::new();
3633 backend
3634 .open_execution(exec, ExecutionKind::AgentTurn)
3635 .await
3636 .unwrap();
3637 backend
3638 .finalize(exec, ExecutionStatus::Completed)
3639 .await
3640 .unwrap();
3641
3642 backend
3644 .finalize(exec, ExecutionStatus::Failed)
3645 .await
3646 .unwrap();
3647
3648 let (status,): (String,) = zeph_db::query_as(sql!(
3649 "SELECT status FROM durable_executions WHERE execution_id = ?"
3650 ))
3651 .bind(exec.as_uuid().to_string())
3652 .fetch_one(backend.pool())
3653 .await
3654 .unwrap();
3655 assert_eq!(
3656 status, "completed",
3657 "the first terminal status must stick; a later finalize call is a no-op"
3658 );
3659 }
3660
3661 #[tokio::test]
3662 async fn finalize_after_abort_is_a_noop() {
3663 let backend = mem_backend(1_048_576).await;
3667 let exec = ExecutionId::new();
3668 backend
3669 .open_execution(exec, ExecutionKind::AgentTurn)
3670 .await
3671 .unwrap();
3672 backend
3673 .finalize(exec, ExecutionStatus::Aborted)
3674 .await
3675 .unwrap();
3676
3677 backend
3678 .finalize(exec, ExecutionStatus::Completed)
3679 .await
3680 .unwrap();
3681
3682 let (status,): (String,) = zeph_db::query_as(sql!(
3683 "SELECT status FROM durable_executions WHERE execution_id = ?"
3684 ))
3685 .bind(exec.as_uuid().to_string())
3686 .fetch_one(backend.pool())
3687 .await
3688 .unwrap();
3689 assert_eq!(
3690 status, "aborted",
3691 "an aborted execution must not be overwritten by a later Completed/Failed call"
3692 );
3693 }
3694
3695 #[tokio::test]
3696 async fn reopening_a_finalized_execution_resets_it_to_running() {
3697 let backend = mem_backend(1_048_576).await;
3701 let exec = ExecutionId::new();
3702 backend
3703 .open_execution(exec, ExecutionKind::AgentTurn)
3704 .await
3705 .unwrap();
3706 backend
3707 .finalize(exec, ExecutionStatus::Completed)
3708 .await
3709 .unwrap();
3710
3711 let is_resume = backend
3712 .open_execution(exec, ExecutionKind::AgentTurn)
3713 .await
3714 .unwrap();
3715 assert!(is_resume, "the row already existed, so this is a resume");
3716
3717 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3718 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3719 ))
3720 .bind(exec.as_uuid().to_string())
3721 .fetch_one(backend.pool())
3722 .await
3723 .unwrap();
3724 assert_eq!(
3725 status, "running",
3726 "reopening a completed execution must un-finalize it"
3727 );
3728 assert!(
3729 finalized.is_none(),
3730 "reopening must clear the stale finalized_at"
3731 );
3732 }
3733
3734 #[tokio::test]
3735 async fn reopening_a_failed_execution_resets_it_to_running() {
3736 let backend = mem_backend(1_048_576).await;
3740 let exec = ExecutionId::new();
3741 backend
3742 .open_execution(exec, ExecutionKind::AgentTurn)
3743 .await
3744 .unwrap();
3745 backend
3746 .finalize(exec, ExecutionStatus::Failed)
3747 .await
3748 .unwrap();
3749
3750 let is_resume = backend
3751 .open_execution(exec, ExecutionKind::AgentTurn)
3752 .await
3753 .unwrap();
3754 assert!(is_resume, "the row already existed, so this is a resume");
3755
3756 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3757 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3758 ))
3759 .bind(exec.as_uuid().to_string())
3760 .fetch_one(backend.pool())
3761 .await
3762 .unwrap();
3763 assert_eq!(
3764 status, "running",
3765 "reopening a failed execution must un-finalize it"
3766 );
3767 assert!(
3768 finalized.is_none(),
3769 "reopening must clear the stale finalized_at"
3770 );
3771 }
3772
3773 #[tokio::test]
3774 async fn reopening_an_aborted_execution_un_finalizes_it() {
3775 let backend = mem_backend(1_048_576).await;
3782 let exec = ExecutionId::new();
3783 backend
3784 .open_execution(exec, ExecutionKind::AgentTurn)
3785 .await
3786 .unwrap();
3787 backend
3788 .finalize(exec, ExecutionStatus::Aborted)
3789 .await
3790 .unwrap();
3791
3792 let is_resume = backend
3793 .open_execution(exec, ExecutionKind::AgentTurn)
3794 .await
3795 .unwrap();
3796 assert!(is_resume, "the row already existed, so this is a resume");
3797
3798 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3799 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3800 ))
3801 .bind(exec.as_uuid().to_string())
3802 .fetch_one(backend.pool())
3803 .await
3804 .unwrap();
3805 assert_eq!(
3806 status, "running",
3807 "reopening an aborted execution must un-finalize it (INV-16)"
3808 );
3809 assert!(
3810 finalized.is_none(),
3811 "reopening must clear the stale finalized_at"
3812 );
3813 }
3814
3815 #[tokio::test]
3816 async fn cancel_execution_with_no_live_owner_cancels_immediately() {
3817 let backend = mem_backend(1_048_576).await;
3820 let exec = ExecutionId::new();
3821 backend
3822 .open_execution(exec, ExecutionKind::AgentTurn)
3823 .await
3824 .unwrap();
3825
3826 let outcome = backend.cancel_execution(exec).await.unwrap();
3827 assert_eq!(outcome, CancelOutcome::Canceled);
3828
3829 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3830 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3831 ))
3832 .bind(exec.as_uuid().to_string())
3833 .fetch_one(backend.pool())
3834 .await
3835 .unwrap();
3836 assert_eq!(status, "canceled");
3837 assert!(finalized.is_some(), "a terminal status stamps finalized_at");
3838 }
3839
3840 #[tokio::test]
3841 async fn cancel_execution_with_no_live_owner_on_file_backed_pool_cancels_immediately() {
3842 let dir = tempfile::tempdir().unwrap();
3845 let backend =
3846 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
3847 .await
3848 .unwrap();
3849 backend.init().await.unwrap();
3850
3851 let exec = ExecutionId::new();
3852 backend
3853 .open_execution(exec, ExecutionKind::AgentTurn)
3854 .await
3855 .unwrap();
3856
3857 let outcome = backend.cancel_execution(exec).await.unwrap();
3858 assert_eq!(outcome, CancelOutcome::Canceled);
3859
3860 let lock_dir = backend.lock_dir.clone().unwrap();
3862 assert!(ExecutionLock::acquire(&lock_dir, exec).is_ok());
3863 }
3864
3865 #[tokio::test]
3866 async fn cancel_execution_refuses_a_live_owner_without_touching_the_row() {
3867 let dir = tempfile::tempdir().unwrap();
3870 let db_path = dir.path().join("durable.db");
3871 let url = db_path.to_string_lossy().into_owned();
3872
3873 let owner = LocalBackend::open(&url, 1_048_576).await.unwrap();
3874 owner.init().await.unwrap();
3875 let canceler = LocalBackend::open(&url, 1_048_576).await.unwrap();
3876
3877 let exec = ExecutionId::new();
3878 let (_, _lock) = owner
3879 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3880 .await
3881 .unwrap();
3882
3883 let outcome = canceler.cancel_execution(exec).await.unwrap();
3884 assert!(
3885 matches!(outcome, CancelOutcome::LiveOwner { pid } if pid == std::process::id()),
3886 "expected LiveOwner{{pid: {}}}, got {outcome:?}",
3887 std::process::id()
3888 );
3889
3890 let (status,): (String,) = zeph_db::query_as(sql!(
3891 "SELECT status FROM durable_executions WHERE execution_id = ?"
3892 ))
3893 .bind(exec.as_uuid().to_string())
3894 .fetch_one(owner.pool())
3895 .await
3896 .unwrap();
3897 assert_eq!(status, "running", "a live-owned row must never be touched");
3898 }
3899
3900 #[tokio::test]
3901 async fn cancel_execution_is_idempotent_on_a_second_call() {
3902 let backend = mem_backend(1_048_576).await;
3904 let exec = ExecutionId::new();
3905 backend
3906 .open_execution(exec, ExecutionKind::AgentTurn)
3907 .await
3908 .unwrap();
3909
3910 assert_eq!(
3911 backend.cancel_execution(exec).await.unwrap(),
3912 CancelOutcome::Canceled
3913 );
3914 let second = backend.cancel_execution(exec).await.unwrap();
3915 assert_eq!(
3916 second,
3917 CancelOutcome::AlreadyTerminal {
3918 status: ExecutionStatus::Canceled
3919 }
3920 );
3921 }
3922
3923 #[tokio::test]
3924 async fn cancel_execution_on_each_other_terminal_status_is_already_terminal() {
3925 for status in [
3926 ExecutionStatus::Completed,
3927 ExecutionStatus::Failed,
3928 ExecutionStatus::Aborted,
3929 ] {
3930 let backend = mem_backend(1_048_576).await;
3931 let exec = ExecutionId::new();
3932 backend
3933 .open_execution(exec, ExecutionKind::AgentTurn)
3934 .await
3935 .unwrap();
3936 backend.finalize(exec, status).await.unwrap();
3937
3938 let outcome = backend.cancel_execution(exec).await.unwrap();
3939 assert_eq!(
3940 outcome,
3941 CancelOutcome::AlreadyTerminal { status },
3942 "canceling a {status:?} execution must be a no-op reporting its own status"
3943 );
3944 }
3945 }
3946
3947 #[tokio::test]
3948 async fn cancel_execution_on_unknown_id_returns_not_found() {
3949 let backend = mem_backend(1_048_576).await;
3950 let outcome = backend.cancel_execution(ExecutionId::new()).await.unwrap();
3951 assert_eq!(outcome, CancelOutcome::NotFound);
3952 }
3953
3954 #[tokio::test]
3955 async fn cancel_execution_races_finalize_exactly_one_terminal_status_wins() {
3956 let dir = tempfile::tempdir().unwrap();
3965 let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
3966 let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
3967 backend.init().await.unwrap();
3968
3969 for _ in 0..20 {
3970 let exec = ExecutionId::new();
3971 backend
3972 .open_execution(exec, ExecutionKind::AgentTurn)
3973 .await
3974 .unwrap();
3975
3976 let cancel_backend = backend.clone();
3977 let cancel = tokio::spawn(async move { cancel_backend.cancel_execution(exec).await });
3978 let finalize_backend = backend.clone();
3979 let finalize = tokio::spawn(async move {
3980 finalize_backend
3981 .finalize(exec, ExecutionStatus::Completed)
3982 .await
3983 });
3984
3985 let (cancel_result, finalize_result) = tokio::join!(cancel, finalize);
3986 let cancel_outcome = cancel_result
3987 .expect("cancel task must not panic")
3988 .expect("cancel_execution must not error under a concurrent finalize");
3989 finalize_result
3990 .expect("finalize task must not panic")
3991 .expect("finalize must not error under a concurrent cancel");
3992
3993 let (status,): (String,) = zeph_db::query_as(sql!(
3994 "SELECT status FROM durable_executions WHERE execution_id = ?"
3995 ))
3996 .bind(exec.as_uuid().to_string())
3997 .fetch_one(backend.pool())
3998 .await
3999 .unwrap();
4000
4001 match cancel_outcome {
4005 CancelOutcome::Canceled => assert_eq!(
4006 status, "canceled",
4007 "cancel_execution won the race — the row must be canceled"
4008 ),
4009 CancelOutcome::AlreadyTerminal {
4010 status: ExecutionStatus::Completed,
4011 } => assert_eq!(
4012 status, "completed",
4013 "finalize won the race — the row must be completed, and cancel's own \
4014 guarded UPDATE must have found it already non-running"
4015 ),
4016 other => panic!(
4017 "cancel_execution must only ever win or lose cleanly against a concurrent \
4018 finalize, got {other:?}"
4019 ),
4020 }
4021 }
4022 }
4023
4024 #[tokio::test]
4025 async fn cancel_execution_races_sweep_orphans_exactly_one_of_canceled_or_aborted_wins() {
4026 let dir = tempfile::tempdir().unwrap();
4035 let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
4036 let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
4037 backend.init().await.unwrap();
4038
4039 let policy = RetentionPolicy {
4040 stale_running_after_secs: 1,
4041 prune_batch_size: 10,
4042 ..RetentionPolicy::default()
4043 };
4044
4045 for _ in 0..20 {
4046 let exec = ExecutionId::new();
4047 backend
4048 .open_execution(exec, ExecutionKind::AgentTurn)
4049 .await
4050 .unwrap();
4051 backdate_updated_at(&backend, exec, 0).await;
4052
4053 let cancel_backend = backend.clone();
4054 let cancel = tokio::spawn(async move { cancel_backend.cancel_execution(exec).await });
4055 let sweep_backend = backend.clone();
4056 let policy_for_task = policy.clone();
4057 let sweep =
4058 tokio::spawn(async move { sweep_backend.sweep_orphans(&policy_for_task).await });
4059
4060 let (cancel_result, sweep_result) = tokio::join!(cancel, sweep);
4061 let cancel_outcome = cancel_result
4062 .expect("cancel task must not panic")
4063 .expect("cancel_execution must not error under a concurrent sweep");
4064 let aborted = sweep_result
4065 .expect("sweep task must not panic")
4066 .expect("sweep_orphans must not error under a concurrent cancel");
4067
4068 let (status,): (String,) = zeph_db::query_as(sql!(
4069 "SELECT status FROM durable_executions WHERE execution_id = ?"
4070 ))
4071 .bind(exec.as_uuid().to_string())
4072 .fetch_one(backend.pool())
4073 .await
4074 .unwrap();
4075
4076 match cancel_outcome {
4077 CancelOutcome::Canceled => {
4078 assert_eq!(aborted, 0, "cancel won the lock — sweep must skip this row");
4079 assert_eq!(status, "canceled");
4080 }
4081 CancelOutcome::LiveOwner { .. } => {
4082 assert_eq!(aborted, 1, "sweep won the lock — it must abort this row");
4083 assert_eq!(status, "aborted");
4084 }
4085 other => panic!(
4086 "cancel_execution must only ever win the lock (Canceled) or lose it \
4087 (LiveOwner) against a concurrent sweep, got {other:?}"
4088 ),
4089 }
4090 }
4091 }
4092
4093 #[tokio::test]
4094 async fn open_execution_on_canceled_row_fails_closed_and_never_resumes() {
4095 let backend = mem_backend(1_048_576).await;
4099 let exec = ExecutionId::new();
4100 backend
4101 .open_execution(exec, ExecutionKind::AgentTurn)
4102 .await
4103 .unwrap();
4104 let outcome = backend.cancel_execution(exec).await.unwrap();
4105 assert_eq!(outcome, CancelOutcome::Canceled);
4106
4107 let err = backend
4108 .open_execution(exec, ExecutionKind::AgentTurn)
4109 .await
4110 .expect_err("reopening a canceled execution must fail closed");
4111 assert!(
4112 matches!(err, DurableError::ExecutionCanceled { execution_id } if execution_id == exec),
4113 "expected ExecutionCanceled, got {err:?}"
4114 );
4115
4116 let (status,): (String,) = zeph_db::query_as(sql!(
4117 "SELECT status FROM durable_executions WHERE execution_id = ?"
4118 ))
4119 .bind(exec.as_uuid().to_string())
4120 .fetch_one(backend.pool())
4121 .await
4122 .unwrap();
4123 assert_eq!(
4124 status, "canceled",
4125 "the row must never be reset to running by a reopen attempt"
4126 );
4127 }
4128
4129 #[tokio::test]
4130 async fn open_execution_exclusive_on_canceled_row_fails_closed_with_lock_released() {
4131 let dir = tempfile::tempdir().unwrap();
4134 let db_path = dir.path().join("durable.db");
4135 let backend = LocalBackend::open(&db_path.to_string_lossy(), 1_048_576)
4136 .await
4137 .unwrap();
4138 backend.init().await.unwrap();
4139
4140 let exec = ExecutionId::new();
4141 backend
4142 .open_execution(exec, ExecutionKind::AgentTurn)
4143 .await
4144 .unwrap();
4145 assert_eq!(
4146 backend.cancel_execution(exec).await.unwrap(),
4147 CancelOutcome::Canceled
4148 );
4149
4150 let err = backend
4151 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
4152 .await
4153 .expect_err("reopening a canceled execution exclusively must fail closed");
4154 assert!(matches!(err, DurableError::ExecutionCanceled { .. }));
4155
4156 let dir2 = backend.lock_dir.clone().unwrap();
4158 assert!(ExecutionLock::acquire(&dir2, exec).is_ok());
4159 }
4160
4161 #[tokio::test]
4162 async fn reopen_of_a_row_deleted_out_from_under_it_starts_fresh() {
4163 let backend = mem_backend(1_048_576).await;
4169 let exec = ExecutionId::new();
4170 backend
4171 .open_execution(exec, ExecutionKind::AgentTurn)
4172 .await
4173 .unwrap();
4174 backend
4175 .finalize(exec, ExecutionStatus::Completed)
4176 .await
4177 .unwrap();
4178
4179 zeph_db::query(sql!(
4181 "DELETE FROM durable_executions WHERE execution_id = ?"
4182 ))
4183 .bind(exec.as_uuid().to_string())
4184 .execute(backend.pool())
4185 .await
4186 .unwrap();
4187
4188 let is_resume = backend
4189 .open_execution(exec, ExecutionKind::AgentTurn)
4190 .await
4191 .unwrap();
4192 assert!(
4193 !is_resume,
4194 "a row deleted by a concurrent prune must be reported as a fresh execution, not a resume"
4195 );
4196
4197 let (status,): (String,) = zeph_db::query_as(sql!(
4198 "SELECT status FROM durable_executions WHERE execution_id = ?"
4199 ))
4200 .bind(exec.as_uuid().to_string())
4201 .fetch_one(backend.pool())
4202 .await
4203 .unwrap();
4204 assert_eq!(status, "running", "the fresh row starts running");
4205 }
4206
4207 #[tokio::test]
4208 async fn prune_does_not_delete_a_row_reopened_since_it_was_finalized() {
4209 let backend = mem_backend(1_048_576).await;
4213 let exec = ExecutionId::new();
4214 backend
4215 .open_execution(exec, ExecutionKind::AgentTurn)
4216 .await
4217 .unwrap();
4218 backend.append(step_result(exec, 0, b"x")).await.unwrap();
4219 zeph_db::query(sql!(
4220 "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
4221 ))
4222 .bind(exec.as_uuid().to_string())
4223 .execute(backend.pool())
4224 .await
4225 .unwrap();
4226
4227 let is_resume = backend
4229 .open_execution(exec, ExecutionKind::AgentTurn)
4230 .await
4231 .unwrap();
4232 assert!(is_resume);
4233
4234 let policy = RetentionPolicy {
4235 ttl_completed_secs: 1,
4236 prune_batch_size: 10,
4237 ..RetentionPolicy::default()
4238 };
4239 let deleted = backend.prune(&policy).await.unwrap();
4240 assert_eq!(
4241 deleted, 0,
4242 "a reopened (un-finalized) execution must not be pruned"
4243 );
4244 assert_eq!(
4245 backend.read_execution(exec).await.unwrap().len(),
4246 1,
4247 "the execution's journal must survive"
4248 );
4249 }
4250
4251 #[tokio::test]
4252 async fn concurrent_prune_and_reopen_never_lose_or_corrupt_the_row() {
4253 let dir = tempfile::tempdir().unwrap();
4268 let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
4269 let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
4270 backend.init().await.unwrap();
4271
4272 let policy = RetentionPolicy {
4273 ttl_completed_secs: 1,
4274 prune_batch_size: 10,
4275 ..RetentionPolicy::default()
4276 };
4277
4278 for _ in 0..20 {
4279 let exec = ExecutionId::new();
4280 backend
4281 .open_execution(exec, ExecutionKind::AgentTurn)
4282 .await
4283 .unwrap();
4284 backend.append(step_result(exec, 0, b"x")).await.unwrap();
4285 zeph_db::query(sql!(
4287 "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
4288 ))
4289 .bind(exec.as_uuid().to_string())
4290 .execute(backend.pool())
4291 .await
4292 .unwrap();
4293
4294 let reopen_backend = backend.clone();
4295 let reopen = tokio::spawn(async move {
4296 reopen_backend
4297 .open_execution(exec, ExecutionKind::AgentTurn)
4298 .await
4299 });
4300 let prune_backend = backend.clone();
4301 let policy_for_task = policy.clone();
4302 let prune = tokio::spawn(async move { prune_backend.prune(&policy_for_task).await });
4303
4304 let (reopen_result, prune_result) = tokio::join!(reopen, prune);
4305 reopen_result
4306 .expect("reopen task must not panic")
4307 .expect("reopen must not error under concurrent prune");
4308 prune_result
4309 .expect("prune task must not panic")
4310 .expect("prune must not error under a concurrent reopen");
4311
4312 let (status,): (String,) = zeph_db::query_as(sql!(
4313 "SELECT status FROM durable_executions WHERE execution_id = ?"
4314 ))
4315 .bind(exec.as_uuid().to_string())
4316 .fetch_one(backend.pool())
4317 .await
4318 .expect(
4319 "the row must exist under either race outcome — reopened-running, or \
4320 deleted-then-reinserted-fresh-running by reopen's fallback",
4321 );
4322 assert_eq!(
4323 status, "running",
4324 "whichever task wins, the row must end up running — never left completed \
4325 (orphaned from a live journal) or absent"
4326 );
4327 }
4328 }
4329
4330 #[tokio::test]
4331 async fn max_seq_reflects_committed_appends() {
4332 let backend = mem_backend(1_048_576).await;
4333 assert_eq!(
4334 backend.max_seq().await.unwrap(),
4335 None,
4336 "empty journal has no max seq"
4337 );
4338
4339 let exec = ExecutionId::new();
4340 backend
4341 .open_execution(exec, ExecutionKind::AgentTurn)
4342 .await
4343 .unwrap();
4344 for step in 0..3 {
4345 backend.append(step_result(exec, step, b"x")).await.unwrap();
4346 }
4347 assert_eq!(backend.max_seq().await.unwrap(), Some(JournalSeq::new(3)));
4348 }
4349
4350 #[tokio::test]
4351 async fn append_batch_group_commits_every_entry() {
4352 let backend = mem_backend(1_048_576).await;
4353 let exec = ExecutionId::new();
4354 backend
4355 .open_execution(exec, ExecutionKind::AgentTurn)
4356 .await
4357 .unwrap();
4358 let batch = vec![
4359 step_result(exec, 0, b"a"),
4360 step_result(exec, 1, b"b"),
4361 step_result(exec, 2, b"c"),
4362 ];
4363 backend.append_batch(&batch).await.unwrap();
4364 assert_eq!(backend.read_execution(exec).await.unwrap().len(), 3);
4365 }
4366
4367 #[tokio::test]
4368 async fn read_execution_range_bounds_the_segment() {
4369 let backend = mem_backend(1_048_576).await;
4370 let exec = ExecutionId::new();
4371 backend
4372 .open_execution(exec, ExecutionKind::AgentTurn)
4373 .await
4374 .unwrap();
4375 for step in 0..5 {
4376 backend.append(step_result(exec, step, b"x")).await.unwrap();
4377 }
4378 let segment = backend.read_execution_range(exec, 2, 2).await.unwrap();
4379 assert_eq!(segment.len(), 2);
4380 assert_eq!(segment[0].step_id, StepId::new(2));
4381 assert_eq!(segment[1].step_id, StepId::new(3));
4382 }
4383
4384 #[tokio::test]
4385 async fn lookup_committed_result_finds_by_idem_key() {
4386 let backend = mem_backend(1_048_576).await;
4387 let exec = ExecutionId::new();
4388 backend
4389 .open_execution(exec, ExecutionKind::AgentTurn)
4390 .await
4391 .unwrap();
4392 let entry = step_result(exec, 0, b"committed");
4393 let idem_key = match &entry.entry {
4394 EntryKind::StepResult {
4395 idempotency_key, ..
4396 } => *idempotency_key,
4397 other => panic!("unexpected entry kind: {other:?}"),
4398 };
4399 backend.append(entry).await.unwrap();
4400
4401 let found = backend
4402 .lookup_committed_result(exec, idem_key)
4403 .await
4404 .unwrap()
4405 .expect("committed result is located by its idempotency key");
4406 match &found.entry {
4407 EntryKind::StepResult { payload, .. } => assert_eq!(payload.as_ref(), b"committed"),
4408 other => panic!("unexpected entry kind: {other:?}"),
4409 }
4410
4411 let absent = IdempotencyKey::derive(exec, StepId::new(99), b"never");
4413 assert!(
4414 backend
4415 .lookup_committed_result(exec, absent)
4416 .await
4417 .unwrap()
4418 .is_none()
4419 );
4420 }
4421
4422 #[tokio::test]
4423 async fn capabilities_describe_the_local_profile() {
4424 let backend = mem_backend(4096).await;
4425 let caps = backend.capabilities();
4426 assert!(caps.parallel_steps);
4427 assert!(
4428 !caps.cross_process,
4429 "the SQLite local backend is in-process"
4430 );
4431 assert_eq!(caps.max_payload, 4096);
4432 }
4433
4434 #[tokio::test]
4435 async fn promise_insert_state_and_resolve_round_trip() {
4436 let backend = mem_backend(1_048_576)
4437 .await
4438 .with_cipher(Arc::new(XorCipher));
4439 let exec = ExecutionId::new();
4440 backend
4441 .open_execution(exec, ExecutionKind::AgentTurn)
4442 .await
4443 .unwrap();
4444 let promise = PromiseId::derive(exec, StepId::new(0));
4445 backend
4446 .insert_promise(promise, exec, [9u8; 32], 100)
4447 .await
4448 .unwrap();
4449
4450 let pending = backend.promise_state(promise).await.unwrap().unwrap();
4451 assert!(!pending.resolved);
4452 assert_eq!(pending.execution_id, exec);
4453 assert_eq!(pending.resolver_token_hash, [9u8; 32]);
4454
4455 assert!(
4457 backend
4458 .resolve_promise(promise, exec, b"answer", 200)
4459 .await
4460 .unwrap()
4461 );
4462 assert!(
4463 !backend
4464 .resolve_promise(promise, exec, b"again", 300)
4465 .await
4466 .unwrap()
4467 );
4468
4469 let resolved = backend.promise_state(promise).await.unwrap().unwrap();
4470 assert!(resolved.resolved);
4471 let sealed = resolved.payload.expect("resolved payload present");
4472 assert_ne!(sealed.as_slice(), b"answer", "payload is sealed at rest");
4473 let opened = backend
4474 .open_promise_payload(promise, exec, &sealed)
4475 .unwrap();
4476 assert_eq!(opened.as_ref(), b"answer");
4477 }
4478
4479 #[tokio::test]
4480 async fn claim_promise_notification_is_single_winner() {
4481 let backend = mem_backend(1_048_576).await;
4482 let exec = ExecutionId::new();
4483 backend
4484 .open_execution(exec, ExecutionKind::AgentTurn)
4485 .await
4486 .unwrap();
4487 let promise = PromiseId::derive(exec, StepId::new(0));
4488 backend
4489 .insert_promise(promise, exec, [9u8; 32], 100)
4490 .await
4491 .unwrap();
4492
4493 assert!(
4495 backend
4496 .claim_promise_notification(promise, 200)
4497 .await
4498 .unwrap()
4499 );
4500 assert!(
4502 !backend
4503 .claim_promise_notification(promise, 300)
4504 .await
4505 .unwrap()
4506 );
4507 }
4508
4509 #[tokio::test]
4510 async fn timer_arm_due_and_fire() {
4511 let backend = mem_backend(1_048_576).await;
4512 let exec = ExecutionId::new();
4513 backend
4514 .open_execution(exec, ExecutionKind::AgentTurn)
4515 .await
4516 .unwrap();
4517 let past = TimerId::derive(exec, StepId::new(0));
4518 let future = TimerId::derive(exec, StepId::new(1));
4519 backend.arm_timer(past, exec, 1_000, 0).await.unwrap();
4520 backend
4521 .arm_timer(future, exec, 9_000_000_000_000, 0)
4522 .await
4523 .unwrap();
4524
4525 let due = backend.due_timers(5_000).await.unwrap();
4527 assert_eq!(due, vec![past]);
4528
4529 assert!(backend.mark_timer_fired(past).await.unwrap());
4530 assert!(
4531 !backend.mark_timer_fired(past).await.unwrap(),
4532 "second fire is a no-op"
4533 );
4534 assert_eq!(
4535 backend.timer_state(past).await.unwrap(),
4536 Some((1_000, true))
4537 );
4538 assert!(backend.due_timers(5_000).await.unwrap().is_empty());
4540 }
4541
4542 #[tokio::test]
4543 async fn prune_deletes_terminal_executions_past_ttl() {
4544 let backend = mem_backend(1_048_576).await;
4545 let old = ExecutionId::new();
4547 backend
4548 .open_execution(old, ExecutionKind::AgentTurn)
4549 .await
4550 .unwrap();
4551 backend.append(step_result(old, 0, b"x")).await.unwrap();
4552 zeph_db::query(sql!(
4554 "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
4555 ))
4556 .bind(old.as_uuid().to_string())
4557 .execute(backend.pool())
4558 .await
4559 .unwrap();
4560
4561 let live = ExecutionId::new();
4562 backend
4563 .open_execution(live, ExecutionKind::AgentTurn)
4564 .await
4565 .unwrap();
4566 backend.append(step_result(live, 0, b"y")).await.unwrap();
4567
4568 let policy = RetentionPolicy {
4569 ttl_completed_secs: 1,
4570 prune_batch_size: 10,
4571 ..RetentionPolicy::default()
4572 };
4573 let deleted = backend.prune(&policy).await.unwrap();
4574 assert_eq!(deleted, 1, "only the aged terminal execution is pruned");
4575
4576 assert!(backend.read_execution(old).await.unwrap().is_empty());
4578 assert!(
4579 backend
4580 .promise_state(PromiseId::derive(old, StepId::new(0)))
4581 .await
4582 .unwrap()
4583 .is_none()
4584 );
4585 assert_eq!(backend.read_execution(live).await.unwrap().len(), 1);
4586 }
4587
4588 #[tokio::test]
4589 async fn count_prunable_and_prune_include_canceled_executions_past_ttl() {
4590 let backend = mem_backend(1_048_576).await;
4593 let exec = ExecutionId::new();
4594 backend
4595 .open_execution(exec, ExecutionKind::AgentTurn)
4596 .await
4597 .unwrap();
4598 assert_eq!(
4599 backend.cancel_execution(exec).await.unwrap(),
4600 CancelOutcome::Canceled
4601 );
4602 zeph_db::query(sql!(
4604 "UPDATE durable_executions SET finalized_at = 1000 WHERE execution_id = ?"
4605 ))
4606 .bind(exec.as_uuid().to_string())
4607 .execute(backend.pool())
4608 .await
4609 .unwrap();
4610
4611 let policy = RetentionPolicy {
4612 ttl_failed_secs: 1,
4613 prune_batch_size: 10,
4614 ..RetentionPolicy::default()
4615 };
4616 let prunable = backend.count_prunable(&policy).await.unwrap();
4617 assert_eq!(
4618 prunable, 1,
4619 "an aged canceled row must be counted as prunable"
4620 );
4621
4622 let deleted = backend.prune(&policy).await.unwrap();
4623 assert_eq!(deleted, 1, "an aged canceled row must actually be pruned");
4624 assert!(backend.read_execution(exec).await.unwrap().is_empty());
4625 }
4626
4627 #[tokio::test]
4639 async fn prune_deletes_a_keyed_execution_and_its_integrity_row() {
4640 let backend = mem_backend(1_048_576).await.with_hwm_key(0, [42u8; 32]);
4641 let old = ExecutionId::new();
4642 backend
4643 .open_execution(old, ExecutionKind::AgentTurn)
4644 .await
4645 .unwrap();
4646 backend.append(step_result(old, 0, b"x")).await.unwrap();
4647
4648 let before: (i64,) = zeph_db::query_as(sql!(
4650 "SELECT COUNT(*) FROM durable_execution_integrity WHERE execution_id = ?"
4651 ))
4652 .bind(old.as_uuid().to_string())
4653 .fetch_one(backend.pool())
4654 .await
4655 .unwrap();
4656 assert_eq!(
4657 before.0, 1,
4658 "a committed StepResult must create an integrity row"
4659 );
4660
4661 zeph_db::query(sql!(
4662 "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
4663 ))
4664 .bind(old.as_uuid().to_string())
4665 .execute(backend.pool())
4666 .await
4667 .unwrap();
4668
4669 let policy = RetentionPolicy {
4670 ttl_completed_secs: 1,
4671 prune_batch_size: 10,
4672 ..RetentionPolicy::default()
4673 };
4674 let deleted = backend
4675 .prune(&policy)
4676 .await
4677 .expect("prune must not fail closed on a keyed execution's FK");
4678 assert_eq!(deleted, 1, "the keyed execution is pruned like any other");
4679
4680 assert!(backend.read_execution(old).await.unwrap().is_empty());
4681 let after: (i64,) = zeph_db::query_as(sql!(
4682 "SELECT COUNT(*) FROM durable_execution_integrity WHERE execution_id = ?"
4683 ))
4684 .bind(old.as_uuid().to_string())
4685 .fetch_one(backend.pool())
4686 .await
4687 .unwrap();
4688 assert_eq!(
4689 after.0, 0,
4690 "the integrity row must be pruned alongside its execution"
4691 );
4692 }
4693
4694 async fn backdate_updated_at(backend: &LocalBackend, id: ExecutionId, updated_at_ms: i64) {
4696 zeph_db::query(sql!(
4697 "UPDATE durable_executions SET updated_at = ? WHERE execution_id = ?"
4698 ))
4699 .bind(updated_at_ms)
4700 .bind(id.as_uuid().to_string())
4701 .execute(backend.pool())
4702 .await
4703 .unwrap();
4704 }
4705
4706 #[tokio::test]
4707 async fn sweep_orphans_disabled_when_threshold_is_zero() {
4708 let dir = tempfile::tempdir().unwrap();
4711 let backend =
4712 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4713 .await
4714 .unwrap();
4715 backend.init().await.unwrap();
4716
4717 let exec = ExecutionId::new();
4718 backend
4719 .open_execution(exec, ExecutionKind::AgentTurn)
4720 .await
4721 .unwrap();
4722 backdate_updated_at(&backend, exec, 0).await;
4723
4724 let policy = RetentionPolicy {
4725 stale_running_after_secs: 0,
4726 ..RetentionPolicy::default()
4727 };
4728 let aborted = backend.sweep_orphans(&policy).await.unwrap();
4729 assert_eq!(
4730 aborted, 0,
4731 "stale_running_after_secs = 0 disables the sweep"
4732 );
4733
4734 let (status,): (String,) = zeph_db::query_as(sql!(
4735 "SELECT status FROM durable_executions WHERE execution_id = ?"
4736 ))
4737 .bind(exec.as_uuid().to_string())
4738 .fetch_one(backend.pool())
4739 .await
4740 .unwrap();
4741 assert_eq!(status, "running");
4742 }
4743
4744 #[tokio::test]
4745 async fn sweep_orphans_is_a_documented_no_op_on_memory_backend() {
4746 let backend = mem_backend(1_048_576).await;
4749 let exec = ExecutionId::new();
4750 backend
4751 .open_execution(exec, ExecutionKind::AgentTurn)
4752 .await
4753 .unwrap();
4754 backdate_updated_at(&backend, exec, 0).await;
4755
4756 let policy = RetentionPolicy {
4757 stale_running_after_secs: 1,
4758 ..RetentionPolicy::default()
4759 };
4760 let aborted = backend.sweep_orphans(&policy).await.unwrap();
4761 assert_eq!(
4762 aborted, 0,
4763 "a lock_dir=None backend must never abort on staleness alone"
4764 );
4765
4766 let (status,): (String,) = zeph_db::query_as(sql!(
4767 "SELECT status FROM durable_executions WHERE execution_id = ?"
4768 ))
4769 .bind(exec.as_uuid().to_string())
4770 .fetch_one(backend.pool())
4771 .await
4772 .unwrap();
4773 assert_eq!(status, "running");
4774 }
4775
4776 #[tokio::test]
4777 async fn sweep_orphans_aborts_a_stale_running_execution_with_no_live_owner() {
4778 let dir = tempfile::tempdir().unwrap();
4780 let backend =
4781 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4782 .await
4783 .unwrap();
4784 backend.init().await.unwrap();
4785
4786 let exec = ExecutionId::new();
4787 backend
4788 .open_execution(exec, ExecutionKind::AgentTurn)
4789 .await
4790 .unwrap();
4791 backdate_updated_at(&backend, exec, 0).await;
4794
4795 let policy = RetentionPolicy {
4796 stale_running_after_secs: 1,
4797 ..RetentionPolicy::default()
4798 };
4799 let aborted = backend.sweep_orphans(&policy).await.unwrap();
4800 assert_eq!(aborted, 1);
4801
4802 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
4803 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
4804 ))
4805 .bind(exec.as_uuid().to_string())
4806 .fetch_one(backend.pool())
4807 .await
4808 .unwrap();
4809 assert_eq!(status, "aborted");
4810 assert!(finalized.is_some());
4811 }
4812
4813 #[tokio::test]
4814 async fn sweep_orphans_skips_an_execution_whose_lock_is_held_by_a_live_owner() {
4815 let dir = tempfile::tempdir().unwrap();
4819 let db_path = dir.path().join("durable.db");
4820 let url = db_path.to_string_lossy().into_owned();
4821
4822 let owner = LocalBackend::open(&url, 1_048_576).await.unwrap();
4823 owner.init().await.unwrap();
4824 let sweeper = LocalBackend::open(&url, 1_048_576).await.unwrap();
4825
4826 let exec = ExecutionId::new();
4827 let (_, _lock) = owner
4828 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
4829 .await
4830 .unwrap();
4831 backdate_updated_at(&owner, exec, 0).await;
4832
4833 let policy = RetentionPolicy {
4834 stale_running_after_secs: 1,
4835 ..RetentionPolicy::default()
4836 };
4837 let aborted = sweeper.sweep_orphans(&policy).await.unwrap();
4838 assert_eq!(aborted, 0, "a live-held lock must never be swept");
4839
4840 let (status,): (String,) = zeph_db::query_as(sql!(
4841 "SELECT status FROM durable_executions WHERE execution_id = ?"
4842 ))
4843 .bind(exec.as_uuid().to_string())
4844 .fetch_one(owner.pool())
4845 .await
4846 .unwrap();
4847 assert_eq!(status, "running");
4848 }
4849
4850 #[tokio::test]
4851 async fn sweep_orphans_leaves_a_fresh_running_execution_untouched() {
4852 let dir = tempfile::tempdir().unwrap();
4854 let backend =
4855 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4856 .await
4857 .unwrap();
4858 backend.init().await.unwrap();
4859
4860 let exec = ExecutionId::new();
4861 backend
4862 .open_execution(exec, ExecutionKind::AgentTurn)
4863 .await
4864 .unwrap();
4865
4866 let policy = RetentionPolicy {
4867 stale_running_after_secs: 3600,
4868 ..RetentionPolicy::default()
4869 };
4870 let aborted = backend.sweep_orphans(&policy).await.unwrap();
4871 assert_eq!(aborted, 0);
4872 }
4873
4874 #[tokio::test]
4875 async fn sweep_orphans_never_touches_a_stale_canceled_row() {
4876 let dir = tempfile::tempdir().unwrap();
4880 let backend =
4881 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4882 .await
4883 .unwrap();
4884 backend.init().await.unwrap();
4885
4886 let exec = ExecutionId::new();
4887 backend
4888 .open_execution(exec, ExecutionKind::AgentTurn)
4889 .await
4890 .unwrap();
4891 assert_eq!(
4892 backend.cancel_execution(exec).await.unwrap(),
4893 CancelOutcome::Canceled
4894 );
4895 backdate_updated_at(&backend, exec, 0).await;
4896
4897 let policy = RetentionPolicy {
4898 stale_running_after_secs: 1,
4899 ..RetentionPolicy::default()
4900 };
4901 for _ in 0..3 {
4902 let aborted = backend.sweep_orphans(&policy).await.unwrap();
4903 assert_eq!(aborted, 0, "a canceled row must never be swept");
4904 }
4905
4906 let (status,): (String,) = zeph_db::query_as(sql!(
4907 "SELECT status FROM durable_executions WHERE execution_id = ?"
4908 ))
4909 .bind(exec.as_uuid().to_string())
4910 .fetch_one(backend.pool())
4911 .await
4912 .unwrap();
4913 assert_eq!(
4914 status, "canceled",
4915 "sweep must never resurrect a canceled row"
4916 );
4917 }
4918
4919 #[tokio::test]
4920 async fn count_orphans_matches_sweep_without_mutating() {
4921 let dir = tempfile::tempdir().unwrap();
4922 let backend =
4923 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4924 .await
4925 .unwrap();
4926 backend.init().await.unwrap();
4927
4928 let exec = ExecutionId::new();
4929 backend
4930 .open_execution(exec, ExecutionKind::AgentTurn)
4931 .await
4932 .unwrap();
4933 backdate_updated_at(&backend, exec, 0).await;
4934
4935 let policy = RetentionPolicy {
4936 stale_running_after_secs: 1,
4937 ..RetentionPolicy::default()
4938 };
4939 let counted = backend.count_orphans(&policy).await.unwrap();
4940 assert_eq!(counted, 1);
4941
4942 let (status,): (String,) = zeph_db::query_as(sql!(
4944 "SELECT status FROM durable_executions WHERE execution_id = ?"
4945 ))
4946 .bind(exec.as_uuid().to_string())
4947 .fetch_one(backend.pool())
4948 .await
4949 .unwrap();
4950 assert_eq!(status, "running");
4951
4952 let aborted = backend.sweep_orphans(&policy).await.unwrap();
4953 assert_eq!(
4954 aborted, counted,
4955 "sweep must abort exactly what count_orphans counted"
4956 );
4957 }
4958
4959 #[tokio::test]
4965 async fn sweep_orphans_processes_every_batch_when_candidates_straddle_the_batch_size() {
4966 let dir = tempfile::tempdir().unwrap();
4967 let backend =
4968 LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
4969 .await
4970 .unwrap();
4971 backend.init().await.unwrap();
4972
4973 let batch_size = 2u64;
4974 let candidate_count = batch_size + 1; let mut execs = Vec::new();
4976 for _ in 0..candidate_count {
4977 let exec = ExecutionId::new();
4978 backend
4979 .open_execution(exec, ExecutionKind::AgentTurn)
4980 .await
4981 .unwrap();
4982 backdate_updated_at(&backend, exec, 0).await;
4983 execs.push(exec);
4984 }
4985
4986 let policy = RetentionPolicy {
4987 stale_running_after_secs: 1,
4988 prune_batch_size: batch_size,
4989 ..RetentionPolicy::default()
4990 };
4991 let aborted = backend.sweep_orphans(&policy).await.unwrap();
4992 assert_eq!(
4993 aborted, candidate_count,
4994 "every candidate must be aborted, including the one past the first batch"
4995 );
4996
4997 for exec in execs {
4998 let (status,): (String,) = zeph_db::query_as(sql!(
4999 "SELECT status FROM durable_executions WHERE execution_id = ?"
5000 ))
5001 .bind(exec.as_uuid().to_string())
5002 .fetch_one(backend.pool())
5003 .await
5004 .unwrap();
5005 assert_eq!(status, "aborted");
5006 }
5007 }
5008
5009 #[tokio::test]
5022 async fn sweep_orphans_terminates_when_lock_held_candidates_exceed_batch_size() {
5023 let dir = tempfile::tempdir().unwrap();
5024 let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
5025
5026 let owner = LocalBackend::open(&db_url, 1_048_576).await.unwrap();
5027 owner.init().await.unwrap();
5028 let sweeper = LocalBackend::open(&db_url, 1_048_576).await.unwrap();
5029
5030 let batch_size = 2u64;
5031 let candidate_count = batch_size * 2 + 1; let mut locks = Vec::new();
5033 for _ in 0..candidate_count {
5034 let exec = ExecutionId::new();
5035 let (_, lock) = owner
5036 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
5037 .await
5038 .unwrap();
5039 backdate_updated_at(&owner, exec, 0).await;
5040 locks.push(lock); }
5042
5043 let policy = RetentionPolicy {
5044 stale_running_after_secs: 1,
5045 prune_batch_size: batch_size,
5046 ..RetentionPolicy::default()
5047 };
5048
5049 let aborted = tokio::time::timeout(
5050 std::time::Duration::from_secs(10),
5051 sweeper.sweep_orphans(&policy),
5052 )
5053 .await
5054 .expect(
5055 "sweep_orphans must terminate even when lock-held candidates exceed prune_batch_size \
5056 (#6254 C1) — it hung instead of returning",
5057 )
5058 .unwrap();
5059
5060 assert_eq!(aborted, 0, "every candidate's lock is held by a live owner");
5061 drop(locks);
5062 }
5063
5064 #[tokio::test]
5073 async fn concurrent_sweep_and_reopen_race_never_corrupts_the_row() {
5074 let dir = tempfile::tempdir().unwrap();
5075 let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
5076 let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
5077 backend.init().await.unwrap();
5078
5079 let policy = RetentionPolicy {
5080 stale_running_after_secs: 1,
5081 prune_batch_size: 10,
5082 ..RetentionPolicy::default()
5083 };
5084
5085 for _ in 0..20 {
5086 let exec = ExecutionId::new();
5087 backend
5088 .open_execution(exec, ExecutionKind::AgentTurn)
5089 .await
5090 .unwrap();
5091 backdate_updated_at(&backend, exec, 0).await;
5092
5093 let sweep_backend = backend.clone();
5094 let policy_for_task = policy.clone();
5095 let sweep =
5096 tokio::spawn(async move { sweep_backend.sweep_orphans(&policy_for_task).await });
5097
5098 let reopen_backend = backend.clone();
5099 let reopen = tokio::spawn(async move {
5100 reopen_backend
5101 .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
5102 .await
5103 });
5104
5105 let (sweep_result, reopen_result) = tokio::join!(sweep, reopen);
5106 let aborted = sweep_result
5107 .expect("sweep task must not panic")
5108 .expect("sweep must not error under a concurrent reopen");
5109 assert!(aborted <= 1, "at most one candidate row exists per trial");
5110
5111 match reopen_result.expect("reopen task must not panic") {
5112 Ok((_is_resume, _lock)) => {
5113 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
5117 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
5118 ))
5119 .bind(exec.as_uuid().to_string())
5120 .fetch_one(backend.pool())
5121 .await
5122 .unwrap();
5123 assert_eq!(status, "running");
5124 assert!(finalized.is_none());
5125 backend
5133 .finalize(exec, ExecutionStatus::Completed)
5134 .await
5135 .unwrap();
5136 }
5137 Err(DurableError::ExecutionLocked { .. }) => {
5138 }
5140 Err(e) => panic!(
5141 "reopen must only ever fail with ExecutionLocked under this race, got {e:?}"
5142 ),
5143 }
5144 }
5145 }
5146
5147 #[tokio::test]
5148 async fn checkpoint_fold_compacts_idempotent_prefix_and_replays() {
5149 let backend = mem_backend(1_048_576)
5150 .await
5151 .with_cipher(Arc::new(XorCipher));
5152 let exec = ExecutionId::new();
5153 backend
5154 .open_execution(exec, ExecutionKind::AgentTurn)
5155 .await
5156 .unwrap();
5157 for step in 0..5 {
5158 backend
5159 .append(step_result(exec, step, format!("v{step}").as_bytes()))
5160 .await
5161 .unwrap();
5162 }
5163
5164 let folded = backend.checkpoint_fold(exec, 3).await.unwrap();
5166 assert_eq!(folded, 3);
5167
5168 let remaining = backend.read_execution(exec).await.unwrap();
5170 let step_results: Vec<u32> = remaining
5171 .iter()
5172 .filter(|e| matches!(e.entry, EntryKind::StepResult { .. }))
5173 .map(|e| e.step_id.value())
5174 .collect();
5175 assert_eq!(step_results, vec![3, 4], "folded step rows are deleted");
5176 assert!(
5177 remaining
5178 .iter()
5179 .any(|e| matches!(e.entry, EntryKind::Checkpoint { .. })),
5180 "a checkpoint entry replaces the folded prefix"
5181 );
5182
5183 let preloaded = backend.read_checkpoints(exec).await.unwrap();
5185 assert_eq!(preloaded.len(), 3);
5186 for (i, entry) in preloaded.iter().enumerate() {
5187 let step = u32::try_from(i).unwrap();
5188 assert_eq!(entry.step_id, StepId::new(step));
5189 match &entry.entry {
5190 EntryKind::StepResult {
5191 payload,
5192 idempotency_key,
5193 ..
5194 } => {
5195 assert_eq!(payload.as_ref(), format!("v{step}").as_bytes());
5196 assert_eq!(
5197 *idempotency_key,
5198 IdempotencyKey::derive(exec, StepId::new(step), b"tool:read")
5199 );
5200 }
5201 other => panic!("unexpected folded entry: {other:?}"),
5202 }
5203 }
5204 }
5205
5206 #[tokio::test]
5212 async fn hwm_is_a_no_op_when_unkeyed() {
5213 let backend = mem_backend(1_048_576).await;
5214 let exec = ExecutionId::new();
5215 assert!(
5216 !backend
5217 .open_execution(exec, ExecutionKind::AgentTurn)
5218 .await
5219 .unwrap()
5220 );
5221 backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5222 assert!(
5224 backend
5225 .open_execution(exec, ExecutionKind::AgentTurn)
5226 .await
5227 .unwrap()
5228 );
5229 }
5230
5231 #[tokio::test]
5232 async fn hwm_verifies_on_resume_after_single_append_and_batch_append() {
5233 let backend = mem_backend(1_048_576).await.with_hwm_key(0, [1u8; 32]);
5234 let exec = ExecutionId::new();
5235 assert!(
5236 !backend
5237 .open_execution(exec, ExecutionKind::AgentTurn)
5238 .await
5239 .unwrap()
5240 );
5241 backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5242 backend
5243 .append_batch(&[step_result(exec, 1, b"v1"), step_result(exec, 2, b"v2")])
5244 .await
5245 .unwrap();
5246
5247 assert!(
5248 backend
5249 .open_execution(exec, ExecutionKind::AgentTurn)
5250 .await
5251 .unwrap(),
5252 "resume must succeed when the recomputed count matches the signed HWM"
5253 );
5254 }
5255
5256 #[tokio::test]
5257 async fn hwm_detects_deletion_of_a_committed_step_result() {
5258 let backend = mem_backend(1_048_576).await.with_hwm_key(0, [2u8; 32]);
5259 let exec = ExecutionId::new();
5260 backend
5261 .open_execution(exec, ExecutionKind::AgentTurn)
5262 .await
5263 .unwrap();
5264 backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5265 backend.append(step_result(exec, 1, b"v1")).await.unwrap();
5266
5267 zeph_db::query(sql!(
5270 "DELETE FROM durable_journal WHERE execution_id = ? AND step_id = 1"
5271 ))
5272 .bind(exec.as_uuid().to_string())
5273 .execute(backend.pool())
5274 .await
5275 .unwrap();
5276
5277 let err = backend
5278 .open_execution(exec, ExecutionKind::AgentTurn)
5279 .await
5280 .unwrap_err();
5281 assert_matches!(
5282 err,
5283 DurableError::HighWaterMarkIntegrity {
5284 reason: "count_mismatch",
5285 ..
5286 }
5287 );
5288
5289 let summaries = backend.list_executions(None, None, 10).await.unwrap();
5292 let summary = summaries.iter().find(|s| s.execution_id == exec).unwrap();
5293 assert_eq!(summary.status, ExecutionStatus::Aborted);
5294 }
5295
5296 #[tokio::test]
5297 async fn hwm_survives_a_legitimate_checkpoint_fold() {
5298 let backend = mem_backend(1_048_576)
5299 .await
5300 .with_cipher(Arc::new(XorCipher))
5301 .with_hwm_key(0, [3u8; 32]);
5302 let exec = ExecutionId::new();
5303 backend
5304 .open_execution(exec, ExecutionKind::AgentTurn)
5305 .await
5306 .unwrap();
5307 for step in 0..5 {
5308 backend
5309 .append(step_result(exec, step, format!("v{step}").as_bytes()))
5310 .await
5311 .unwrap();
5312 }
5313
5314 let folded = backend.checkpoint_fold(exec, 3).await.unwrap();
5315 assert_eq!(folded, 3);
5316
5317 assert!(
5318 backend
5319 .open_execution(exec, ExecutionKind::AgentTurn)
5320 .await
5321 .unwrap(),
5322 "a legitimate fold must not trip the HWM check: committed_result_count is invariant \
5323 across it (folded_count restores what the DELETE removed)"
5324 );
5325 }
5326
5327 #[tokio::test]
5337 async fn count_integrity_rows_under_epoch_catches_a_checkpoint_folded_pre_rotation_execution() {
5338 let pre_rotation = mem_backend(1_048_576)
5339 .await
5340 .with_cipher(Arc::new(RotatingKeyedCipher {
5341 current_id: 0,
5342 previous_id: None,
5343 }))
5344 .with_hwm_key(0, [20u8; 32]);
5345 let exec = ExecutionId::new();
5346 pre_rotation
5347 .open_execution(exec, ExecutionKind::AgentTurn)
5348 .await
5349 .unwrap();
5350 for step in 0..3 {
5351 pre_rotation
5352 .append(step_result(exec, step, format!("v{step}").as_bytes()))
5353 .await
5354 .unwrap();
5355 }
5356
5357 let post_rotation = LocalBackend::new(pre_rotation.pool().clone(), 1_048_576)
5362 .with_cipher(Arc::new(RotatingKeyedCipher {
5363 current_id: 1,
5364 previous_id: Some(0),
5365 }))
5366 .with_hwm_key(1, [21u8; 32])
5367 .with_previous_hwm_key(0, [20u8; 32]);
5368
5369 let folded = post_rotation.checkpoint_fold(exec, 3).await.unwrap();
5370 assert_eq!(
5371 folded, 3,
5372 "fold must compact every committed StepResult, leaving none live"
5373 );
5374
5375 assert_eq!(
5376 post_rotation.count_sealed_under_key_id(0).await.unwrap(),
5377 0,
5378 "every pre-rotation payload was folded away and resealed under the new key_id; the \
5379 AEAD scan sees nothing left sealed under the previous key_id"
5380 );
5381 assert_eq!(
5382 post_rotation
5383 .count_integrity_rows_under_epoch(0)
5384 .await
5385 .unwrap(),
5386 1,
5387 "the folded execution's HWM row still carries the previous epoch -- checkpoint_fold \
5388 never re-signs it (S1)"
5389 );
5390 assert_eq!(
5391 post_rotation
5392 .count_integrity_rows_under_epoch(1)
5393 .await
5394 .unwrap(),
5395 0,
5396 "the row has not migrated to the current epoch -- only a fresh StepResult commit \
5397 after resume would bump it"
5398 );
5399
5400 assert!(
5404 post_rotation
5405 .open_execution(exec, ExecutionKind::AgentTurn)
5406 .await
5407 .unwrap(),
5408 "a folded pre-rotation execution must still resume through the open rotation window"
5409 );
5410 }
5411
5412 #[tokio::test]
5413 async fn hwm_detects_deletion_that_a_fold_does_not_cover() {
5414 let backend = mem_backend(1_048_576)
5415 .await
5416 .with_cipher(Arc::new(XorCipher))
5417 .with_hwm_key(0, [4u8; 32]);
5418 let exec = ExecutionId::new();
5419 backend
5420 .open_execution(exec, ExecutionKind::AgentTurn)
5421 .await
5422 .unwrap();
5423 for step in 0..5 {
5424 backend
5425 .append(step_result(exec, step, format!("v{step}").as_bytes()))
5426 .await
5427 .unwrap();
5428 }
5429 backend.checkpoint_fold(exec, 3).await.unwrap();
5430
5431 zeph_db::query(sql!(
5433 "DELETE FROM durable_journal WHERE execution_id = ? AND step_id = 4 AND entry_kind = 'step_result'"
5434 ))
5435 .bind(exec.as_uuid().to_string())
5436 .execute(backend.pool())
5437 .await
5438 .unwrap();
5439
5440 assert_matches!(
5441 backend
5442 .open_execution(exec, ExecutionKind::AgentTurn)
5443 .await
5444 .unwrap_err(),
5445 DurableError::HighWaterMarkIntegrity {
5446 reason: "count_mismatch",
5447 ..
5448 }
5449 );
5450 }
5451
5452 #[tokio::test]
5453 async fn hwm_unresolvable_key_epoch_fails_closed_not_legacy() {
5454 let writer = mem_backend(1_048_576).await.with_hwm_key(0, [5u8; 32]);
5455 let exec = ExecutionId::new();
5456 writer
5457 .open_execution(exec, ExecutionKind::AgentTurn)
5458 .await
5459 .unwrap();
5460 writer.append(step_result(exec, 0, b"v0")).await.unwrap();
5461
5462 let reader = LocalBackend::new(writer.pool().clone(), 1_048_576).with_hwm_key(9, [6u8; 32]);
5466 assert_matches!(
5467 reader
5468 .open_execution(exec, ExecutionKind::AgentTurn)
5469 .await
5470 .unwrap_err(),
5471 DurableError::HighWaterMarkIntegrity {
5472 reason: "key_epoch_unresolvable",
5473 ..
5474 }
5475 );
5476 }
5477
5478 #[tokio::test]
5479 async fn hwm_previous_epoch_key_resolves_as_rekeyed_not_tampered() {
5480 let writer = mem_backend(1_048_576).await.with_hwm_key(0, [7u8; 32]);
5481 let exec = ExecutionId::new();
5482 writer
5483 .open_execution(exec, ExecutionKind::AgentTurn)
5484 .await
5485 .unwrap();
5486 writer.append(step_result(exec, 0, b"v0")).await.unwrap();
5487
5488 let reader = LocalBackend::new(writer.pool().clone(), 1_048_576)
5492 .with_hwm_key(1, [8u8; 32])
5493 .with_previous_hwm_key(0, [7u8; 32]);
5494 assert!(
5495 reader
5496 .open_execution(exec, ExecutionKind::AgentTurn)
5497 .await
5498 .unwrap(),
5499 "a row signed under a registered previous epoch must verify, not fail as tampered"
5500 );
5501 }
5502
5503 #[tokio::test]
5504 async fn hwm_wrong_key_under_the_same_epoch_is_tamper() {
5505 let writer = mem_backend(1_048_576).await.with_hwm_key(0, [9u8; 32]);
5506 let exec = ExecutionId::new();
5507 writer
5508 .open_execution(exec, ExecutionKind::AgentTurn)
5509 .await
5510 .unwrap();
5511 writer.append(step_result(exec, 0, b"v0")).await.unwrap();
5512
5513 let reader =
5514 LocalBackend::new(writer.pool().clone(), 1_048_576).with_hwm_key(0, [10u8; 32]);
5515 assert_matches!(
5516 reader
5517 .open_execution(exec, ExecutionKind::AgentTurn)
5518 .await
5519 .unwrap_err(),
5520 DurableError::HighWaterMarkIntegrity {
5521 reason: "hmac_mismatch",
5522 ..
5523 }
5524 );
5525 }
5526
5527 #[tokio::test]
5528 async fn hwm_accepts_a_legacy_execution_with_no_integrity_row() {
5529 let unkeyed_writer = mem_backend(1_048_576).await;
5533 let exec = ExecutionId::new();
5534 unkeyed_writer
5535 .open_execution(exec, ExecutionKind::AgentTurn)
5536 .await
5537 .unwrap();
5538 unkeyed_writer
5539 .append(step_result(exec, 0, b"v0"))
5540 .await
5541 .unwrap();
5542
5543 let keyed_reader =
5544 LocalBackend::new(unkeyed_writer.pool().clone(), 1_048_576).with_hwm_key(0, [11u8; 32]);
5545 assert!(
5546 keyed_reader
5547 .open_execution(exec, ExecutionKind::AgentTurn)
5548 .await
5549 .unwrap(),
5550 "an execution with no integrity row at all is legacy, not tampered"
5551 );
5552 }
5553
5554 #[tokio::test]
5557 async fn hwm_unsealed_absent_row_after_deletion_is_still_ok() {
5558 let backend = mem_backend(1_048_576).await.with_hwm_key(0, [30u8; 32]);
5562 let exec = ExecutionId::new();
5563 backend
5564 .open_execution(exec, ExecutionKind::AgentTurn)
5565 .await
5566 .unwrap();
5567 backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5568
5569 zeph_db::query(sql!(
5570 "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5571 ))
5572 .bind(exec.as_uuid().to_string())
5573 .execute(backend.pool())
5574 .await
5575 .unwrap();
5576
5577 assert!(
5578 backend
5579 .open_execution(exec, ExecutionKind::AgentTurn)
5580 .await
5581 .unwrap(),
5582 "unsealed backend must not treat an absent integrity row as tamper"
5583 );
5584 }
5585
5586 #[tokio::test]
5587 async fn hwm_post_seal_absent_row_with_committed_results_is_tamper() {
5588 let backend = mem_backend(1_048_576)
5589 .await
5590 .with_hwm_key(0, [31u8; 32])
5591 .with_integrity_sealed(true);
5592 let exec = ExecutionId::new();
5593 backend
5594 .open_execution(exec, ExecutionKind::AgentTurn)
5595 .await
5596 .unwrap();
5597 backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5598
5599 zeph_db::query(sql!(
5602 "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5603 ))
5604 .bind(exec.as_uuid().to_string())
5605 .execute(backend.pool())
5606 .await
5607 .unwrap();
5608
5609 let err = backend
5610 .open_execution(exec, ExecutionKind::AgentTurn)
5611 .await
5612 .unwrap_err();
5613 assert_matches!(
5614 err,
5615 DurableError::HighWaterMarkIntegrity {
5616 reason: "integrity_row_absent_post_seal",
5617 ..
5618 }
5619 );
5620 }
5621
5622 #[tokio::test]
5623 async fn hwm_post_seal_forged_created_at_does_not_evade_the_seal() {
5624 let backend = mem_backend(1_048_576)
5627 .await
5628 .with_hwm_key(0, [32u8; 32])
5629 .with_integrity_sealed(true);
5630 let exec = ExecutionId::new();
5631 backend
5632 .open_execution(exec, ExecutionKind::AgentTurn)
5633 .await
5634 .unwrap();
5635 backend.append(step_result(exec, 0, b"v0")).await.unwrap();
5636
5637 zeph_db::query(sql!(
5638 "UPDATE durable_executions SET created_at = 0 WHERE execution_id = ?"
5639 ))
5640 .bind(exec.as_uuid().to_string())
5641 .execute(backend.pool())
5642 .await
5643 .unwrap();
5644 zeph_db::query(sql!(
5645 "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5646 ))
5647 .bind(exec.as_uuid().to_string())
5648 .execute(backend.pool())
5649 .await
5650 .unwrap();
5651
5652 let err = backend
5653 .open_execution(exec, ExecutionKind::AgentTurn)
5654 .await
5655 .unwrap_err();
5656 assert_matches!(
5657 err,
5658 DurableError::HighWaterMarkIntegrity {
5659 reason: "integrity_row_absent_post_seal",
5660 ..
5661 },
5662 "forging created_at must not evade the seal — it is never consulted"
5663 );
5664 }
5665
5666 #[tokio::test]
5667 async fn hwm_grandfathered_execution_absent_row_is_ok() {
5668 let exec = ExecutionId::new();
5669 let writer = mem_backend(1_048_576).await.with_hwm_key(0, [33u8; 32]);
5670 writer
5671 .open_execution(exec, ExecutionKind::AgentTurn)
5672 .await
5673 .unwrap();
5674 writer.append(step_result(exec, 0, b"v0")).await.unwrap();
5675 zeph_db::query(sql!(
5676 "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5677 ))
5678 .bind(exec.as_uuid().to_string())
5679 .execute(writer.pool())
5680 .await
5681 .unwrap();
5682
5683 let sealed_but_grandfathered = LocalBackend::new(writer.pool().clone(), 1_048_576)
5684 .with_hwm_key(0, [33u8; 32])
5685 .with_integrity_sealed(true)
5686 .with_grandfather(std::collections::HashSet::from([exec]));
5687
5688 assert!(
5689 sealed_but_grandfathered
5690 .open_execution(exec, ExecutionKind::AgentTurn)
5691 .await
5692 .unwrap(),
5693 "a grandfathered execution_id must resume despite the seal"
5694 );
5695 }
5696
5697 #[tokio::test]
5698 async fn find_unsealed_resumable_executions_finds_only_the_offending_set() {
5699 let backend = mem_backend(1_048_576).await.with_hwm_key(0, [35u8; 32]);
5700
5701 let offending = ExecutionId::new();
5703 backend
5704 .open_execution(offending, ExecutionKind::AgentTurn)
5705 .await
5706 .unwrap();
5707 backend
5708 .append(step_result(offending, 0, b"v0"))
5709 .await
5710 .unwrap();
5711 zeph_db::query(sql!(
5712 "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5713 ))
5714 .bind(offending.as_uuid().to_string())
5715 .execute(backend.pool())
5716 .await
5717 .unwrap();
5718
5719 let intact = ExecutionId::new();
5721 backend
5722 .open_execution(intact, ExecutionKind::AgentTurn)
5723 .await
5724 .unwrap();
5725 backend.append(step_result(intact, 0, b"v0")).await.unwrap();
5726
5727 let empty = ExecutionId::new();
5729 backend
5730 .open_execution(empty, ExecutionKind::AgentTurn)
5731 .await
5732 .unwrap();
5733
5734 let terminal = ExecutionId::new();
5736 backend
5737 .open_execution(terminal, ExecutionKind::AgentTurn)
5738 .await
5739 .unwrap();
5740 backend
5741 .append(step_result(terminal, 0, b"v0"))
5742 .await
5743 .unwrap();
5744 zeph_db::query(sql!(
5745 "DELETE FROM durable_execution_integrity WHERE execution_id = ?"
5746 ))
5747 .bind(terminal.as_uuid().to_string())
5748 .execute(backend.pool())
5749 .await
5750 .unwrap();
5751 backend
5752 .finalize(terminal, ExecutionStatus::Completed)
5753 .await
5754 .unwrap();
5755
5756 let found = backend.find_unsealed_resumable_executions().await.unwrap();
5757 assert_eq!(
5758 found,
5759 vec![offending],
5760 "only the truly offending execution must be returned"
5761 );
5762 }
5763
5764 #[tokio::test]
5765 async fn hwm_post_seal_absent_row_with_zero_committed_results_is_ok() {
5766 let backend = mem_backend(1_048_576)
5769 .await
5770 .with_hwm_key(0, [34u8; 32])
5771 .with_integrity_sealed(true);
5772 let exec = ExecutionId::new();
5773 backend
5774 .open_execution(exec, ExecutionKind::AgentTurn)
5775 .await
5776 .unwrap();
5777
5778 assert!(
5779 backend
5780 .open_execution(exec, ExecutionKind::AgentTurn)
5781 .await
5782 .unwrap(),
5783 "zero committed results, post-seal, must not be treated as tamper"
5784 );
5785 }
5786
5787 #[tokio::test]
5788 async fn hwm_ignores_effect_intent_and_control_entries() {
5789 let backend = mem_backend(1_048_576).await.with_hwm_key(0, [12u8; 32]);
5792 let exec = ExecutionId::new();
5793 backend
5794 .open_execution(exec, ExecutionKind::AgentTurn)
5795 .await
5796 .unwrap();
5797 backend.append(effect_intent(exec, 0)).await.unwrap();
5798 backend.append(step_result(exec, 1, b"v1")).await.unwrap();
5799
5800 let stored: (i64,) = zeph_db::query_as(sql!(
5801 "SELECT committed_result_count FROM durable_execution_integrity WHERE execution_id = ?"
5802 ))
5803 .bind(exec.as_uuid().to_string())
5804 .fetch_one(backend.pool())
5805 .await
5806 .unwrap();
5807 assert_eq!(
5808 stored.0, 1,
5809 "only the StepResult row counts, not the EffectIntent"
5810 );
5811
5812 assert!(
5813 backend
5814 .open_execution(exec, ExecutionKind::AgentTurn)
5815 .await
5816 .unwrap()
5817 );
5818 }
5819}