1use std::fmt;
36use std::fmt::Write as _;
37use std::sync::Arc;
38use std::time::{SystemTime, UNIX_EPOCH};
39
40use bytes::Bytes;
41use zeph_db::{DbPool, sql};
42
43use crate::backend::{BackendCapabilities, ExecutionBackend, ExecutionSummary, RedactedEntry};
44use crate::cipher::{EntryKindTag, PayloadAad, PayloadCipher, ensure_payload_within_limit};
45use crate::config::RetentionPolicy;
46use crate::error::DurableError;
47use crate::ids::{
48 ExecutionId, ExecutionKind, IdempotencyKey, JournalSeq, PromiseId, StepId, TimerId,
49};
50use crate::journal::{EntryKind, ExecutionStatus, Journal, JournalEntry};
51use crate::promise::PromiseRecord;
52use crate::retention::{CheckpointSnapshot, FoldedStep, decode_checkpoint, encode_checkpoint};
53use crate::waiters::NotifyRegistry;
54use tracing::Instrument as _;
55
56const SEAL_OVERHEAD_SLACK: u64 = 128;
64
65type ExecutionRow = (String, String, String, i64, i64, Option<i64>, i64);
67
68type RedactedRow = (
70 i64,
71 i64,
72 String,
73 Option<Vec<u8>>,
74 Option<String>,
75 Option<i64>,
76 i64,
77);
78
79fn idem_key_prefix(bytes: &[u8]) -> String {
81 bytes.iter().take(8).fold(String::new(), |mut acc, b| {
82 let _ = write!(acc, "{b:02x}");
83 acc
84 })
85}
86
87pub struct LocalBackend {
105 pool: DbPool,
106 cipher: Option<Arc<dyn PayloadCipher>>,
107 hmac_key: Option<[u8; 32]>,
108 max_payload_bytes: u64,
109 promise_waiters: NotifyRegistry,
111 timer_waiters: NotifyRegistry,
113}
114
115impl fmt::Debug for LocalBackend {
116 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
118 f.debug_struct("LocalBackend")
119 .field("cipher", &self.cipher.as_ref().map(|_| "<cipher>"))
120 .field("hmac_key", &self.hmac_key.as_ref().map(|_| "<redacted>"))
121 .field("max_payload_bytes", &self.max_payload_bytes)
122 .finish_non_exhaustive()
123 }
124}
125
126impl LocalBackend {
127 #[must_use]
133 pub fn new(pool: DbPool, max_payload_bytes: u64) -> Self {
134 Self {
135 pool,
136 cipher: None,
137 hmac_key: None,
138 max_payload_bytes,
139 promise_waiters: NotifyRegistry::default(),
140 timer_waiters: NotifyRegistry::default(),
141 }
142 }
143
144 pub async fn open(path: &str, max_payload_bytes: u64) -> Result<Self, DurableError> {
153 let pool = zeph_db::DbConfig {
154 url: path.to_string(),
155 max_connections: 5,
156 pool_size: 5,
157 }
158 .connect()
159 .await
160 .map_err(|e| DurableError::storage("open", e))?;
161 Ok(Self::new(pool, max_payload_bytes))
162 }
163
164 #[must_use]
166 pub fn with_cipher(mut self, cipher: Arc<dyn PayloadCipher>) -> Self {
167 self.cipher = Some(cipher);
168 self
169 }
170
171 #[must_use]
174 pub fn with_hmac_key(mut self, key: [u8; 32]) -> Self {
175 self.hmac_key = Some(key);
176 self
177 }
178
179 #[must_use]
181 pub fn pool(&self) -> &DbPool {
182 &self.pool
183 }
184
185 pub async fn init(&self) -> Result<(), DurableError> {
193 zeph_db::run_migrations(&self.pool)
194 .await
195 .map_err(|e| DurableError::storage("init", e))?;
196 Ok(())
197 }
198
199 pub async fn list_executions(
214 &self,
215 status: Option<&str>,
216 kind: Option<&str>,
217 limit: i64,
218 ) -> Result<Vec<ExecutionSummary>, DurableError> {
219 let span = tracing::info_span!(
220 "durable.backend.list",
221 status = status.unwrap_or("*"),
222 kind = kind.unwrap_or("*"),
223 count = tracing::field::Empty,
224 );
225 async move {
226 let rows: Vec<ExecutionRow> =
230 zeph_db::query_as(sql!(
231 "SELECT
232 e.execution_id,
233 e.kind,
234 e.status,
235 e.created_at,
236 e.updated_at,
237 e.finalized_at,
238 (SELECT COUNT(*) FROM durable_journal j WHERE j.execution_id = e.execution_id)
239 FROM durable_executions e
240 WHERE e.status = COALESCE(?, e.status)
241 AND e.kind = COALESCE(?, e.kind)
242 ORDER BY e.created_at DESC
243 LIMIT ?"
244 ))
245 .bind(status)
246 .bind(kind)
247 .bind(limit)
248 .fetch_all(&self.pool)
249 .await
250 .map_err(|e| DurableError::storage("list", e))?;
251 tracing::Span::current().record("count", rows.len());
252 rows.into_iter()
253 .map(|(id, kind, status, created, updated, finalized, steps)| {
254 Ok(ExecutionSummary {
255 execution_id: parse_execution_id(&id)?,
256 kind,
257 status: ExecutionStatus::from_tag(&status).ok_or(DurableError::Decode {
258 context: "execution status is not a recognized CHECK-constrained value",
259 })?,
260 created_at_ms: created,
261 updated_at_ms: updated,
262 finalized_at_ms: finalized,
263 step_count: steps.max(0).cast_unsigned(),
264 })
265 })
266 .collect()
267 }
268 .instrument(span)
269 .await
270 }
271
272 pub async fn read_execution_redacted(
285 &self,
286 id: ExecutionId,
287 ) -> Result<Vec<RedactedEntry>, DurableError> {
288 let exec = id.as_uuid().to_string();
289 let rows: Vec<RedactedRow> = zeph_db::query_as(sql!(
290 "SELECT seq, step_id, entry_kind, idem_key, effect_class, LENGTH(payload), created_at
291 FROM durable_journal WHERE execution_id = ? ORDER BY seq"
292 ))
293 .bind(&exec)
294 .fetch_all(&self.pool)
295 .await
296 .map_err(|e| DurableError::storage("read_redacted", e))?;
297 Ok(rows
298 .into_iter()
299 .map(
300 |(seq, step, entry_kind, idem, effect_class, payload_len, created)| RedactedEntry {
301 seq,
302 step_id: StepId::new(u32::try_from(step).unwrap_or(0)),
303 entry_kind,
304 effect_class,
305 idem_key_prefix: idem.as_deref().map(idem_key_prefix),
306 payload_len: payload_len.unwrap_or(0).max(0).cast_unsigned(),
307 created_at_ms: created,
308 },
309 )
310 .collect())
311 }
312
313 pub async fn count_prunable(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
322 let cutoffs = crate::retention::PruneCutoffs::from_policy(policy, now_unix_millis());
323 let (count,): (i64,) = zeph_db::query_as(sql!(
324 "SELECT COUNT(*) FROM durable_executions
325 WHERE finalized_at IS NOT NULL
326 AND ( (status = 'completed' AND finalized_at <= ?)
327 OR (status IN ('failed', 'aborted') AND finalized_at <= ?) )"
328 ))
329 .bind(cutoffs.completed_before_ms)
330 .bind(cutoffs.failed_before_ms)
331 .fetch_one(&self.pool)
332 .await
333 .map_err(|e| DurableError::storage("count_prunable", e))?;
334 Ok(count.max(0).cast_unsigned())
335 }
336
337 pub async fn open_execution(
349 &self,
350 id: ExecutionId,
351 kind: ExecutionKind,
352 ) -> Result<bool, DurableError> {
353 let span = tracing::info_span!(
354 "durable.backend.open",
355 execution_id = %id.as_uuid(),
356 kind = kind.as_str(),
357 is_resume = tracing::field::Empty,
358 );
359 async move {
360 let exec = id.as_uuid().to_string();
361 let existing: Option<(String,)> = zeph_db::query_as(sql!(
362 "SELECT status FROM durable_executions WHERE execution_id = ?"
363 ))
364 .bind(&exec)
365 .fetch_optional(&self.pool)
366 .await
367 .map_err(|e| DurableError::storage("open", e))?;
368 if existing.is_some() {
369 tracing::Span::current().record("is_resume", true);
370 return Ok(true);
371 }
372 let now = now_unix_millis();
373 zeph_db::query(sql!(
374 "INSERT INTO durable_executions
375 (execution_id, kind, status, created_at, updated_at, finalized_at)
376 VALUES (?, ?, 'running', ?, ?, NULL)"
377 ))
378 .bind(&exec)
379 .bind(kind.as_str())
380 .bind(now)
381 .bind(now)
382 .execute(&self.pool)
383 .await
384 .map_err(|e| DurableError::storage("open", e))?;
385 tracing::Span::current().record("is_resume", false);
386 Ok(false)
387 }
388 .instrument(span)
389 .await
390 }
391
392 pub(crate) async fn append_batch(&self, entries: &[JournalEntry]) -> Result<(), DurableError> {
405 if entries.is_empty() {
406 return Ok(());
407 }
408 let mut rows = Vec::with_capacity(entries.len());
409 for entry in entries {
410 rows.push(self.prepare_row(entry)?);
411 }
412 let insert = sql!(
416 "INSERT INTO durable_journal
417 (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
418 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
419 );
420 let mut tx = zeph_db::begin_write(&self.pool)
421 .await
422 .map_err(|e| DurableError::storage("append_batch", e))?;
423 for row in rows {
424 zeph_db::query(insert)
425 .bind(row.execution_id)
426 .bind(row.step_id)
427 .bind(row.entry_kind)
428 .bind(row.idem_key)
429 .bind(row.effect_class)
430 .bind(row.payload)
431 .bind(row.payload_version)
432 .bind(row.hmac)
433 .bind(row.created_at)
434 .execute(&mut *tx)
435 .await
436 .map_err(|e| DurableError::storage("append_batch", e))?;
437 }
438 tx.commit()
439 .await
440 .map_err(|e| DurableError::storage("append_batch", e))?;
441 Ok(())
442 }
443
444 pub(crate) async fn lookup_committed_result(
458 &self,
459 id: ExecutionId,
460 idem_key: IdempotencyKey,
461 ) -> Result<Option<JournalEntry>, DurableError> {
462 let span = tracing::info_span!(
463 "durable.journal.lookup_idem",
464 execution_id = %id.as_uuid(),
465 found = tracing::field::Empty,
466 );
467 async move {
468 let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
469 "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
470 FROM durable_journal
471 WHERE execution_id = ? AND idem_key = ? AND entry_kind = 'step_result'
472 ORDER BY seq LIMIT 1"
473 ))
474 .bind(id.as_uuid().to_string())
475 .bind(idem_key.as_bytes().to_vec())
476 .fetch_all(&self.pool)
477 .await
478 .map_err(|e| DurableError::storage("lookup_idem", e))?;
479 let entry = self.rows_to_entries(id, rows).await?.into_iter().next();
480 tracing::Span::current().record("found", entry.is_some());
481 Ok(entry)
482 }
483 .instrument(span)
484 .await
485 }
486
487 pub(crate) async fn max_seq(&self) -> Result<Option<JournalSeq>, DurableError> {
497 let max: Option<i64> = zeph_db::query_scalar(sql!("SELECT MAX(seq) FROM durable_journal"))
498 .fetch_one(&self.pool)
499 .await
500 .map_err(|e| DurableError::storage("max_seq", e))?;
501 Ok(max.map(JournalSeq::new))
502 }
503
504 pub(crate) fn promise_waiters(&self) -> &NotifyRegistry {
506 &self.promise_waiters
507 }
508
509 pub(crate) fn timer_waiters(&self) -> &NotifyRegistry {
511 &self.timer_waiters
512 }
513
514 pub(crate) async fn insert_promise(
523 &self,
524 id: PromiseId,
525 execution_id: ExecutionId,
526 resolver_token_hash: [u8; 32],
527 created_at_ms: i64,
528 ) -> Result<(), DurableError> {
529 let span = tracing::info_span!("durable.promise.create", promise_id = %id.as_uuid());
530 async move {
531 zeph_db::query(sql!(
532 "INSERT INTO durable_promises
533 (promise_id, execution_id, resolver_token_hash, resolved, payload, created_at, resolved_at)
534 VALUES (?, ?, ?, 0, NULL, ?, NULL)"
535 ))
536 .bind(id.as_uuid().to_string())
537 .bind(execution_id.as_uuid().to_string())
538 .bind(resolver_token_hash.to_vec())
539 .bind(created_at_ms)
540 .execute(&self.pool)
541 .await
542 .map_err(|e| DurableError::storage("insert_promise", e))?;
543 Ok(())
544 }
545 .instrument(span)
546 .await
547 }
548
549 pub(crate) async fn promise_state(
556 &self,
557 id: PromiseId,
558 ) -> Result<Option<PromiseRecord>, DurableError> {
559 let row: Option<PromiseRowRead> = zeph_db::query_as(sql!(
560 "SELECT execution_id, resolver_token_hash, resolved, payload
561 FROM durable_promises WHERE promise_id = ?"
562 ))
563 .bind(id.as_uuid().to_string())
564 .fetch_optional(&self.pool)
565 .await
566 .map_err(|e| DurableError::storage("promise_state", e))?;
567 let Some((exec, hash, resolved, payload)) = row else {
568 return Ok(None);
569 };
570 Ok(Some(PromiseRecord {
571 execution_id: parse_execution_id(&exec)?,
572 resolver_token_hash: slice_to_array32(&hash, "promise resolver_token_hash")?,
573 resolved: resolved != 0,
574 payload,
575 }))
576 }
577
578 pub(crate) async fn resolve_promise(
589 &self,
590 id: PromiseId,
591 execution_id: ExecutionId,
592 value_plaintext: &[u8],
593 resolved_at_ms: i64,
594 ) -> Result<bool, DurableError> {
595 let span = tracing::info_span!("durable.promise.resolve", promise_id = %id.as_uuid());
596 async move {
597 ensure_payload_within_limit(value_plaintext.len(), self.max_payload_bytes)?;
598 let aad = promise_payload_aad(execution_id, id);
599 let sealed = self.seal_payload(value_plaintext, &aad)?;
600 let affected = zeph_db::query(sql!(
601 "UPDATE durable_promises SET resolved = 1, payload = ?, resolved_at = ?
602 WHERE promise_id = ? AND resolved = 0"
603 ))
604 .bind(sealed)
605 .bind(resolved_at_ms)
606 .bind(id.as_uuid().to_string())
607 .execute(&self.pool)
608 .await
609 .map_err(|e| DurableError::storage("resolve_promise", e))?
610 .rows_affected();
611 if affected > 0 {
612 self.promise_waiters.wake(id.as_uuid());
613 }
614 Ok(affected > 0)
615 }
616 .instrument(span)
617 .await
618 }
619
620 pub(crate) fn open_promise_payload(
627 &self,
628 id: PromiseId,
629 execution_id: ExecutionId,
630 sealed: &[u8],
631 ) -> Result<Bytes, DurableError> {
632 ensure_payload_within_limit(
633 sealed.len(),
634 self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
635 )?;
636 let aad = promise_payload_aad(execution_id, id);
637 self.open_payload(sealed, &aad)
638 }
639
640 pub(crate) async fn arm_timer(
648 &self,
649 id: TimerId,
650 execution_id: ExecutionId,
651 due_at_ms: i64,
652 created_at_ms: i64,
653 ) -> Result<(), DurableError> {
654 let span = tracing::info_span!("durable.timer.arm", timer_id = %id.as_uuid(), due_at_ms);
655 async move {
656 zeph_db::query(sql!(
657 "INSERT INTO durable_timers (timer_id, execution_id, due_at, fired, created_at)
658 VALUES (?, ?, ?, 0, ?)"
659 ))
660 .bind(id.as_uuid().to_string())
661 .bind(execution_id.as_uuid().to_string())
662 .bind(due_at_ms)
663 .bind(created_at_ms)
664 .execute(&self.pool)
665 .await
666 .map_err(|e| DurableError::storage("arm_timer", e))?;
667 Ok(())
668 }
669 .instrument(span)
670 .await
671 }
672
673 pub(crate) async fn timer_state(
679 &self,
680 id: TimerId,
681 ) -> Result<Option<(i64, bool)>, DurableError> {
682 let row: Option<(i64, i64)> = zeph_db::query_as(sql!(
683 "SELECT due_at, fired FROM durable_timers WHERE timer_id = ?"
684 ))
685 .bind(id.as_uuid().to_string())
686 .fetch_optional(&self.pool)
687 .await
688 .map_err(|e| DurableError::storage("timer_state", e))?;
689 Ok(row.map(|(due_at, fired)| (due_at, fired != 0)))
690 }
691
692 pub(crate) async fn due_timers(&self, now_ms: i64) -> Result<Vec<TimerId>, DurableError> {
702 let rows: Vec<(String,)> = zeph_db::query_as(sql!(
703 "SELECT timer_id FROM durable_timers WHERE fired = 0 AND due_at <= ? ORDER BY due_at"
704 ))
705 .bind(now_ms)
706 .fetch_all(&self.pool)
707 .await
708 .map_err(|e| DurableError::storage("due_timers", e))?;
709 rows.into_iter().map(|(id,)| parse_timer_id(&id)).collect()
710 }
711
712 pub(crate) async fn mark_timer_fired(&self, id: TimerId) -> Result<bool, DurableError> {
720 let span = tracing::info_span!("durable.timer.fire", timer_id = %id.as_uuid());
721 async move {
722 let affected = zeph_db::query(sql!(
723 "UPDATE durable_timers SET fired = 1 WHERE timer_id = ? AND fired = 0"
724 ))
725 .bind(id.as_uuid().to_string())
726 .execute(&self.pool)
727 .await
728 .map_err(|e| DurableError::storage("mark_timer_fired", e))?
729 .rows_affected();
730 if affected > 0 {
731 self.timer_waiters.wake(id.as_uuid());
732 }
733 Ok(affected > 0)
734 }
735 .instrument(span)
736 .await
737 }
738
739 fn open_foldable_steps(
745 &self,
746 execution_id: ExecutionId,
747 rows: Vec<FoldableRowRead>,
748 ) -> Result<Vec<FoldedStep>, DurableError> {
749 let mut folded = Vec::with_capacity(rows.len());
750 for (step_raw, idem, version, payload) in rows {
751 let step = u32::try_from(step_raw).map_err(|_| DurableError::Decode {
752 context: "checkpoint step_id out of u32 range",
753 })?;
754 let idem_bytes = idem.ok_or(DurableError::Decode {
755 context: "checkpoint step result missing idem_key",
756 })?;
757 let idem_key =
758 IdempotencyKey::from_bytes(slice_to_array32(&idem_bytes, "checkpoint idem_key")?);
759 let sealed = payload.ok_or(DurableError::Decode {
760 context: "checkpoint step result missing payload",
761 })?;
762 let aad = PayloadAad::new(
763 execution_id,
764 StepId::new(step),
765 EntryKindTag::StepResult,
766 Some(idem_key),
767 );
768 let plaintext = self.open_payload(&sealed, &aad)?;
769 let payload_version =
770 u8::try_from(version.unwrap_or(1)).map_err(|_| DurableError::Decode {
771 context: "checkpoint payload_version out of u8 range",
772 })?;
773 folded.push(FoldedStep {
774 step_id: step,
775 idem_key: *idem_key.as_bytes(),
776 payload_version,
777 payload: plaintext,
778 });
779 }
780 Ok(folded)
781 }
782
783 pub(crate) async fn checkpoint_fold(
797 &self,
798 execution_id: ExecutionId,
799 up_to_step: u32,
800 ) -> Result<u64, DurableError> {
801 let span = tracing::info_span!(
802 "durable.journal.checkpoint",
803 execution_id = %execution_id.as_uuid(),
804 folded_count = tracing::field::Empty,
805 );
806 async move {
807 let exec = execution_id.as_uuid().to_string();
808 let rows: Vec<FoldableRowRead> = zeph_db::query_as(sql!(
809 "SELECT step_id, idem_key, payload_version, payload FROM durable_journal
810 WHERE execution_id = ? AND entry_kind = 'step_result'
811 AND effect_class = 'idempotent' AND step_id < ?
812 ORDER BY step_id"
813 ))
814 .bind(&exec)
815 .bind(i64::from(up_to_step))
816 .fetch_all(&self.pool)
817 .await
818 .map_err(|e| DurableError::storage("checkpoint", e))?;
819 if rows.is_empty() {
820 return Ok(0);
821 }
822
823 let mut folded = self.open_foldable_steps(execution_id, rows)?;
825 let lens: Vec<usize> = folded.iter().map(|s| s.payload.len()).collect();
826 let take = crate::retention::fold_prefix_len(
827 &lens,
828 crate::retention::checkpoint_budget(self.max_payload_bytes),
829 );
830 if take == 0 {
831 return Ok(0);
834 }
835 folded.truncate(take);
836 let fold_end = folded.last().map_or(up_to_step, |s| s.step_id.saturating_add(1));
837
838 let snapshot = encode_checkpoint(&folded);
839 let snap_aad =
840 PayloadAad::new(execution_id, StepId::new(fold_end), EntryKindTag::Checkpoint, None);
841 let sealed_snapshot = self.seal_payload(&snapshot, &snap_aad)?;
842
843 let mut tx = zeph_db::begin_write(&self.pool)
844 .await
845 .map_err(|e| DurableError::storage("checkpoint", e))?;
846 zeph_db::query(sql!(
847 "INSERT INTO durable_journal
848 (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
849 VALUES (?, ?, 'checkpoint', NULL, NULL, ?, ?, NULL, ?)"
850 ))
851 .bind(&exec)
852 .bind(i64::from(fold_end))
853 .bind(sealed_snapshot)
854 .bind(i32::from(crate::step::PAYLOAD_VERSION))
855 .bind(now_unix_millis())
856 .execute(&mut *tx)
857 .await
858 .map_err(|e| DurableError::storage("checkpoint", e))?;
859 zeph_db::query(sql!(
860 "DELETE FROM durable_journal
861 WHERE execution_id = ? AND entry_kind = 'step_result'
862 AND effect_class = 'idempotent' AND step_id < ?"
863 ))
864 .bind(&exec)
865 .bind(i64::from(fold_end))
866 .execute(&mut *tx)
867 .await
868 .map_err(|e| DurableError::storage("checkpoint", e))?;
869 tx.commit()
870 .await
871 .map_err(|e| DurableError::storage("checkpoint", e))?;
872
873 let count = folded.len() as u64;
874 tracing::Span::current().record("folded_count", count);
875 Ok(count)
876 }
877 .instrument(span)
878 .await
879 }
880
881 pub(crate) async fn read_checkpoints(
894 &self,
895 execution_id: ExecutionId,
896 ) -> Result<Vec<JournalEntry>, DurableError> {
897 let rows: Vec<(i64, Option<Vec<u8>>)> = zeph_db::query_as(sql!(
898 "SELECT step_id, payload FROM durable_journal
899 WHERE execution_id = ? AND entry_kind = 'checkpoint' ORDER BY step_id"
900 ))
901 .bind(execution_id.as_uuid().to_string())
902 .fetch_all(&self.pool)
903 .await
904 .map_err(|e| DurableError::storage("read_checkpoints", e))?;
905 if rows.is_empty() {
906 return Ok(Vec::new());
907 }
908 let mut folded: CheckpointSnapshot = Vec::new();
909 for (up_to, payload) in rows {
910 let up_to = u32::try_from(up_to).map_err(|_| DurableError::Decode {
911 context: "checkpoint up_to_step out of u32 range",
912 })?;
913 let sealed = payload.ok_or(DurableError::Decode {
914 context: "checkpoint entry missing snapshot payload",
915 })?;
916 ensure_payload_within_limit(
917 sealed.len(),
918 self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
919 )?;
920 let aad = PayloadAad::new(
921 execution_id,
922 StepId::new(up_to),
923 EntryKindTag::Checkpoint,
924 None,
925 );
926 let plaintext = self.open_payload(&sealed, &aad)?;
927 folded.extend(decode_checkpoint(&plaintext)?);
928 }
929 let kind = self.lookup_kind(execution_id).await?;
932 let entries = folded
933 .into_iter()
934 .map(|step| JournalEntry {
935 seq: None,
936 execution_id,
937 kind,
938 step_id: StepId::new(step.step_id),
939 entry: EntryKind::StepResult {
940 idempotency_key: IdempotencyKey::from_bytes(step.idem_key),
941 payload: step.payload,
942 effect: crate::EffectClass::Idempotent,
943 payload_version: step.payload_version,
944 },
945 created_at_ms: 0,
946 })
947 .collect();
948 Ok(entries)
949 }
950
951 async fn delete_prune_batch(
958 &self,
959 cutoffs: crate::retention::PruneCutoffs,
960 batch: u64,
961 ) -> Result<u64, DurableError> {
962 let ids: Vec<(String,)> = zeph_db::query_as(sql!(
963 "SELECT execution_id FROM durable_executions
964 WHERE finalized_at IS NOT NULL
965 AND ( (status = 'completed' AND finalized_at <= ?)
966 OR (status IN ('failed', 'aborted') AND finalized_at <= ?) )
967 ORDER BY finalized_at LIMIT ?"
968 ))
969 .bind(cutoffs.completed_before_ms)
970 .bind(cutoffs.failed_before_ms)
971 .bind(i64::try_from(batch).unwrap_or(i64::MAX))
972 .fetch_all(&self.pool)
973 .await
974 .map_err(|e| DurableError::storage("prune", e))?;
975 if ids.is_empty() {
976 return Ok(0);
977 }
978 let journal = sql!("DELETE FROM durable_journal WHERE execution_id = ?");
979 let promises = sql!("DELETE FROM durable_promises WHERE execution_id = ?");
980 let timers = sql!("DELETE FROM durable_timers WHERE execution_id = ?");
981 let executions = sql!("DELETE FROM durable_executions WHERE execution_id = ?");
982 let mut tx = zeph_db::begin_write(&self.pool)
983 .await
984 .map_err(|e| DurableError::storage("prune", e))?;
985 for (id,) in &ids {
986 for stmt in [journal, promises, timers, executions] {
987 zeph_db::query(stmt)
988 .bind(id)
989 .execute(&mut *tx)
990 .await
991 .map_err(|e| DurableError::storage("prune", e))?;
992 }
993 }
994 tx.commit()
995 .await
996 .map_err(|e| DurableError::storage("prune", e))?;
997 Ok(ids.len() as u64)
998 }
999
1000 fn seal_payload(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, DurableError> {
1002 match &self.cipher {
1003 Some(cipher) => Ok(cipher.seal(plaintext, aad)?),
1004 None => Ok(plaintext.to_vec()),
1005 }
1006 }
1007
1008 fn open_payload(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Bytes, DurableError> {
1010 match &self.cipher {
1011 Some(cipher) => Ok(Bytes::from(cipher.open(sealed, aad)?)),
1012 None => Ok(Bytes::copy_from_slice(sealed)),
1013 }
1014 }
1015
1016 fn control_hmac(
1021 &self,
1022 entry: &JournalEntry,
1023 idem_key: Option<&IdempotencyKey>,
1024 ) -> Option<Vec<u8>> {
1025 let key = self.hmac_key.as_ref()?;
1026 let mut input = Vec::with_capacity(16 + 4 + 16 + 32);
1027 input.extend_from_slice(entry.execution_id.as_bytes());
1028 input.extend_from_slice(&entry.step_id.value().to_le_bytes());
1029 input.extend_from_slice(entry.entry.tag().as_bytes());
1030 if let Some(k) = idem_key {
1031 input.extend_from_slice(k.as_bytes());
1032 }
1033 Some(blake3::keyed_hash(key, &input).as_bytes().to_vec())
1034 }
1035
1036 fn prepare_row(&self, entry: &JournalEntry) -> Result<JournalRow, DurableError> {
1038 let execution_id = entry.execution_id.as_uuid().to_string();
1039 let step_id = i64::from(entry.step_id.value());
1040 let created_at = entry.created_at_ms;
1041 let entry_kind = entry.entry.tag();
1042 match &entry.entry {
1043 EntryKind::StepResult {
1044 idempotency_key,
1045 payload,
1046 effect,
1047 payload_version,
1048 } => {
1049 ensure_payload_within_limit(payload.len(), self.max_payload_bytes)?;
1050 let aad = PayloadAad::new(
1051 entry.execution_id,
1052 entry.step_id,
1053 EntryKindTag::StepResult,
1054 Some(*idempotency_key),
1055 );
1056 let sealed = self.seal_payload(payload.as_ref(), &aad)?;
1057 Ok(JournalRow {
1058 execution_id,
1059 step_id,
1060 entry_kind,
1061 idem_key: Some(idempotency_key.as_bytes().to_vec()),
1062 effect_class: Some(effect.as_str()),
1063 payload: Some(sealed),
1064 payload_version: Some(i32::from(*payload_version)),
1065 hmac: None,
1066 created_at,
1067 })
1068 }
1069 EntryKind::EffectIntent {
1070 idempotency_key,
1071 effect,
1072 hmac: _,
1073 } => {
1074 let hmac = self.control_hmac(entry, Some(idempotency_key));
1077 Ok(JournalRow {
1078 execution_id,
1079 step_id,
1080 entry_kind,
1081 idem_key: Some(idempotency_key.as_bytes().to_vec()),
1082 effect_class: Some(effect.as_str()),
1083 payload: None,
1084 payload_version: None,
1085 hmac,
1086 created_at,
1087 })
1088 }
1089 EntryKind::PromiseCreated { .. }
1090 | EntryKind::PromiseResolved { .. }
1091 | EntryKind::TimerArmed { .. }
1092 | EntryKind::TimerFired { .. }
1093 | EntryKind::Checkpoint { .. } => {
1094 Err(DurableError::UnsupportedEntryKind { kind: entry_kind })
1095 }
1096 }
1097 }
1098
1099 async fn lookup_kind(&self, id: ExecutionId) -> Result<ExecutionKind, DurableError> {
1101 let kind: Option<String> = zeph_db::query_scalar(sql!(
1102 "SELECT kind FROM durable_executions WHERE execution_id = ?"
1103 ))
1104 .bind(id.as_uuid().to_string())
1105 .fetch_optional(&self.pool)
1106 .await
1107 .map_err(|e| DurableError::storage("read", e))?;
1108 let kind = kind.ok_or(DurableError::Decode {
1109 context: "journaled entries reference a missing execution row",
1110 })?;
1111 ExecutionKind::from_tag(&kind).ok_or(DurableError::Decode {
1112 context: "execution kind is not reconstructible (custom kind read-back unsupported)",
1113 })
1114 }
1115
1116 fn row_to_entry(
1118 &self,
1119 id: ExecutionId,
1120 kind: ExecutionKind,
1121 row: JournalRowRead,
1122 ) -> Result<JournalEntry, DurableError> {
1123 let (
1124 seq,
1125 step_id_raw,
1126 entry_kind,
1127 idem_key,
1128 effect_class,
1129 payload,
1130 payload_version,
1131 hmac,
1132 created_at,
1133 ) = row;
1134 let step_id =
1135 StepId::new(
1136 u32::try_from(step_id_raw).map_err(|_| DurableError::Decode {
1137 context: "step_id out of u32 range",
1138 })?,
1139 );
1140 let entry = match entry_kind.as_str() {
1141 "step_result" => {
1142 let idem_bytes = idem_key.ok_or(DurableError::Decode {
1143 context: "step_result idem_key missing",
1144 })?;
1145 let idem_key = IdempotencyKey::from_bytes(slice_to_array32(
1146 &idem_bytes,
1147 "step_result idem_key",
1148 )?);
1149 let effect = effect_class
1150 .as_deref()
1151 .and_then(crate::EffectClass::from_tag)
1152 .ok_or(DurableError::Decode {
1153 context: "step_result effect_class missing or invalid",
1154 })?;
1155 let sealed = payload.ok_or(DurableError::Decode {
1156 context: "step_result payload missing",
1157 })?;
1158 ensure_payload_within_limit(
1159 sealed.len(),
1160 self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1161 )?;
1162 let aad = PayloadAad::new(id, step_id, EntryKindTag::StepResult, Some(idem_key));
1163 let opened = self.open_payload(&sealed, &aad)?;
1164 let version = u8::try_from(payload_version.unwrap_or(1)).map_err(|_| {
1165 DurableError::Decode {
1166 context: "payload_version out of u8 range",
1167 }
1168 })?;
1169 EntryKind::StepResult {
1170 idempotency_key: idem_key,
1171 payload: opened,
1172 effect,
1173 payload_version: version,
1174 }
1175 }
1176 "effect_intent" => {
1177 let idem_bytes = idem_key.ok_or(DurableError::Decode {
1178 context: "effect_intent idem_key missing",
1179 })?;
1180 let idem_key = IdempotencyKey::from_bytes(slice_to_array32(
1181 &idem_bytes,
1182 "effect_intent idem_key",
1183 )?);
1184 let effect = effect_class
1185 .as_deref()
1186 .and_then(crate::EffectClass::from_tag)
1187 .ok_or(DurableError::Decode {
1188 context: "effect_intent effect_class missing or invalid",
1189 })?;
1190 let hmac = hmac
1191 .map(|bytes| slice_to_array32(&bytes, "effect_intent hmac"))
1192 .transpose()?;
1193 EntryKind::EffectIntent {
1194 idempotency_key: idem_key,
1195 effect,
1196 hmac,
1197 }
1198 }
1199 "checkpoint" => self.checkpoint_entry(id, step_id, payload)?,
1200 other => {
1201 return Err(DurableError::UnsupportedEntryKind {
1202 kind: static_entry_tag(other),
1203 });
1204 }
1205 };
1206 Ok(JournalEntry {
1207 seq: Some(JournalSeq::new(seq)),
1208 execution_id: id,
1209 kind,
1210 step_id,
1211 entry,
1212 created_at_ms: created_at,
1213 })
1214 }
1215
1216 fn checkpoint_entry(
1221 &self,
1222 id: ExecutionId,
1223 step_id: StepId,
1224 payload: Option<Vec<u8>>,
1225 ) -> Result<EntryKind, DurableError> {
1226 let sealed = payload.ok_or(DurableError::Decode {
1227 context: "checkpoint entry missing snapshot payload",
1228 })?;
1229 ensure_payload_within_limit(
1230 sealed.len(),
1231 self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1232 )?;
1233 let aad = PayloadAad::new(id, step_id, EntryKindTag::Checkpoint, None);
1234 let snapshot = self.open_payload(&sealed, &aad)?;
1235 Ok(EntryKind::Checkpoint {
1236 up_to_step: step_id.value(),
1237 snapshot,
1238 })
1239 }
1240
1241 async fn rows_to_entries(
1243 &self,
1244 id: ExecutionId,
1245 rows: Vec<JournalRowRead>,
1246 ) -> Result<Vec<JournalEntry>, DurableError> {
1247 if rows.is_empty() {
1248 return Ok(Vec::new());
1249 }
1250 let kind = self.lookup_kind(id).await?;
1251 let mut entries = Vec::with_capacity(rows.len());
1252 for row in rows {
1253 entries.push(self.row_to_entry(id, kind, row)?);
1254 }
1255 Ok(entries)
1256 }
1257}
1258
1259impl Journal for LocalBackend {
1260 async fn append(&self, entry: JournalEntry) -> Result<JournalSeq, DurableError> {
1261 let span = tracing::info_span!(
1262 "durable.journal.append",
1263 execution_id = %entry.execution_id.as_uuid(),
1264 step_id = entry.step_id.value(),
1265 entry_kind = entry.entry.tag(),
1266 );
1267 async move {
1268 let row = self.prepare_row(&entry)?;
1269 let (seq,): (i64,) = zeph_db::query_as(sql!(
1270 "INSERT INTO durable_journal
1271 (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
1272 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
1273 RETURNING seq"
1274 ))
1275 .bind(row.execution_id)
1276 .bind(row.step_id)
1277 .bind(row.entry_kind)
1278 .bind(row.idem_key)
1279 .bind(row.effect_class)
1280 .bind(row.payload)
1281 .bind(row.payload_version)
1282 .bind(row.hmac)
1283 .bind(row.created_at)
1284 .fetch_one(&self.pool)
1285 .await
1286 .map_err(|e| DurableError::storage("append", e))?;
1287 Ok(JournalSeq::new(seq))
1288 }
1289 .instrument(span)
1290 .await
1291 }
1292
1293 async fn read_execution(&self, id: ExecutionId) -> Result<Vec<JournalEntry>, DurableError> {
1294 let span = tracing::info_span!(
1295 "durable.journal.read",
1296 execution_id = %id.as_uuid(),
1297 step_count = tracing::field::Empty,
1298 );
1299 async move {
1300 let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
1301 "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
1302 FROM durable_journal WHERE execution_id = ? ORDER BY seq"
1303 ))
1304 .bind(id.as_uuid().to_string())
1305 .fetch_all(&self.pool)
1306 .await
1307 .map_err(|e| DurableError::storage("read", e))?;
1308 let entries = self.rows_to_entries(id, rows).await?;
1309 tracing::Span::current().record("step_count", entries.len());
1310 Ok(entries)
1311 }
1312 .instrument(span)
1313 .await
1314 }
1315
1316 async fn read_execution_range(
1317 &self,
1318 id: ExecutionId,
1319 from_step_id: u32,
1320 limit: usize,
1321 ) -> Result<Vec<JournalEntry>, DurableError> {
1322 let span = tracing::info_span!(
1323 "durable.journal.read_segment",
1324 execution_id = %id.as_uuid(),
1325 from_step_id,
1326 count = tracing::field::Empty,
1327 );
1328 async move {
1329 let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
1330 "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
1331 FROM durable_journal WHERE execution_id = ? AND step_id >= ? ORDER BY step_id, seq LIMIT ?"
1332 ))
1333 .bind(id.as_uuid().to_string())
1334 .bind(i64::from(from_step_id))
1335 .bind(i64::try_from(limit).unwrap_or(i64::MAX))
1336 .fetch_all(&self.pool)
1337 .await
1338 .map_err(|e| DurableError::storage("read_segment", e))?;
1339 let entries = self.rows_to_entries(id, rows).await?;
1340 tracing::Span::current().record("count", entries.len());
1341 Ok(entries)
1342 }
1343 .instrument(span)
1344 .await
1345 }
1346
1347 async fn finalize(&self, id: ExecutionId, status: ExecutionStatus) -> Result<(), DurableError> {
1348 let span = tracing::info_span!(
1349 "durable.journal.finalize",
1350 execution_id = %id.as_uuid(),
1351 status = status.as_str(),
1352 );
1353 async move {
1354 let now = now_unix_millis();
1355 let finalized_at = (!status.is_running()).then_some(now);
1356 let mut tx = zeph_db::begin_write(&self.pool)
1357 .await
1358 .map_err(|e| DurableError::storage("finalize", e))?;
1359 zeph_db::query(sql!(
1360 "UPDATE durable_executions SET status = ?, updated_at = ?, finalized_at = ?
1361 WHERE execution_id = ?"
1362 ))
1363 .bind(status.as_str())
1364 .bind(now)
1365 .bind(finalized_at)
1366 .bind(id.as_uuid().to_string())
1367 .execute(&mut *tx)
1368 .await
1369 .map_err(|e| DurableError::storage("finalize", e))?;
1370 tx.commit()
1371 .await
1372 .map_err(|e| DurableError::storage("finalize", e))?;
1373 Ok(())
1374 }
1375 .instrument(span)
1376 .await
1377 }
1378
1379 async fn prune(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
1380 let now = now_unix_millis();
1381 crate::retention::prune_in_batches(policy, now, |cutoffs, batch| {
1382 self.delete_prune_batch(cutoffs, batch)
1383 })
1384 .await
1385 }
1386}
1387
1388impl crate::sealed::Sealed for LocalBackend {}
1389
1390impl ExecutionBackend for LocalBackend {
1391 fn capabilities(&self) -> BackendCapabilities {
1392 BackendCapabilities {
1393 parallel_steps: true,
1394 cross_process: cfg!(feature = "postgres"),
1396 max_payload: usize::try_from(self.max_payload_bytes).unwrap_or(usize::MAX),
1397 }
1398 }
1399
1400 async fn lookup_committed_result(
1401 &self,
1402 id: ExecutionId,
1403 idem_key: IdempotencyKey,
1404 ) -> Result<Option<JournalEntry>, DurableError> {
1405 LocalBackend::lookup_committed_result(self, id, idem_key).await
1406 }
1407}
1408
1409struct JournalRow {
1411 execution_id: String,
1412 step_id: i64,
1413 entry_kind: &'static str,
1414 idem_key: Option<Vec<u8>>,
1415 effect_class: Option<&'static str>,
1416 payload: Option<Vec<u8>>,
1417 payload_version: Option<i32>,
1418 hmac: Option<Vec<u8>>,
1419 created_at: i64,
1420}
1421
1422type JournalRowRead = (
1430 i64,
1431 i64,
1432 String,
1433 Option<Vec<u8>>,
1434 Option<String>,
1435 Option<Vec<u8>>,
1436 Option<i32>,
1437 Option<Vec<u8>>,
1438 i64,
1439);
1440
1441type PromiseRowRead = (String, Vec<u8>, i64, Option<Vec<u8>>);
1444
1445type FoldableRowRead = (i64, Option<Vec<u8>>, Option<i32>, Option<Vec<u8>>);
1448
1449pub(crate) fn now_unix_millis() -> i64 {
1451 SystemTime::now()
1452 .duration_since(UNIX_EPOCH)
1453 .map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX))
1454}
1455
1456fn slice_to_array32(bytes: &[u8], field: &'static str) -> Result<[u8; 32], DurableError> {
1458 <[u8; 32]>::try_from(bytes).map_err(|_| DurableError::Decode { context: field })
1459}
1460
1461fn parse_execution_id(text: &str) -> Result<ExecutionId, DurableError> {
1463 uuid::Uuid::parse_str(text)
1464 .map(ExecutionId::from_uuid)
1465 .map_err(|_| DurableError::Decode {
1466 context: "execution_id is not a valid UUID",
1467 })
1468}
1469
1470fn parse_timer_id(text: &str) -> Result<TimerId, DurableError> {
1472 uuid::Uuid::parse_str(text)
1473 .map(TimerId::from_uuid)
1474 .map_err(|_| DurableError::Decode {
1475 context: "timer_id is not a valid UUID",
1476 })
1477}
1478
1479fn promise_payload_aad(execution_id: ExecutionId, promise_id: PromiseId) -> PayloadAad {
1484 let binding = IdempotencyKey::derive(
1485 execution_id,
1486 StepId::new(0),
1487 promise_id.as_uuid().as_bytes(),
1488 );
1489 PayloadAad::new(
1490 execution_id,
1491 StepId::new(0),
1492 EntryKindTag::PromiseResolved,
1493 Some(binding),
1494 )
1495}
1496
1497fn static_entry_tag(tag: &str) -> &'static str {
1499 match tag {
1500 "promise_created" => "promise_created",
1501 "promise_resolved" => "promise_resolved",
1502 "timer_armed" => "timer_armed",
1503 "timer_fired" => "timer_fired",
1504 "checkpoint" => "checkpoint",
1505 _ => "unknown",
1506 }
1507}
1508
1509#[cfg(all(test, feature = "sqlite"))]
1514mod tests {
1515 use std::assert_matches;
1516
1517 use super::*;
1518 use crate::cipher::CipherError;
1519 use crate::effect::EffectClass;
1520
1521 struct XorCipher;
1524 const XOR_MASK: u8 = 0x5A;
1525
1526 impl PayloadCipher for XorCipher {
1527 fn seal(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
1528 let tag = blake3::hash(&aad.canonical_bytes());
1529 let mut out = tag.as_bytes()[..8].to_vec();
1530 out.extend(plaintext.iter().map(|b| b ^ XOR_MASK));
1531 Ok(out)
1532 }
1533
1534 fn open(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
1535 if sealed.len() < 8 {
1536 return Err(CipherError::Malformed {
1537 context: "sealed blob shorter than the aad tag",
1538 });
1539 }
1540 let expected = blake3::hash(&aad.canonical_bytes());
1541 if sealed[..8] != expected.as_bytes()[..8] {
1542 return Err(CipherError::Authentication);
1543 }
1544 Ok(sealed[8..].iter().map(|b| b ^ XOR_MASK).collect())
1545 }
1546 }
1547
1548 async fn mem_backend(max_payload_bytes: u64) -> LocalBackend {
1549 let backend = LocalBackend::open(":memory:", max_payload_bytes)
1550 .await
1551 .expect("open in-memory backend");
1552 backend.init().await.expect("apply migrations");
1553 backend
1554 }
1555
1556 fn step_result(exec: ExecutionId, step: u32, payload: &[u8]) -> JournalEntry {
1557 let step_id = StepId::new(step);
1558 JournalEntry {
1559 seq: None,
1560 execution_id: exec,
1561 kind: ExecutionKind::AgentTurn,
1562 step_id,
1563 entry: EntryKind::StepResult {
1564 idempotency_key: IdempotencyKey::derive(exec, step_id, b"tool:read"),
1565 payload: Bytes::copy_from_slice(payload),
1566 effect: EffectClass::Idempotent,
1567 payload_version: 1,
1568 },
1569 created_at_ms: 100,
1570 }
1571 }
1572
1573 fn effect_intent(exec: ExecutionId, step: u32) -> JournalEntry {
1574 let step_id = StepId::new(step);
1575 JournalEntry {
1576 seq: None,
1577 execution_id: exec,
1578 kind: ExecutionKind::AgentTurn,
1579 step_id,
1580 entry: EntryKind::EffectIntent {
1581 idempotency_key: IdempotencyKey::derive(exec, step_id, b"transfer"),
1582 effect: EffectClass::ExactlyOnceGuarded,
1583 hmac: None,
1584 },
1585 created_at_ms: 100,
1586 }
1587 }
1588
1589 #[tokio::test]
1590 async fn open_execution_is_fresh_then_resume() {
1591 let backend = mem_backend(1_048_576).await;
1592 let exec = ExecutionId::new();
1593 assert!(
1594 !backend
1595 .open_execution(exec, ExecutionKind::AgentTurn)
1596 .await
1597 .unwrap()
1598 );
1599 assert!(
1600 backend
1601 .open_execution(exec, ExecutionKind::AgentTurn)
1602 .await
1603 .unwrap()
1604 );
1605 }
1606
1607 #[tokio::test]
1608 async fn list_executions_summarizes_and_filters() {
1609 let backend = mem_backend(1_048_576).await;
1610 let turn = ExecutionId::new();
1611 let dag = ExecutionId::new();
1612 backend
1613 .open_execution(turn, ExecutionKind::AgentTurn)
1614 .await
1615 .unwrap();
1616 backend
1617 .open_execution(dag, ExecutionKind::DagRun)
1618 .await
1619 .unwrap();
1620 backend.append(step_result(turn, 0, b"a")).await.unwrap();
1621 backend.append(step_result(turn, 1, b"b")).await.unwrap();
1622 backend.append(step_result(dag, 0, b"c")).await.unwrap();
1623 backend
1624 .finalize(turn, ExecutionStatus::Completed)
1625 .await
1626 .unwrap();
1627
1628 let all = backend.list_executions(None, None, 10).await.unwrap();
1630 assert_eq!(all.len(), 2);
1631
1632 let turn_row = all
1633 .iter()
1634 .find(|e| e.execution_id == turn)
1635 .expect("turn present");
1636 assert_eq!(turn_row.kind, "agent_turn");
1637 assert_eq!(turn_row.status, ExecutionStatus::Completed);
1638 assert_eq!(turn_row.step_count, 2);
1639 assert!(turn_row.finalized_at_ms.is_some());
1640
1641 let dag_row = all
1642 .iter()
1643 .find(|e| e.execution_id == dag)
1644 .expect("dag present");
1645 assert_eq!(dag_row.status, ExecutionStatus::Running);
1646 assert_eq!(dag_row.step_count, 1);
1647 assert!(dag_row.finalized_at_ms.is_none());
1648
1649 let running = backend
1651 .list_executions(Some("running"), None, 10)
1652 .await
1653 .unwrap();
1654 assert_eq!(running.len(), 1);
1655 assert_eq!(running[0].execution_id, dag);
1656
1657 let dags = backend
1659 .list_executions(None, Some("dag_run"), 10)
1660 .await
1661 .unwrap();
1662 assert_eq!(dags.len(), 1);
1663 assert_eq!(dags[0].execution_id, dag);
1664
1665 let one = backend.list_executions(None, None, 1).await.unwrap();
1667 assert_eq!(one.len(), 1);
1668 }
1669
1670 #[tokio::test]
1671 async fn append_and_read_round_trips_step_result() {
1672 let backend = mem_backend(1_048_576).await;
1673 let exec = ExecutionId::new();
1674 backend
1675 .open_execution(exec, ExecutionKind::AgentTurn)
1676 .await
1677 .unwrap();
1678
1679 let seq = backend
1680 .append(step_result(exec, 0, b"hello"))
1681 .await
1682 .unwrap();
1683 assert_eq!(seq.value(), 1, "first append takes seq 1");
1684
1685 let entries = backend.read_execution(exec).await.unwrap();
1686 assert_eq!(entries.len(), 1);
1687 match &entries[0].entry {
1688 EntryKind::StepResult {
1689 payload, effect, ..
1690 } => {
1691 assert_eq!(payload.as_ref(), b"hello");
1692 assert_eq!(*effect, EffectClass::Idempotent);
1693 }
1694 other => panic!("unexpected entry kind: {other:?}"),
1695 }
1696 assert_eq!(entries[0].seq, Some(seq));
1697 }
1698
1699 #[tokio::test]
1700 async fn cipher_seals_payload_at_rest_but_round_trips() {
1701 let backend = mem_backend(1_048_576)
1702 .await
1703 .with_cipher(Arc::new(XorCipher));
1704 let exec = ExecutionId::new();
1705 backend
1706 .open_execution(exec, ExecutionKind::AgentTurn)
1707 .await
1708 .unwrap();
1709 backend
1710 .append(step_result(exec, 0, b"secret-payload"))
1711 .await
1712 .unwrap();
1713
1714 let (stored,): (Option<Vec<u8>>,) = zeph_db::query_as(sql!(
1716 "SELECT payload FROM durable_journal WHERE execution_id = ?"
1717 ))
1718 .bind(exec.as_uuid().to_string())
1719 .fetch_one(backend.pool())
1720 .await
1721 .unwrap();
1722 let stored = stored.expect("payload present");
1723 assert_ne!(
1724 stored.as_slice(),
1725 b"secret-payload",
1726 "payload must be sealed at rest"
1727 );
1728
1729 let entries = backend.read_execution(exec).await.unwrap();
1731 match &entries[0].entry {
1732 EntryKind::StepResult { payload, .. } => {
1733 assert_eq!(payload.as_ref(), b"secret-payload");
1734 }
1735 other => panic!("unexpected entry kind: {other:?}"),
1736 }
1737 }
1738
1739 #[tokio::test]
1740 async fn control_entry_hmac_is_stamped_only_when_keyed() {
1741 let exec = ExecutionId::new();
1742
1743 let unkeyed = mem_backend(1_048_576).await;
1744 unkeyed
1745 .open_execution(exec, ExecutionKind::AgentTurn)
1746 .await
1747 .unwrap();
1748 unkeyed.append(effect_intent(exec, 0)).await.unwrap();
1749 match &unkeyed.read_execution(exec).await.unwrap()[0].entry {
1750 EntryKind::EffectIntent { hmac, .. } => assert!(hmac.is_none()),
1751 other => panic!("unexpected entry kind: {other:?}"),
1752 }
1753
1754 let keyed = mem_backend(1_048_576).await.with_hmac_key([7u8; 32]);
1755 let exec2 = ExecutionId::new();
1756 keyed
1757 .open_execution(exec2, ExecutionKind::AgentTurn)
1758 .await
1759 .unwrap();
1760 keyed.append(effect_intent(exec2, 0)).await.unwrap();
1761 match &keyed.read_execution(exec2).await.unwrap()[0].entry {
1762 EntryKind::EffectIntent { hmac, .. } => {
1763 assert!(
1764 hmac.is_some(),
1765 "keyed backend stamps a row HMAC over control entries"
1766 );
1767 }
1768 other => panic!("unexpected entry kind: {other:?}"),
1769 }
1770 }
1771
1772 #[tokio::test]
1773 async fn promise_and_timer_entries_fail_closed() {
1774 let backend = mem_backend(1_048_576).await;
1775 let exec = ExecutionId::new();
1776 backend
1777 .open_execution(exec, ExecutionKind::AgentTurn)
1778 .await
1779 .unwrap();
1780 let timer = JournalEntry {
1781 seq: None,
1782 execution_id: exec,
1783 kind: ExecutionKind::AgentTurn,
1784 step_id: StepId::new(0),
1785 entry: EntryKind::TimerArmed {
1786 timer_id: crate::TimerId::new(),
1787 due_at_ms: 1_000,
1788 hmac: None,
1789 },
1790 created_at_ms: 0,
1791 };
1792 assert_matches!(
1793 backend.append(timer).await,
1794 Err(DurableError::UnsupportedEntryKind {
1795 kind: "timer_armed"
1796 })
1797 );
1798 }
1799
1800 #[tokio::test]
1801 async fn payload_over_limit_is_rejected_fail_closed() {
1802 let backend = mem_backend(8).await;
1803 let exec = ExecutionId::new();
1804 backend
1805 .open_execution(exec, ExecutionKind::AgentTurn)
1806 .await
1807 .unwrap();
1808 let big = vec![0u8; 64];
1809 assert_matches!(
1810 backend.append(step_result(exec, 0, &big)).await,
1811 Err(DurableError::PayloadTooLarge { .. })
1812 );
1813 }
1814
1815 #[tokio::test]
1816 async fn finalize_marks_terminal_status_and_time() {
1817 let backend = mem_backend(1_048_576).await;
1818 let exec = ExecutionId::new();
1819 backend
1820 .open_execution(exec, ExecutionKind::AgentTurn)
1821 .await
1822 .unwrap();
1823 backend
1824 .finalize(exec, ExecutionStatus::Completed)
1825 .await
1826 .unwrap();
1827
1828 let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
1829 "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
1830 ))
1831 .bind(exec.as_uuid().to_string())
1832 .fetch_one(backend.pool())
1833 .await
1834 .unwrap();
1835 assert_eq!(status, "completed");
1836 assert!(finalized.is_some(), "a terminal status stamps finalized_at");
1837 }
1838
1839 #[tokio::test]
1840 async fn max_seq_reflects_committed_appends() {
1841 let backend = mem_backend(1_048_576).await;
1842 assert_eq!(
1843 backend.max_seq().await.unwrap(),
1844 None,
1845 "empty journal has no max seq"
1846 );
1847
1848 let exec = ExecutionId::new();
1849 backend
1850 .open_execution(exec, ExecutionKind::AgentTurn)
1851 .await
1852 .unwrap();
1853 for step in 0..3 {
1854 backend.append(step_result(exec, step, b"x")).await.unwrap();
1855 }
1856 assert_eq!(backend.max_seq().await.unwrap(), Some(JournalSeq::new(3)));
1857 }
1858
1859 #[tokio::test]
1860 async fn append_batch_group_commits_every_entry() {
1861 let backend = mem_backend(1_048_576).await;
1862 let exec = ExecutionId::new();
1863 backend
1864 .open_execution(exec, ExecutionKind::AgentTurn)
1865 .await
1866 .unwrap();
1867 let batch = vec![
1868 step_result(exec, 0, b"a"),
1869 step_result(exec, 1, b"b"),
1870 step_result(exec, 2, b"c"),
1871 ];
1872 backend.append_batch(&batch).await.unwrap();
1873 assert_eq!(backend.read_execution(exec).await.unwrap().len(), 3);
1874 }
1875
1876 #[tokio::test]
1877 async fn read_execution_range_bounds_the_segment() {
1878 let backend = mem_backend(1_048_576).await;
1879 let exec = ExecutionId::new();
1880 backend
1881 .open_execution(exec, ExecutionKind::AgentTurn)
1882 .await
1883 .unwrap();
1884 for step in 0..5 {
1885 backend.append(step_result(exec, step, b"x")).await.unwrap();
1886 }
1887 let segment = backend.read_execution_range(exec, 2, 2).await.unwrap();
1888 assert_eq!(segment.len(), 2);
1889 assert_eq!(segment[0].step_id, StepId::new(2));
1890 assert_eq!(segment[1].step_id, StepId::new(3));
1891 }
1892
1893 #[tokio::test]
1894 async fn lookup_committed_result_finds_by_idem_key() {
1895 let backend = mem_backend(1_048_576).await;
1896 let exec = ExecutionId::new();
1897 backend
1898 .open_execution(exec, ExecutionKind::AgentTurn)
1899 .await
1900 .unwrap();
1901 let entry = step_result(exec, 0, b"committed");
1902 let idem_key = match &entry.entry {
1903 EntryKind::StepResult {
1904 idempotency_key, ..
1905 } => *idempotency_key,
1906 other => panic!("unexpected entry kind: {other:?}"),
1907 };
1908 backend.append(entry).await.unwrap();
1909
1910 let found = backend
1911 .lookup_committed_result(exec, idem_key)
1912 .await
1913 .unwrap()
1914 .expect("committed result is located by its idempotency key");
1915 match &found.entry {
1916 EntryKind::StepResult { payload, .. } => assert_eq!(payload.as_ref(), b"committed"),
1917 other => panic!("unexpected entry kind: {other:?}"),
1918 }
1919
1920 let absent = IdempotencyKey::derive(exec, StepId::new(99), b"never");
1922 assert!(
1923 backend
1924 .lookup_committed_result(exec, absent)
1925 .await
1926 .unwrap()
1927 .is_none()
1928 );
1929 }
1930
1931 #[tokio::test]
1932 async fn capabilities_describe_the_local_profile() {
1933 let backend = mem_backend(4096).await;
1934 let caps = backend.capabilities();
1935 assert!(caps.parallel_steps);
1936 assert!(
1937 !caps.cross_process,
1938 "the SQLite local backend is in-process"
1939 );
1940 assert_eq!(caps.max_payload, 4096);
1941 }
1942
1943 #[tokio::test]
1944 async fn promise_insert_state_and_resolve_round_trip() {
1945 let backend = mem_backend(1_048_576)
1946 .await
1947 .with_cipher(Arc::new(XorCipher));
1948 let exec = ExecutionId::new();
1949 backend
1950 .open_execution(exec, ExecutionKind::AgentTurn)
1951 .await
1952 .unwrap();
1953 let promise = PromiseId::derive(exec, StepId::new(0));
1954 backend
1955 .insert_promise(promise, exec, [9u8; 32], 100)
1956 .await
1957 .unwrap();
1958
1959 let pending = backend.promise_state(promise).await.unwrap().unwrap();
1960 assert!(!pending.resolved);
1961 assert_eq!(pending.execution_id, exec);
1962 assert_eq!(pending.resolver_token_hash, [9u8; 32]);
1963
1964 assert!(
1966 backend
1967 .resolve_promise(promise, exec, b"answer", 200)
1968 .await
1969 .unwrap()
1970 );
1971 assert!(
1972 !backend
1973 .resolve_promise(promise, exec, b"again", 300)
1974 .await
1975 .unwrap()
1976 );
1977
1978 let resolved = backend.promise_state(promise).await.unwrap().unwrap();
1979 assert!(resolved.resolved);
1980 let sealed = resolved.payload.expect("resolved payload present");
1981 assert_ne!(sealed.as_slice(), b"answer", "payload is sealed at rest");
1982 let opened = backend
1983 .open_promise_payload(promise, exec, &sealed)
1984 .unwrap();
1985 assert_eq!(opened.as_ref(), b"answer");
1986 }
1987
1988 #[tokio::test]
1989 async fn timer_arm_due_and_fire() {
1990 let backend = mem_backend(1_048_576).await;
1991 let exec = ExecutionId::new();
1992 backend
1993 .open_execution(exec, ExecutionKind::AgentTurn)
1994 .await
1995 .unwrap();
1996 let past = TimerId::derive(exec, StepId::new(0));
1997 let future = TimerId::derive(exec, StepId::new(1));
1998 backend.arm_timer(past, exec, 1_000, 0).await.unwrap();
1999 backend
2000 .arm_timer(future, exec, 9_000_000_000_000, 0)
2001 .await
2002 .unwrap();
2003
2004 let due = backend.due_timers(5_000).await.unwrap();
2006 assert_eq!(due, vec![past]);
2007
2008 assert!(backend.mark_timer_fired(past).await.unwrap());
2009 assert!(
2010 !backend.mark_timer_fired(past).await.unwrap(),
2011 "second fire is a no-op"
2012 );
2013 assert_eq!(
2014 backend.timer_state(past).await.unwrap(),
2015 Some((1_000, true))
2016 );
2017 assert!(backend.due_timers(5_000).await.unwrap().is_empty());
2019 }
2020
2021 #[tokio::test]
2022 async fn prune_deletes_terminal_executions_past_ttl() {
2023 let backend = mem_backend(1_048_576).await;
2024 let old = ExecutionId::new();
2026 backend
2027 .open_execution(old, ExecutionKind::AgentTurn)
2028 .await
2029 .unwrap();
2030 backend.append(step_result(old, 0, b"x")).await.unwrap();
2031 zeph_db::query(sql!(
2033 "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
2034 ))
2035 .bind(old.as_uuid().to_string())
2036 .execute(backend.pool())
2037 .await
2038 .unwrap();
2039
2040 let live = ExecutionId::new();
2041 backend
2042 .open_execution(live, ExecutionKind::AgentTurn)
2043 .await
2044 .unwrap();
2045 backend.append(step_result(live, 0, b"y")).await.unwrap();
2046
2047 let policy = RetentionPolicy {
2048 ttl_completed_secs: 1,
2049 prune_batch_size: 10,
2050 ..RetentionPolicy::default()
2051 };
2052 let deleted = backend.prune(&policy).await.unwrap();
2053 assert_eq!(deleted, 1, "only the aged terminal execution is pruned");
2054
2055 assert!(backend.read_execution(old).await.unwrap().is_empty());
2057 assert!(
2058 backend
2059 .promise_state(PromiseId::derive(old, StepId::new(0)))
2060 .await
2061 .unwrap()
2062 .is_none()
2063 );
2064 assert_eq!(backend.read_execution(live).await.unwrap().len(), 1);
2065 }
2066
2067 #[tokio::test]
2068 async fn checkpoint_fold_compacts_idempotent_prefix_and_replays() {
2069 let backend = mem_backend(1_048_576)
2070 .await
2071 .with_cipher(Arc::new(XorCipher));
2072 let exec = ExecutionId::new();
2073 backend
2074 .open_execution(exec, ExecutionKind::AgentTurn)
2075 .await
2076 .unwrap();
2077 for step in 0..5 {
2078 backend
2079 .append(step_result(exec, step, format!("v{step}").as_bytes()))
2080 .await
2081 .unwrap();
2082 }
2083
2084 let folded = backend.checkpoint_fold(exec, 3).await.unwrap();
2086 assert_eq!(folded, 3);
2087
2088 let remaining = backend.read_execution(exec).await.unwrap();
2090 let step_results: Vec<u32> = remaining
2091 .iter()
2092 .filter(|e| matches!(e.entry, EntryKind::StepResult { .. }))
2093 .map(|e| e.step_id.value())
2094 .collect();
2095 assert_eq!(step_results, vec![3, 4], "folded step rows are deleted");
2096 assert!(
2097 remaining
2098 .iter()
2099 .any(|e| matches!(e.entry, EntryKind::Checkpoint { .. })),
2100 "a checkpoint entry replaces the folded prefix"
2101 );
2102
2103 let preloaded = backend.read_checkpoints(exec).await.unwrap();
2105 assert_eq!(preloaded.len(), 3);
2106 for (i, entry) in preloaded.iter().enumerate() {
2107 let step = u32::try_from(i).unwrap();
2108 assert_eq!(entry.step_id, StepId::new(step));
2109 match &entry.entry {
2110 EntryKind::StepResult {
2111 payload,
2112 idempotency_key,
2113 ..
2114 } => {
2115 assert_eq!(payload.as_ref(), format!("v{step}").as_bytes());
2116 assert_eq!(
2117 *idempotency_key,
2118 IdempotencyKey::derive(exec, StepId::new(step), b"tool:read")
2119 );
2120 }
2121 other => panic!("unexpected folded entry: {other:?}"),
2122 }
2123 }
2124 }
2125}