1use std::fmt::Write as _;
41use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
42use std::sync::{Arc, Mutex};
43use std::time::{Duration, SystemTime};
44
45use rand::Rng as _;
46use serde::Serialize;
47use serde::de::DeserializeOwned;
48use tokio::task::JoinSet;
49use tracing::Instrument as _;
50use zeroize::Zeroizing;
51
52use crate::backend::local::now_unix_millis;
53use crate::backend::{DurableBackendEnum, ExecutionBackend as _};
54use crate::config::DurableConfig;
55use crate::effect::{EffectClass, OnAmbiguous};
56use crate::error::DurableError;
57use crate::ids::{ExecutionId, ExecutionKind, IdempotencyKey, PromiseId, StepId, TimerId};
58use crate::journal::{EntryKind, ExecutionStatus, Journal as _, JournalEntry};
59use crate::promise::{DurablePromise, RESOLVER_TOKEN_LEN, resolver_token_hash};
60use crate::replay::{DEFAULT_SEGMENT_STEPS, ReplayCursor, StepReplay};
61use crate::retention::step_cap_thresholds;
62use crate::step::{
63 DurableStep, PAYLOAD_VERSION, StepDescriptor, StepError, StepHandle, deserialize_result,
64 serialize_result,
65};
66use crate::waiters::wait_on_notify_or_poll;
67use crate::writer::JournalWriterHandle;
68
69#[derive(Debug)]
76pub struct DurableContext {
77 execution_id: ExecutionId,
78 kind: ExecutionKind,
79 next_step: AtomicU32,
80 diverged: AtomicBool,
81 is_resume: bool,
82 cursor: ReplayCursor,
83 backend: Arc<DurableBackendEnum>,
84 writer: JournalWriterHandle,
85 max_steps_per_execution: u32,
86 max_payload_bytes: u64,
87 soft_step_cap: u32,
89 poll_interval: Duration,
91 max_parked_promises: u32,
93 checkpoint_requested: AtomicBool,
95 fold_tasks: Mutex<JoinSet<()>>,
97}
98
99impl DurableContext {
100 #[must_use]
108 pub fn new(
109 execution_id: ExecutionId,
110 kind: ExecutionKind,
111 is_resume: bool,
112 backend: Arc<DurableBackendEnum>,
113 writer: JournalWriterHandle,
114 config: &DurableConfig,
115 ) -> Self {
116 let cursor = ReplayCursor::new(backend.clone(), execution_id, DEFAULT_SEGMENT_STEPS);
117 let (soft_step_cap, _hard) = step_cap_thresholds(config.max_steps_per_execution);
118 Self {
119 execution_id,
120 kind,
121 next_step: AtomicU32::new(0),
122 diverged: AtomicBool::new(false),
123 is_resume,
124 cursor,
125 backend,
126 writer,
127 max_steps_per_execution: config.max_steps_per_execution,
128 max_payload_bytes: config.max_payload_bytes,
129 soft_step_cap,
130 poll_interval: Duration::from_secs(config.promise_poll_interval_secs.max(1)),
131 max_parked_promises: config.max_parked_promises,
132 checkpoint_requested: AtomicBool::new(false),
133 fold_tasks: Mutex::new(JoinSet::new()),
134 }
135 }
136
137 #[must_use]
139 pub fn execution_id(&self) -> ExecutionId {
140 self.execution_id
141 }
142
143 #[must_use]
145 pub fn kind(&self) -> ExecutionKind {
146 self.kind
147 }
148
149 #[tracing::instrument(
183 name = "durable.context.step",
184 skip_all,
185 fields(execution_id = %self.execution_id.as_uuid(), step_name = desc.name())
186 )]
187 pub async fn step<T, F, Fut>(&self, desc: StepDescriptor, op: F) -> Result<T, DurableError>
188 where
189 T: Serialize + DeserializeOwned + Send,
190 F: FnOnce(StepHandle) -> Fut + Send,
191 Fut: Future<Output = Result<T, StepError>> + Send,
192 {
193 let step_id = self.assign_step_id();
194 self.run_step_at(step_id, desc, op)
195 .await
196 .map(DurableStep::into_value)
197 }
198
199 #[tracing::instrument(
205 name = "durable.context.step_recorded",
206 skip_all,
207 fields(execution_id = %self.execution_id.as_uuid(), step_name = desc.name())
208 )]
209 pub async fn step_recorded<T, F, Fut>(
210 &self,
211 desc: StepDescriptor,
212 op: F,
213 ) -> Result<DurableStep<T>, DurableError>
214 where
215 T: Serialize + DeserializeOwned + Send,
216 F: FnOnce(StepHandle) -> Fut + Send,
217 Fut: Future<Output = Result<T, StepError>> + Send,
218 {
219 let step_id = self.assign_step_id();
220 self.run_step_at(step_id, desc, op).await
221 }
222
223 #[must_use]
229 pub fn parallel(&self) -> ParallelScope<'_> {
230 ParallelScope { ctx: self }
231 }
232
233 #[tracing::instrument(
246 name = "durable.context.promise",
247 skip_all,
248 fields(execution_id = %self.execution_id.as_uuid())
249 )]
250 pub async fn promise<T>(&self) -> Result<DurablePromise<T>, DurableError> {
251 let step_id = self.checked_step_id()?;
252 let promise_id = PromiseId::derive(self.execution_id, step_id);
253
254 if self.backend.promise_state(promise_id).await?.is_some() {
256 return Ok(DurablePromise::resumed(promise_id));
257 }
258
259 let mut token = Zeroizing::new([0u8; RESOLVER_TOKEN_LEN]);
260 rand::rng().fill_bytes(&mut *token);
261 let hash = resolver_token_hash(promise_id, self.execution_id, &token);
262 self.backend
263 .insert_promise(
264 promise_id,
265 self.execution_id,
266 *hash.as_bytes(),
267 now_unix_millis(),
268 )
269 .await?;
270 Ok(DurablePromise::fresh(promise_id, token))
271 }
272
273 pub async fn await_promise<T: DeserializeOwned>(
286 &self,
287 promise: DurablePromise<T>,
288 ) -> Result<T, DurableError> {
289 let id = promise.id();
290 let key = id.as_uuid();
291 let cap = usize::try_from(self.max_parked_promises).unwrap_or(usize::MAX);
292 let span = tracing::info_span!("durable.promise.await", promise_id = %key);
293 wait_on_notify_or_poll(
294 self.backend.promise_waiters(),
295 key,
296 Some(cap),
297 || self.poll_interval,
298 || self.take_resolved_promise::<T>(id),
299 || self.take_resolved_promise::<T>(id),
300 )
301 .instrument(span)
302 .await
303 }
304
305 #[tracing::instrument(name = "durable.context.take_resolved_promise", skip_all, fields(promise_id = %id.as_uuid()))]
307 async fn take_resolved_promise<T: DeserializeOwned>(
308 &self,
309 id: PromiseId,
310 ) -> Result<Option<T>, DurableError> {
311 let record = self
312 .backend
313 .promise_state(id)
314 .await?
315 .ok_or(DurableError::UnknownPromise)?;
316 if !record.resolved {
317 return Ok(None);
318 }
319 let sealed = record.payload.ok_or(DurableError::Decode {
320 context: "resolved promise is missing its payload",
321 })?;
322 let plaintext = self
323 .backend
324 .open_promise_payload(id, record.execution_id, &sealed)?;
325 deserialize_result(&plaintext).map(Some)
326 }
327
328 #[tracing::instrument(
341 name = "durable.context.sleep_until",
342 skip_all,
343 fields(execution_id = %self.execution_id.as_uuid())
344 )]
345 pub async fn sleep_until(&self, due: SystemTime) -> Result<(), DurableError> {
346 let step_id = self.checked_step_id()?;
347 let timer_id = TimerId::derive(self.execution_id, step_id);
348 let due_ms = system_time_to_millis(due);
349
350 match self.backend.timer_state(timer_id).await? {
351 Some((_, true)) => return Ok(()),
353 Some((_, false)) => {}
355 None => {
357 self.backend
358 .arm_timer(timer_id, self.execution_id, due_ms, now_unix_millis())
359 .await?;
360 }
361 }
362
363 let key = timer_id.as_uuid();
364 wait_on_notify_or_poll(
365 self.backend.timer_waiters(),
366 key,
367 None,
368 || {
369 let remaining =
370 u64::try_from(due_ms.saturating_sub(now_unix_millis())).unwrap_or(u64::MAX);
371 self.poll_interval.min(Duration::from_millis(remaining))
372 },
373 || self.check_timer_due(timer_id, due_ms),
376 || async move {
377 if let Some(value) = self.check_timer_due(timer_id, due_ms).await? {
381 return Ok(Some(value));
382 }
383 if matches!(self.backend.timer_state(timer_id).await?, Some((_, true))) {
384 return Ok(Some(()));
385 }
386 Ok(None)
387 },
388 )
389 .await
390 }
391
392 async fn check_timer_due(
397 &self,
398 timer_id: TimerId,
399 due_ms: i64,
400 ) -> Result<Option<()>, DurableError> {
401 if now_unix_millis() >= due_ms {
402 self.backend.mark_timer_fired(timer_id).await?;
404 return Ok(Some(()));
405 }
406 Ok(None)
407 }
408
409 #[must_use]
415 pub fn resolver_handle(&self) -> crate::promise::DurableHandle {
416 crate::promise::DurableHandle::new(self.backend.clone())
417 }
418
419 #[tracing::instrument(name = "durable.context.drain_background", skip_all, fields(execution_id = %self.execution_id.as_uuid()))]
424 pub async fn drain_background(&self) {
425 let mut set = {
426 let mut guard = self
427 .fold_tasks
428 .lock()
429 .unwrap_or_else(std::sync::PoisonError::into_inner);
430 std::mem::take(&mut *guard)
431 };
432 while set.join_next().await.is_some() {}
433 }
434
435 fn assign_step_id(&self) -> StepId {
437 StepId::new(self.next_step.fetch_add(1, Ordering::Relaxed))
438 }
439
440 fn checked_step_id(&self) -> Result<StepId, DurableError> {
442 let step_id = self.assign_step_id();
443 if self.max_steps_per_execution != 0 && step_id.value() >= self.max_steps_per_execution {
444 return Err(DurableError::StepCapExceeded {
445 cap: self.max_steps_per_execution,
446 });
447 }
448 Ok(step_id)
449 }
450
451 fn maybe_checkpoint(&self, step_id: StepId) {
458 if step_id.value() < self.soft_step_cap {
459 return;
460 }
461 if self.checkpoint_requested.swap(true, Ordering::AcqRel) {
462 return;
463 }
464 let backend = self.backend.clone();
465 let execution_id = self.execution_id;
466 let up_to = self.soft_step_cap;
467 let mut guard = self
468 .fold_tasks
469 .lock()
470 .unwrap_or_else(std::sync::PoisonError::into_inner);
471 guard.spawn(async move {
472 match backend.checkpoint_fold(execution_id, up_to).await {
473 Ok(folded) => {
474 tracing::info!(
475 execution_id = %execution_id.as_uuid(),
476 folded,
477 "durable checkpoint fold compacted the idempotent prefix"
478 );
479 }
480 Err(error) => {
481 tracing::warn!(%error, "durable checkpoint fold failed");
482 }
483 }
484 });
485 }
486
487 fn replay_active(&self) -> bool {
489 self.is_resume && !self.diverged.load(Ordering::Acquire)
490 }
491
492 async fn run_step_at<T, F, Fut>(
494 &self,
495 step_id: StepId,
496 desc: StepDescriptor,
497 op: F,
498 ) -> Result<DurableStep<T>, DurableError>
499 where
500 T: Serialize + DeserializeOwned + Send,
501 F: FnOnce(StepHandle) -> Fut + Send,
502 Fut: Future<Output = Result<T, StepError>> + Send,
503 {
504 if self.max_steps_per_execution != 0 && step_id.value() >= self.max_steps_per_execution {
505 return Err(DurableError::StepCapExceeded {
506 cap: self.max_steps_per_execution,
507 });
508 }
509 self.maybe_checkpoint(step_id);
511 let effect = desc.effect();
512 let idem_key =
513 IdempotencyKey::derive(self.execution_id, step_id, &desc.fingerprint_input());
514
515 let span = tracing::info_span!(
516 "durable.step.run",
517 step_id = step_id.value(),
518 effect_class = effect.as_str(),
519 replayed = tracing::field::Empty,
520 );
521 async move {
522 if self.replay_active() {
524 match self.cursor.lookup(step_id).await? {
525 StepReplay::Result(entry) => {
526 self.check_divergence(step_id, idem_key, &entry).await?;
527 let value = replay_value::<T>(step_id, effect, &entry)?;
528 tracing::Span::current().record("replayed", true);
529 return Ok(DurableStep::replayed(step_id, idem_key, value));
530 }
531 StepReplay::IntentOnly(entry) => {
532 self.check_divergence(step_id, idem_key, &entry).await?;
533 return self.resolve_ambiguous(step_id, idem_key, &desc, op).await;
534 }
535 StepReplay::Fresh => {}
536 }
537 }
538
539 if effect == EffectClass::ExactlyOnceGuarded
542 && let Some(entry) = self
543 .backend
544 .lookup_committed_result(self.execution_id, idem_key)
545 .await?
546 {
547 let value = replay_value::<T>(step_id, effect, &entry)?;
548 tracing::Span::current().record("replayed", true);
549 return Ok(DurableStep::replayed(step_id, idem_key, value));
550 }
551
552 tracing::Span::current().record("replayed", false);
554 if effect == EffectClass::ExactlyOnceGuarded {
555 let intent = self.intent_entry(step_id, idem_key, effect);
557 self.append_acked_degrading(intent, desc.name()).await?;
558 }
559 let value = self.run_op(op, step_id, idem_key, desc.name()).await?;
560 let payload = serialize_result(&value, desc.name())?;
564 self.journal_result(payload, step_id, idem_key, effect, desc.name())
565 .await?;
566 Ok(DurableStep::live(step_id, idem_key, value))
567 }
568 .instrument(span)
569 .await
570 }
571
572 #[tracing::instrument(name = "durable.context.check_divergence", skip_all, fields(step_id = step_id.value()))]
574 async fn check_divergence(
575 &self,
576 step_id: StepId,
577 expected: IdempotencyKey,
578 entry: &JournalEntry,
579 ) -> Result<(), DurableError> {
580 if entry.entry.idempotency_key() == Some(expected) {
583 return Ok(());
584 }
585 self.on_divergence(step_id).await;
586 Err(DurableError::ReplayDivergence { step_id })
587 }
588
589 #[tracing::instrument(name = "durable.context.on_divergence", skip_all, fields(step_id = step_id.value(), execution_id = %self.execution_id.as_uuid()))]
591 async fn on_divergence(&self, step_id: StepId) {
592 self.diverged.store(true, Ordering::Release);
593 tracing::warn!(
594 execution_id = %self.execution_id.as_uuid(),
595 step_id = step_id.value(),
596 "replay divergence detected; marking journal aborted and restarting fresh"
597 );
598 if let Err(error) = self
599 .backend
600 .finalize(self.execution_id, ExecutionStatus::Aborted)
601 .await
602 {
603 tracing::warn!(%error, "failed to mark diverged execution aborted");
604 }
605 }
606
607 #[tracing::instrument(name = "durable.context.resolve_ambiguous", skip_all, fields(step_id = step_id.value(), execution_id = %self.execution_id.as_uuid()))]
609 async fn resolve_ambiguous<T, F, Fut>(
610 &self,
611 step_id: StepId,
612 idem_key: IdempotencyKey,
613 desc: &StepDescriptor,
614 op: F,
615 ) -> Result<DurableStep<T>, DurableError>
616 where
617 T: Serialize + DeserializeOwned + Send,
618 F: FnOnce(StepHandle) -> Fut + Send,
619 Fut: Future<Output = Result<T, StepError>> + Send,
620 {
621 let effect = desc.effect();
622 let policy = desc.on_ambiguous().unwrap_or(OnAmbiguous::Fail);
623 self.emit_ambiguous_audit(step_id, effect, idem_key, policy);
624 match policy {
625 OnAmbiguous::Fail => Err(DurableError::AmbiguousEffect { step_id }),
627 OnAmbiguous::Skip | OnAmbiguous::Rerun => {
632 let value = self.run_op(op, step_id, idem_key, desc.name()).await?;
633 let payload = serialize_result(&value, desc.name())?;
634 self.journal_result(payload, step_id, idem_key, effect, desc.name())
635 .await?;
636 Ok(DurableStep::live(step_id, idem_key, value))
637 }
638 }
639 }
640
641 #[tracing::instrument(name = "durable.context.run_op", skip_all, fields(step_id = step_id.value(), step_name = name))]
643 async fn run_op<T, F, Fut>(
644 &self,
645 op: F,
646 step_id: StepId,
647 idem_key: IdempotencyKey,
648 name: &'static str,
649 ) -> Result<T, DurableError>
650 where
651 F: FnOnce(StepHandle) -> Fut + Send,
652 Fut: Future<Output = Result<T, StepError>> + Send,
653 {
654 let handle = StepHandle::new(step_id, idem_key);
655 op(handle)
656 .await
657 .map_err(|err| DurableError::step_failed(name, err))
658 }
659
660 #[tracing::instrument(name = "durable.context.journal_result", skip_all, fields(step_id = step_id.value(), effect_class = effect.as_str(), step_name = name))]
662 async fn journal_result(
663 &self,
664 payload: bytes::Bytes,
665 step_id: StepId,
666 idem_key: IdempotencyKey,
667 effect: EffectClass,
668 name: &'static str,
669 ) -> Result<(), DurableError> {
670 crate::cipher::ensure_payload_within_limit(payload.len(), self.max_payload_bytes)?;
672 let entry = JournalEntry {
673 seq: None,
674 execution_id: self.execution_id,
675 kind: self.kind,
676 step_id,
677 entry: EntryKind::StepResult {
678 idempotency_key: idem_key,
679 payload,
680 effect,
681 payload_version: PAYLOAD_VERSION,
682 },
683 created_at_ms: now_unix_millis(),
684 };
685 match effect {
686 EffectClass::ExactlyOnceGuarded => self.append_acked_degrading(entry, name).await,
688 EffectClass::Idempotent | EffectClass::AtLeastOnce => {
690 self.writer.append_buffered(entry);
691 Ok(())
692 }
693 }
694 }
695
696 fn intent_entry(
698 &self,
699 step_id: StepId,
700 idem_key: IdempotencyKey,
701 effect: EffectClass,
702 ) -> JournalEntry {
703 JournalEntry {
704 seq: None,
705 execution_id: self.execution_id,
706 kind: self.kind,
707 step_id,
708 entry: EntryKind::EffectIntent {
709 idempotency_key: idem_key,
710 effect,
711 hmac: None,
713 },
714 created_at_ms: now_unix_millis(),
715 }
716 }
717
718 #[tracing::instrument(name = "durable.context.append_acked_degrading", skip_all, fields(step_name = name))]
720 async fn append_acked_degrading(
721 &self,
722 entry: JournalEntry,
723 name: &'static str,
724 ) -> Result<(), DurableError> {
725 match self.writer.append_acked(entry).await {
726 Ok(_) => Ok(()),
727 Err(DurableError::JournalUnavailable) => {
728 tracing::warn!(
729 step = name,
730 "journal writer unavailable; this step degrades to non-durable mode"
731 );
732 metrics::counter!("durable.journal.writer.degraded_appends_total").increment(1);
733 Ok(())
734 }
735 Err(error) => Err(error),
736 }
737 }
738
739 fn emit_ambiguous_audit(
741 &self,
742 step_id: StepId,
743 effect: EffectClass,
744 idem_key: IdempotencyKey,
745 policy: OnAmbiguous,
746 ) {
747 tracing::warn!(
748 target: "durable.audit",
749 execution_id = %self.execution_id.as_uuid(),
750 step_id = step_id.value(),
751 effect_class = effect.as_str(),
752 idem_key = %idem_key_hex8(idem_key),
753 on_ambiguous = policy.as_str(),
754 "durable step resumed in the ambiguous window; applying on_ambiguous policy"
755 );
756 }
757}
758
759#[derive(Debug, Clone, Copy)]
765pub struct ParallelScope<'a> {
766 ctx: &'a DurableContext,
767}
768
769impl<'a> ParallelScope<'a> {
770 pub fn step<T, F, Fut>(
779 &self,
780 desc: StepDescriptor,
781 op: F,
782 ) -> impl Future<Output = Result<DurableStep<T>, DurableError>> + Send + 'a
783 where
784 T: Serialize + DeserializeOwned + Send + 'a,
785 F: FnOnce(StepHandle) -> Fut + Send + 'a,
786 Fut: Future<Output = Result<T, StepError>> + Send + 'a,
787 {
788 let step_id = self.ctx.assign_step_id();
789 let ctx = self.ctx;
790 async move { ctx.run_step_at(step_id, desc, op).await }
791 }
792}
793
794fn replay_value<T: DeserializeOwned>(
796 step_id: StepId,
797 effect: EffectClass,
798 entry: &JournalEntry,
799) -> Result<T, DurableError> {
800 let _span = tracing::info_span!(
801 "durable.step.replay",
802 step_id = step_id.value(),
803 effect_class = effect.as_str(),
804 )
805 .entered();
806 match &entry.entry {
807 EntryKind::StepResult { payload, .. } => deserialize_result(payload),
808 _ => Err(DurableError::Decode {
809 context: "replayed entry is not a step result",
810 }),
811 }
812}
813
814fn system_time_to_millis(time: SystemTime) -> i64 {
818 time.duration_since(SystemTime::UNIX_EPOCH)
819 .map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX))
820}
821
822fn idem_key_hex8(key: IdempotencyKey) -> String {
827 let mut out = String::with_capacity(16);
828 for byte in &key.as_bytes()[..8] {
829 let _ = write!(out, "{byte:02x}");
830 }
831 out
832}
833
834#[cfg(all(test, feature = "sqlite"))]
835mod tests {
836 use std::assert_matches;
837
838 use super::*;
839 use crate::backend::local::LocalBackend;
840 use crate::config::DurableConfig;
841 use crate::effect::EffectIntentSubClass;
842 use crate::timer::DurableTimerService;
843 use crate::writer::JournalWriter;
844 use std::pin::Pin;
845 use std::sync::atomic::AtomicU32;
846 use tokio::task::JoinHandle;
847
848 type StepFut<'a> =
851 Pin<Box<dyn Future<Output = Result<DurableStep<u32>, DurableError>> + Send + 'a>>;
852
853 fn fast_config() -> DurableConfig {
854 DurableConfig {
855 journal_flush_interval_ms: 5,
856 journal_ack_timeout_ms: 2000,
857 ..DurableConfig::default()
858 }
859 }
860
861 struct Harness {
863 ctx: DurableContext,
864 backend: Arc<LocalBackend>,
865 writer_task: JoinHandle<()>,
866 handle: JournalWriterHandle,
867 }
868
869 impl Harness {
870 async fn open(exec: ExecutionId, is_resume: bool) -> Self {
871 let local = Arc::new(LocalBackend::open(":memory:", 1_048_576).await.unwrap());
872 local.init().await.unwrap();
873 local
874 .open_execution(exec, ExecutionKind::AgentTurn)
875 .await
876 .unwrap();
877 let (writer, handle) = JournalWriter::new(local.clone(), &fast_config());
878 let writer_task = tokio::spawn(writer.run());
879 let backend = Arc::new(DurableBackendEnum::Local(local.clone()));
880 let ctx = DurableContext::new(
881 exec,
882 ExecutionKind::AgentTurn,
883 is_resume,
884 backend,
885 handle.clone(),
886 &fast_config(),
887 );
888 Self {
889 ctx,
890 backend: local,
891 writer_task,
892 handle,
893 }
894 }
895
896 fn resume(&self) -> DurableContext {
898 let backend = Arc::new(DurableBackendEnum::Local(self.backend.clone()));
899 DurableContext::new(
900 self.ctx.execution_id,
901 ExecutionKind::AgentTurn,
902 true,
903 backend,
904 self.handle.clone(),
905 &fast_config(),
906 )
907 }
908
909 async fn shutdown(self) {
910 self.writer_task.abort();
915 let _ = self.writer_task.await;
916 }
917 }
918
919 #[tokio::test]
920 async fn fresh_step_runs_op_and_journals_result() {
921 let exec = ExecutionId::new();
923 let h = Harness::open(exec, false).await;
924 let value: u32 = h
925 .ctx
926 .step(
927 StepDescriptor::idempotent("count", b"op".to_vec()),
928 |_| async { Ok(7) },
929 )
930 .await
931 .unwrap();
932 assert_eq!(value, 7);
933 h.handle.flush().await.unwrap();
934
935 let entries = h.backend.read_execution(exec).await.unwrap();
936 assert_eq!(entries.len(), 1);
937 assert_matches!(entries[0].entry, EntryKind::StepResult { .. });
938 h.shutdown().await;
939 }
940
941 #[tokio::test]
942 async fn replayed_idempotent_step_skips_op() {
943 let exec = ExecutionId::new();
945 let h = Harness::open(exec, false).await;
946 let desc = || StepDescriptor::idempotent("count", b"op".to_vec());
947 let first: u32 = h.ctx.step(desc(), |_| async { Ok(11) }).await.unwrap();
948 assert_eq!(first, 11);
949 h.handle.flush().await.unwrap();
950
951 let resumed = h.resume();
952 let ran_again = Arc::new(AtomicU32::new(0));
953 let counter = ran_again.clone();
954 let replayed: u32 = resumed
955 .step(desc(), move |_| {
956 let counter = counter.clone();
957 async move {
958 counter.fetch_add(1, Ordering::SeqCst);
959 Ok(999)
960 }
961 })
962 .await
963 .unwrap();
964 assert_eq!(
965 replayed, 11,
966 "the journaled value is returned, not the new one"
967 );
968 assert_eq!(
969 ran_again.load(Ordering::SeqCst),
970 0,
971 "the operation closure must not run on replay"
972 );
973 h.shutdown().await;
974 }
975
976 #[tokio::test]
977 async fn guarded_step_commits_intent_before_result() {
978 let exec = ExecutionId::new();
980 let h = Harness::open(exec, false).await;
981 let desc = StepDescriptor::exactly_once_guarded(
982 "charge",
983 EffectIntentSubClass::CostBearingOrBoundaryIdempotent,
984 Some(OnAmbiguous::Skip),
985 b"op".to_vec(),
986 )
987 .unwrap();
988 let _: u32 = h.ctx.step(desc, |_| async { Ok(5) }).await.unwrap();
989 h.handle.flush().await.unwrap();
990
991 let entries = h.backend.read_execution(exec).await.unwrap();
992 let kinds: Vec<_> = entries.iter().map(|e| e.entry.tag()).collect();
993 assert_eq!(
994 kinds,
995 vec!["effect_intent", "step_result"],
996 "intent is journaled before the result"
997 );
998 h.shutdown().await;
999 }
1000
1001 #[tokio::test]
1002 async fn replay_divergence_on_fingerprint_mismatch() {
1003 let exec = ExecutionId::new();
1005 let h = Harness::open(exec, false).await;
1006 let _: u32 = h
1007 .ctx
1008 .step(
1009 StepDescriptor::idempotent("count", b"v1".to_vec()),
1010 |_| async { Ok(1) },
1011 )
1012 .await
1013 .unwrap();
1014 h.handle.flush().await.unwrap();
1015
1016 let resumed = h.resume();
1017 let err = resumed
1019 .step::<u32, _, _>(
1020 StepDescriptor::idempotent("count", b"v2".to_vec()),
1021 |_| async { Ok(2) },
1022 )
1023 .await
1024 .unwrap_err();
1025 assert_matches!(err, DurableError::ReplayDivergence { .. });
1026
1027 let (status,): (String,) = zeph_db::query_as(zeph_db::sql!(
1028 "SELECT status FROM durable_executions WHERE execution_id = ?"
1029 ))
1030 .bind(exec.as_uuid().to_string())
1031 .fetch_one(h.backend.pool())
1032 .await
1033 .unwrap();
1034 assert_eq!(status, "aborted", "the diverged journal is marked aborted");
1035 h.shutdown().await;
1036 }
1037
1038 #[tokio::test]
1039 async fn ambiguous_window_fail_policy_surfaces_error() {
1040 let exec = ExecutionId::new();
1042 let h = Harness::open(exec, false).await;
1043 let step_id = StepId::new(0);
1044 let idem = IdempotencyKey::derive(
1045 exec,
1046 step_id,
1047 &StepDescriptor::exactly_once_guarded(
1048 "delete",
1049 EffectIntentSubClass::Destructive,
1050 Some(OnAmbiguous::Fail),
1051 b"op".to_vec(),
1052 )
1053 .unwrap()
1054 .fingerprint_input(),
1055 );
1056 h.backend
1058 .append(JournalEntry {
1059 seq: None,
1060 execution_id: exec,
1061 kind: ExecutionKind::AgentTurn,
1062 step_id,
1063 entry: EntryKind::EffectIntent {
1064 idempotency_key: idem,
1065 effect: EffectClass::ExactlyOnceGuarded,
1066 hmac: None,
1067 },
1068 created_at_ms: 0,
1069 })
1070 .await
1071 .unwrap();
1072
1073 let resumed = h.resume();
1074 let ran = Arc::new(AtomicU32::new(0));
1075 let counter = ran.clone();
1076 let err = resumed
1077 .step::<u32, _, _>(
1078 StepDescriptor::exactly_once_guarded(
1079 "delete",
1080 EffectIntentSubClass::Destructive,
1081 Some(OnAmbiguous::Fail),
1082 b"op".to_vec(),
1083 )
1084 .unwrap(),
1085 move |_| {
1086 let counter = counter.clone();
1087 async move {
1088 counter.fetch_add(1, Ordering::SeqCst);
1089 Ok(1)
1090 }
1091 },
1092 )
1093 .await
1094 .unwrap_err();
1095 assert_matches!(err, DurableError::AmbiguousEffect { .. });
1096 assert_eq!(
1097 ran.load(Ordering::SeqCst),
1098 0,
1099 "a fail-policy ambiguous step must not re-fire the effect"
1100 );
1101 h.shutdown().await;
1102 }
1103
1104 #[tokio::test]
1105 async fn inv13_committed_guarded_result_is_not_refired() {
1106 let exec = ExecutionId::new();
1109 let h = Harness::open(exec, false).await;
1110 let desc = || {
1111 StepDescriptor::exactly_once_guarded(
1112 "transfer",
1113 EffectIntentSubClass::MoneyMoving,
1114 Some(OnAmbiguous::Fail),
1115 b"op".to_vec(),
1116 )
1117 .unwrap()
1118 };
1119 let first: u32 = h.ctx.step(desc(), |_| async { Ok(500) }).await.unwrap();
1120 assert_eq!(first, 500);
1121 h.handle.flush().await.unwrap();
1122
1123 let backend = Arc::new(DurableBackendEnum::Local(h.backend.clone()));
1126 let fresh = DurableContext::new(
1127 exec,
1128 ExecutionKind::AgentTurn,
1129 false,
1130 backend,
1131 h.handle.clone(),
1132 &fast_config(),
1133 );
1134 let ran = Arc::new(AtomicU32::new(0));
1135 let counter = ran.clone();
1136 let value: u32 = fresh
1137 .step(desc(), move |_| {
1138 let counter = counter.clone();
1139 async move {
1140 counter.fetch_add(1, Ordering::SeqCst);
1141 Ok(0)
1142 }
1143 })
1144 .await
1145 .unwrap();
1146 assert_eq!(value, 500, "the pre-committed guarded result is returned");
1147 assert_eq!(
1148 ran.load(Ordering::SeqCst),
1149 0,
1150 "the guarded effect must not re-fire"
1151 );
1152 h.shutdown().await;
1153 }
1154
1155 #[tokio::test]
1156 async fn parallel_step_ids_are_completion_order_independent() {
1157 let exec = ExecutionId::new();
1159 let h = Harness::open(exec, false).await;
1160 let scope = h.ctx.parallel();
1161 let futures: Vec<StepFut> = vec![
1164 Box::pin(scope.step::<u32, _, _>(
1165 StepDescriptor::idempotent("a", b"a".to_vec()),
1166 |handle: StepHandle| async move {
1167 tokio::time::sleep(std::time::Duration::from_millis(30)).await;
1169 Ok(handle.step_id().value())
1170 },
1171 )),
1172 Box::pin(scope.step::<u32, _, _>(
1173 StepDescriptor::idempotent("b", b"b".to_vec()),
1174 |handle: StepHandle| async move {
1175 tokio::time::sleep(std::time::Duration::from_millis(5)).await;
1176 Ok(handle.step_id().value())
1177 },
1178 )),
1179 Box::pin(scope.step::<u32, _, _>(
1180 StepDescriptor::idempotent("c", b"c".to_vec()),
1181 |handle: StepHandle| async move { Ok(handle.step_id().value()) },
1182 )),
1183 ];
1184 let results = futures::future::try_join_all(futures).await.unwrap();
1185 let ids: Vec<u32> = results
1186 .iter()
1187 .map(DurableStep::step_id)
1188 .map(StepId::value)
1189 .collect();
1190 assert_eq!(ids, vec![0, 1, 2]);
1192 h.shutdown().await;
1193 }
1194
1195 #[tokio::test]
1196 async fn concurrent_steps_under_shared_ref_are_sound() {
1197 let exec = ExecutionId::new();
1200 let h = Harness::open(exec, false).await;
1201 let scope = h.ctx.parallel();
1202 let futures: Vec<StepFut> = (0..16)
1203 .map(|i| {
1204 Box::pin(scope.step::<u32, _, _>(
1205 StepDescriptor::idempotent("worker", format!("op:{i}").into_bytes()),
1206 move |handle: StepHandle| async move { Ok(handle.step_id().value()) },
1207 )) as StepFut
1208 })
1209 .collect();
1210 let results = futures::future::try_join_all(futures).await.unwrap();
1211 let mut ids: Vec<u32> = results
1212 .iter()
1213 .map(DurableStep::step_id)
1214 .map(StepId::value)
1215 .collect();
1216 ids.sort_unstable();
1217 ids.dedup();
1218 assert_eq!(ids.len(), 16, "all 16 concurrent steps got unique ids");
1219 h.handle.flush().await.unwrap();
1220 assert_eq!(h.backend.read_execution(exec).await.unwrap().len(), 16);
1221 h.shutdown().await;
1222 }
1223
1224 #[tokio::test]
1225 async fn op_failure_surfaces_as_step_failed_without_journaling() {
1226 let exec = ExecutionId::new();
1227 let h = Harness::open(exec, false).await;
1228 let err = h
1229 .ctx
1230 .step::<u32, _, _>(
1231 StepDescriptor::idempotent("boom", b"op".to_vec()),
1232 |_| async { Err(StepError::new("op exploded")) },
1233 )
1234 .await
1235 .unwrap_err();
1236 assert_matches!(err, DurableError::StepFailed { step: "boom", .. });
1237 h.handle.flush().await.unwrap();
1238 assert!(
1239 h.backend.read_execution(exec).await.unwrap().is_empty(),
1240 "a failed step journals no result"
1241 );
1242 h.shutdown().await;
1243 }
1244
1245 fn context_with(
1247 local: &Arc<LocalBackend>,
1248 handle: &JournalWriterHandle,
1249 exec: ExecutionId,
1250 is_resume: bool,
1251 config: &DurableConfig,
1252 ) -> DurableContext {
1253 let dispatch = Arc::new(DurableBackendEnum::Local(local.clone()));
1254 DurableContext::new(
1255 exec,
1256 ExecutionKind::AgentTurn,
1257 is_resume,
1258 dispatch,
1259 handle.clone(),
1260 config,
1261 )
1262 }
1263
1264 #[tokio::test]
1265 async fn promise_resolves_with_token_and_await_returns_value() {
1266 let exec = ExecutionId::new();
1268 let h = Harness::open(exec, false).await;
1269 let promise = h.ctx.promise::<u32>().await.unwrap();
1270 assert!(!promise.is_resumed());
1271 let token = *promise
1272 .resolver_token()
1273 .expect("fresh promise carries a token");
1274 let id = promise.id();
1275 let resolver = h.ctx.resolver_handle();
1276
1277 let (awaited, ack) = tokio::join!(h.ctx.await_promise::<u32>(promise), async {
1278 tokio::time::sleep(Duration::from_millis(25)).await;
1279 resolver.resolve(id, &token, 1234u32).await
1280 });
1281 ack.unwrap();
1282 assert_eq!(
1283 awaited.unwrap(),
1284 1234,
1285 "the awaiter receives the resolved value"
1286 );
1287 h.shutdown().await;
1288 }
1289
1290 #[tokio::test]
1291 async fn wrong_resolver_token_is_rejected_but_correct_one_resolves() {
1292 let exec = ExecutionId::new();
1294 let h = Harness::open(exec, false).await;
1295 let promise = h.ctx.promise::<String>().await.unwrap();
1296 let id = promise.id();
1297 let token = *promise.resolver_token().unwrap();
1298 let resolver = h.ctx.resolver_handle();
1299
1300 let mut wrong = token;
1303 wrong[0] ^= 0xFF;
1304 assert_matches!(
1305 resolver.resolve(id, &wrong, "forged".to_string()).await,
1306 Err(DurableError::PromiseRejected)
1307 );
1308 assert!(
1309 !h.backend.promise_state(id).await.unwrap().unwrap().resolved,
1310 "a rejected resolution must not resolve the promise"
1311 );
1312
1313 resolver
1315 .resolve(id, &token, "ok".to_string())
1316 .await
1317 .unwrap();
1318 assert!(h.backend.promise_state(id).await.unwrap().unwrap().resolved);
1319
1320 assert_matches!(
1322 resolver
1323 .resolve(PromiseId::new(), &token, "x".to_string())
1324 .await,
1325 Err(DurableError::UnknownPromise)
1326 );
1327 h.shutdown().await;
1328 }
1329
1330 #[tokio::test]
1331 async fn resumed_promise_awaits_the_resolved_value() {
1332 let exec = ExecutionId::new();
1334 let h = Harness::open(exec, false).await;
1335 let promise = h.ctx.promise::<u32>().await.unwrap();
1336 let id = promise.id();
1337 let token = *promise.resolver_token().unwrap();
1338 h.ctx
1339 .resolver_handle()
1340 .resolve(id, &token, 77u32)
1341 .await
1342 .unwrap();
1343
1344 let resumed = h.resume();
1346 let promise2 = resumed.promise::<u32>().await.unwrap();
1347 assert!(promise2.is_resumed());
1348 assert_eq!(
1349 promise2.id(),
1350 id,
1351 "the resumed promise re-derives the same id"
1352 );
1353 assert_eq!(resumed.await_promise::<u32>(promise2).await.unwrap(), 77);
1354 h.shutdown().await;
1355 }
1356
1357 #[tokio::test]
1358 async fn sleep_until_returns_when_the_instant_passes() {
1359 let exec = ExecutionId::new();
1360 let h = Harness::open(exec, false).await;
1361 let due = SystemTime::now() + Duration::from_millis(40);
1363 tokio::time::timeout(Duration::from_secs(2), h.ctx.sleep_until(due))
1364 .await
1365 .expect("sleep_until completes before the test timeout")
1366 .expect("sleep_until succeeds");
1367 h.shutdown().await;
1368 }
1369
1370 #[tokio::test]
1371 async fn sleep_until_wakes_on_concurrent_fire_before_due() {
1372 let exec = ExecutionId::new();
1376 let h = Harness::open(exec, false).await;
1377 let due = SystemTime::now() + Duration::from_secs(30);
1379 let timer_id = TimerId::derive(exec, StepId::new(0));
1380
1381 let (result, marked) = tokio::join!(
1382 tokio::time::timeout(Duration::from_secs(2), h.ctx.sleep_until(due)),
1383 async {
1384 tokio::time::sleep(Duration::from_millis(25)).await;
1386 h.backend.mark_timer_fired(timer_id).await
1387 }
1388 );
1389 assert!(marked.unwrap(), "the timer transitions to fired");
1390 result
1391 .expect("sleep_until wakes on the concurrent fire before the test timeout")
1392 .expect("sleep_until succeeds");
1393 h.shutdown().await;
1394 }
1395
1396 #[tokio::test]
1397 async fn sleep_until_past_due_returns_immediately_on_resume() {
1398 let exec = ExecutionId::new();
1400 let h = Harness::open(exec, false).await;
1401 let timer = TimerId::derive(exec, StepId::new(0));
1403 h.backend.arm_timer(timer, exec, 1_000, 0).await.unwrap();
1404
1405 let service = DurableTimerService::new(
1407 Arc::new(DurableBackendEnum::Local(h.backend.clone())),
1408 Duration::from_millis(5),
1409 );
1410 service.fire_due().await;
1411 assert_eq!(
1412 h.backend.timer_state(timer).await.unwrap(),
1413 Some((1_000, true))
1414 );
1415
1416 let resumed = h.resume();
1418 tokio::time::timeout(
1419 Duration::from_millis(200),
1420 resumed.sleep_until(SystemTime::now() + Duration::from_hours(1)),
1421 )
1422 .await
1423 .expect("resumed sleep_until returns immediately")
1424 .unwrap();
1425 h.shutdown().await;
1426 }
1427
1428 #[tokio::test]
1429 async fn soft_cap_triggers_checkpoint_fold_and_replay_skips_folded_steps() {
1430 let exec = ExecutionId::new();
1432 let local = Arc::new(LocalBackend::open(":memory:", 1_048_576).await.unwrap());
1433 local.init().await.unwrap();
1434 local
1435 .open_execution(exec, ExecutionKind::AgentTurn)
1436 .await
1437 .unwrap();
1438 let (writer, handle) = JournalWriter::new(local.clone(), &fast_config());
1439 let task = tokio::spawn(writer.run());
1440 let config = DurableConfig {
1441 max_steps_per_execution: 10,
1442 ..fast_config()
1443 };
1444 let ctx = context_with(&local, &handle, exec, false, &config);
1445
1446 let desc = |i: u32| StepDescriptor::idempotent("s", format!("op:{i}").into_bytes());
1447 for i in 0..9 {
1449 let v: u32 = ctx
1450 .step(desc(i), move |_| async move { Ok(i) })
1451 .await
1452 .unwrap();
1453 assert_eq!(v, i);
1454 }
1455 handle.flush().await.unwrap();
1456 ctx.step::<u32, _, _>(desc(9), |_| async { Ok(9) })
1458 .await
1459 .unwrap();
1460 ctx.drain_background().await;
1461 handle.flush().await.unwrap();
1462
1463 let entries = local.read_execution(exec).await.unwrap();
1465 let checkpoints = entries
1466 .iter()
1467 .filter(|e| matches!(e.entry, EntryKind::Checkpoint { .. }))
1468 .count();
1469 assert_eq!(checkpoints, 1, "the soft cap folded one checkpoint");
1470 let surviving: Vec<u32> = entries
1471 .iter()
1472 .filter(|e| matches!(e.entry, EntryKind::StepResult { .. }))
1473 .map(|e| e.step_id.value())
1474 .collect();
1475 assert_eq!(surviving, vec![9], "only the post-fold step row survives");
1476
1477 let resumed = context_with(&local, &handle, exec, true, &config);
1479 let reran = Arc::new(AtomicU32::new(0));
1480 for i in 0..9 {
1481 let counter = reran.clone();
1482 let v: u32 = resumed
1483 .step(desc(i), move |_| {
1484 let counter = counter.clone();
1485 async move {
1486 counter.fetch_add(1, Ordering::SeqCst);
1487 Ok(999)
1488 }
1489 })
1490 .await
1491 .unwrap();
1492 assert_eq!(v, i, "folded step {i} replays its journaled value");
1493 }
1494 assert_eq!(
1495 reran.load(Ordering::SeqCst),
1496 0,
1497 "no folded operation closure re-ran on replay"
1498 );
1499
1500 drop(ctx);
1501 drop(resumed);
1502 drop(handle);
1503 task.await.unwrap();
1504 }
1505
1506 #[tokio::test]
1507 async fn step_cap_is_enforced() {
1508 let exec = ExecutionId::new();
1509 let local = Arc::new(LocalBackend::open(":memory:", 1_048_576).await.unwrap());
1510 local.init().await.unwrap();
1511 local
1512 .open_execution(exec, ExecutionKind::AgentTurn)
1513 .await
1514 .unwrap();
1515 let (writer, handle) = JournalWriter::new(local.clone(), &fast_config());
1516 let task = tokio::spawn(writer.run());
1517 let backend = Arc::new(DurableBackendEnum::Local(local.clone()));
1518 let ctx = DurableContext::new(
1519 exec,
1520 ExecutionKind::AgentTurn,
1521 false,
1522 backend,
1523 handle.clone(),
1524 &DurableConfig {
1525 max_steps_per_execution: 1,
1526 ..fast_config()
1527 },
1528 );
1529 ctx.step::<u32, _, _>(
1531 StepDescriptor::idempotent("ok", b"op".to_vec()),
1532 |_| async { Ok(0) },
1533 )
1534 .await
1535 .unwrap();
1536 let err = ctx
1537 .step::<u32, _, _>(
1538 StepDescriptor::idempotent("over", b"op".to_vec()),
1539 |_| async { Ok(0) },
1540 )
1541 .await
1542 .unwrap_err();
1543 assert_matches!(err, DurableError::StepCapExceeded { cap: 1 });
1544 drop(ctx);
1545 drop(handle);
1546 task.await.unwrap();
1547 }
1548}