Skip to main content

zeph_durable/
handle.rs

1// SPDX-FileCopyrightText: 2026 Andrei G <bug-ops>
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! The `&self` durable execution context.
5//!
6//! [`DurableContext`] is the front door to durable execution: a consumer wraps each unit of work in
7//! [`step`](DurableContext::step) (or a [`parallel`](DurableContext::parallel) batch) and the
8//! context journals it, replays it on resume, and enforces the exactly-once contract. The entire
9//! surface is `&self` — step ids come from an [`AtomicU32`], so concurrent steps under a single
10//! shared context are sound without a mutable borrow (system-invariants §10).
11//!
12//! # Deterministic step ids (INV-2)
13//!
14//! A step id is assigned the moment a step is *started*, in program order, via `fetch_add`. A
15//! [`ParallelScope`] assigns each child's id eagerly when the child future is constructed (before
16//! any of them is polled), so a parallel batch's ids are fixed by argument order and are independent
17//! of completion order. The same program therefore re-derives the same ids on replay.
18//!
19//! # Replay and the divergence guard (INV-3)
20//!
21//! On resume the program re-runs and each step consults the `ReplayCursor`:
22//!
23//! - a committed result replays without invoking the closure (INV-10);
24//! - an intent-only entry means the *ambiguous window* — the step's
25//!   [`OnAmbiguous`] policy decides (and a mandatory audit record is emitted, FR-DE-10);
26//! - nothing journaled means run fresh.
27//!
28//! Before replaying a result the context compares the journaled step's [`IdempotencyKey`] — the
29//! step's structural fingerprint — against the key derived from the *current* descriptor. A mismatch
30//! is a [`DurableError::ReplayDivergence`]: the journal is marked `aborted` and replay is disabled
31//! so the execution restarts fresh, never returning a result for a structurally different step.
32//!
33//! # Exactly-once and the ambiguous window (FR-DE-04, INV-13)
34//!
35//! A guarded step commits its `EffectIntent` (acknowledged) *before* the closure runs and its
36//! `StepResult` (acknowledged) *after*. If a replay divergence forces a fresh run, a guarded effect
37//! that already committed a result is recognized by its idempotency key (a point lookup) and its
38//! journaled value is returned rather than re-firing the effect.
39
40use std::fmt::Write as _;
41use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
42use std::sync::{Arc, Mutex};
43use std::time::{Duration, SystemTime};
44
45use rand::Rng as _;
46use serde::Serialize;
47use serde::de::DeserializeOwned;
48use tokio::task::JoinSet;
49use tracing::Instrument as _;
50use zeroize::Zeroizing;
51
52use crate::backend::local::now_unix_millis;
53use crate::backend::{DurableBackendEnum, ExecutionBackend as _};
54use crate::config::DurableConfig;
55use crate::effect::{EffectClass, OnAmbiguous};
56use crate::error::DurableError;
57use crate::ids::{ExecutionId, ExecutionKind, IdempotencyKey, PromiseId, StepId, TimerId};
58use crate::journal::{EntryKind, ExecutionStatus, Journal as _, JournalEntry};
59use crate::promise::{DurablePromise, RESOLVER_TOKEN_LEN, resolver_token_hash};
60use crate::replay::{DEFAULT_SEGMENT_STEPS, ReplayCursor, StepReplay};
61use crate::retention::step_cap_thresholds;
62use crate::step::{
63    DurableStep, PAYLOAD_VERSION, StepDescriptor, StepError, StepHandle, deserialize_result,
64    serialize_result,
65};
66use crate::waiters::wait_on_notify_or_poll;
67use crate::writer::JournalWriterHandle;
68
69/// The `&self` durable execution context handed to a consumer's program.
70///
71/// Construct it with [`DurableContext::new`] from a shared backend (the read path) and a
72/// [`JournalWriterHandle`] (the write path) — both bound to the same `durable.db`. A fresh
73/// execution opens with `is_resume = false`; a resumed one with `is_resume = true`, which activates
74/// the `ReplayCursor`.
75#[derive(Debug)]
76pub struct DurableContext {
77    execution_id: ExecutionId,
78    kind: ExecutionKind,
79    next_step: AtomicU32,
80    diverged: AtomicBool,
81    is_resume: bool,
82    cursor: ReplayCursor,
83    backend: Arc<DurableBackendEnum>,
84    writer: JournalWriterHandle,
85    max_steps_per_execution: u32,
86    max_payload_bytes: u64,
87    /// Soft step-cap threshold (90% of the cap): the first step at or past it folds a checkpoint.
88    soft_step_cap: u32,
89    /// Database fallback poll interval for parked promises and timers.
90    poll_interval: Duration,
91    /// Above this many concurrently-parked promises, awaits fall back to pure polling.
92    max_parked_promises: u32,
93    /// Fires the soft-cap checkpoint fold exactly once per execution.
94    checkpoint_requested: AtomicBool,
95    /// Tracks the spawned background checkpoint-fold task(s) so they are abortable and drainable.
96    fold_tasks: Mutex<JoinSet<()>>,
97}
98
99impl DurableContext {
100    /// Build a context for an execution.
101    ///
102    /// `backend` is the shared read path (the `ReplayCursor` reads segments through it) and must
103    /// be the same `durable.db` instance the `writer` commits to. `is_resume` activates replay:
104    /// pass `true` when reopening an execution that already has journal entries (as
105    /// [`LocalBackend::open_execution`](crate::LocalBackend::open_execution) reports), `false` for a
106    /// brand-new execution.
107    #[must_use]
108    pub fn new(
109        execution_id: ExecutionId,
110        kind: ExecutionKind,
111        is_resume: bool,
112        backend: Arc<DurableBackendEnum>,
113        writer: JournalWriterHandle,
114        config: &DurableConfig,
115    ) -> Self {
116        let cursor = ReplayCursor::new(backend.clone(), execution_id, DEFAULT_SEGMENT_STEPS);
117        let (soft_step_cap, _hard) = step_cap_thresholds(config.max_steps_per_execution);
118        Self {
119            execution_id,
120            kind,
121            next_step: AtomicU32::new(0),
122            diverged: AtomicBool::new(false),
123            is_resume,
124            cursor,
125            backend,
126            writer,
127            max_steps_per_execution: config.max_steps_per_execution,
128            max_payload_bytes: config.max_payload_bytes,
129            soft_step_cap,
130            poll_interval: Duration::from_secs(config.promise_poll_interval_secs.max(1)),
131            max_parked_promises: config.max_parked_promises,
132            checkpoint_requested: AtomicBool::new(false),
133            fold_tasks: Mutex::new(JoinSet::new()),
134        }
135    }
136
137    /// The execution this context drives.
138    #[must_use]
139    pub fn execution_id(&self) -> ExecutionId {
140        self.execution_id
141    }
142
143    /// The execution's category.
144    #[must_use]
145    pub fn kind(&self) -> ExecutionKind {
146        self.kind
147    }
148
149    /// Run a durable step, returning just its value.
150    ///
151    /// On a fresh execution the operation `op` runs and its result is journaled; on replay the
152    /// journaled result is returned and `op` is never invoked (INV-10). The closure receives a
153    /// [`StepHandle`] carrying the step's [`IdempotencyKey`] for boundary deduplication.
154    ///
155    /// Use [`step_recorded`](DurableContext::step_recorded) when the live/replayed distinction or the
156    /// step id matters.
157    ///
158    /// # Errors
159    ///
160    /// - [`DurableError::StepFailed`] if `op` returns an error on a fresh run.
161    /// - [`DurableError::ReplayDivergence`] if the journaled step at this position has a different
162    ///   structural fingerprint (INV-3).
163    /// - [`DurableError::AmbiguousEffect`] for an [`OnAmbiguous::Fail`] guarded step caught in the
164    ///   ambiguous window.
165    /// - [`DurableError::Serialize`] / [`DurableError::Decode`] on a payload codec failure, or a
166    ///   storage error from the journal.
167    ///
168    /// # Examples
169    ///
170    /// ```no_run
171    /// # async fn run(ctx: &zeph_durable::DurableContext) -> Result<(), zeph_durable::DurableError> {
172    /// use zeph_durable::StepDescriptor;
173    ///
174    /// let lines: usize = ctx
175    ///     .step(StepDescriptor::idempotent("count", b"tool:count".to_vec()), |_handle| async {
176    ///         Ok(42)
177    ///     })
178    ///     .await?;
179    /// assert_eq!(lines, 42);
180    /// # Ok(()) }
181    /// ```
182    #[tracing::instrument(
183        name = "durable.context.step",
184        skip_all,
185        fields(execution_id = %self.execution_id.as_uuid(), step_name = desc.name())
186    )]
187    pub async fn step<T, F, Fut>(&self, desc: StepDescriptor, op: F) -> Result<T, DurableError>
188    where
189        T: Serialize + DeserializeOwned + Send,
190        F: FnOnce(StepHandle) -> Fut + Send,
191        Fut: Future<Output = Result<T, StepError>> + Send,
192    {
193        let step_id = self.assign_step_id();
194        self.run_step_at(step_id, desc, op)
195            .await
196            .map(DurableStep::into_value)
197    }
198
199    /// Run a durable step, returning the full [`DurableStep`] record (id, key, and outcome).
200    ///
201    /// # Errors
202    ///
203    /// Identical to [`step`](DurableContext::step).
204    #[tracing::instrument(
205        name = "durable.context.step_recorded",
206        skip_all,
207        fields(execution_id = %self.execution_id.as_uuid(), step_name = desc.name())
208    )]
209    pub async fn step_recorded<T, F, Fut>(
210        &self,
211        desc: StepDescriptor,
212        op: F,
213    ) -> Result<DurableStep<T>, DurableError>
214    where
215        T: Serialize + DeserializeOwned + Send,
216        F: FnOnce(StepHandle) -> Fut + Send,
217        Fut: Future<Output = Result<T, StepError>> + Send,
218    {
219        let step_id = self.assign_step_id();
220        self.run_step_at(step_id, desc, op).await
221    }
222
223    /// Open a [`ParallelScope`] whose children receive contiguous, eagerly-assigned step ids.
224    ///
225    /// Construct the children synchronously (e.g. with a `Vec` or `.map(...).collect()`), then drive
226    /// them with `join_all`/`try_join_all`; their ids are fixed by construction order and are stable
227    /// across replay regardless of completion order (INV-2).
228    #[must_use]
229    pub fn parallel(&self) -> ParallelScope<'_> {
230        ParallelScope { ctx: self }
231    }
232
233    /// Create a durable promise resolved out of band by an operator or A2A reply (FR-DE-05).
234    ///
235    /// The promise occupies a deterministic program position, so a resumed execution re-derives the
236    /// same [`PromiseId`] and re-attaches to the pending row rather than minting an orphan. A *fresh*
237    /// promise carries its 32-byte resolver token — hand it to the resolving channel via
238    /// [`DurablePromise::resolver_token`]; a *resumed* promise carries none (the original token was
239    /// delivered before the crash). Await the result with [`await_promise`](Self::await_promise).
240    ///
241    /// # Errors
242    ///
243    /// - [`DurableError::StepCapExceeded`] if the promise would exceed the per-execution step cap.
244    /// - A storage error if the promise row cannot be read or inserted.
245    #[tracing::instrument(
246        name = "durable.context.promise",
247        skip_all,
248        fields(execution_id = %self.execution_id.as_uuid())
249    )]
250    pub async fn promise<T>(&self) -> Result<DurablePromise<T>, DurableError> {
251        let step_id = self.checked_step_id()?;
252        let promise_id = PromiseId::derive(self.execution_id, step_id);
253
254        // Replay/resume: a row at this position means the promise was already created in a prior run.
255        if self.backend.promise_state(promise_id).await?.is_some() {
256            return Ok(DurablePromise::resumed(promise_id));
257        }
258
259        let mut token = Zeroizing::new([0u8; RESOLVER_TOKEN_LEN]);
260        rand::rng().fill_bytes(&mut *token);
261        let hash = resolver_token_hash(promise_id, self.execution_id, &token);
262        self.backend
263            .insert_promise(
264                promise_id,
265                self.execution_id,
266                *hash.as_bytes(),
267                now_unix_millis(),
268            )
269            .await?;
270        Ok(DurablePromise::fresh(promise_id, token))
271    }
272
273    /// Await a durable promise's resolved value, parking until it is resolved.
274    ///
275    /// Returns immediately if the promise is already resolved (the common replay case). Otherwise it
276    /// parks on an in-process notify keyed by the promise id and falls back to a database poll every
277    /// `promise_poll_interval_secs`; above `max_parked_promises` concurrent waiters it polls without
278    /// parking. A resolution committed by [`DurableHandle::resolve`](crate::DurableHandle::resolve)
279    /// wakes the waiter at once.
280    ///
281    /// # Errors
282    ///
283    /// - [`DurableError::UnknownPromise`] if the promise row is missing (e.g. pruned).
284    /// - A decode/integrity error if the resolved payload cannot be opened into `T`.
285    pub async fn await_promise<T: DeserializeOwned>(
286        &self,
287        promise: DurablePromise<T>,
288    ) -> Result<T, DurableError> {
289        let id = promise.id();
290        let key = id.as_uuid();
291        let cap = usize::try_from(self.max_parked_promises).unwrap_or(usize::MAX);
292        let span = tracing::info_span!("durable.promise.await", promise_id = %key);
293        wait_on_notify_or_poll(
294            self.backend.promise_waiters(),
295            key,
296            Some(cap),
297            || self.poll_interval,
298            || self.take_resolved_promise::<T>(id),
299            || self.take_resolved_promise::<T>(id),
300        )
301        .instrument(span)
302        .await
303    }
304
305    /// Read a promise's state and, if resolved, open and decode its value.
306    #[tracing::instrument(name = "durable.context.take_resolved_promise", skip_all, fields(promise_id = %id.as_uuid()))]
307    async fn take_resolved_promise<T: DeserializeOwned>(
308        &self,
309        id: PromiseId,
310    ) -> Result<Option<T>, DurableError> {
311        let record = self
312            .backend
313            .promise_state(id)
314            .await?
315            .ok_or(DurableError::UnknownPromise)?;
316        if !record.resolved {
317            return Ok(None);
318        }
319        let sealed = record.payload.ok_or(DurableError::Decode {
320            context: "resolved promise is missing its payload",
321        })?;
322        let plaintext = self
323            .backend
324            .open_promise_payload(id, record.execution_id, &sealed)?;
325        deserialize_result(&plaintext).map(Some)
326    }
327
328    /// Durably sleep until `due`, surviving a process restart (FR-DE-06).
329    ///
330    /// Arms a `durable_timers` row at a deterministic position (so a resume re-attaches to it),
331    /// then parks until the instant arrives — firing the timer itself when due, or returning at once
332    /// if a restart finds it already fired or past due. The
333    /// [`DurableTimerService`](crate::DurableTimerService), when running, fires due timers and wakes
334    /// the waiter; without it, this loop still makes progress on its own.
335    ///
336    /// # Errors
337    ///
338    /// - [`DurableError::StepCapExceeded`] if the timer would exceed the per-execution step cap.
339    /// - A storage error if the timer cannot be armed or its state read.
340    #[tracing::instrument(
341        name = "durable.context.sleep_until",
342        skip_all,
343        fields(execution_id = %self.execution_id.as_uuid())
344    )]
345    pub async fn sleep_until(&self, due: SystemTime) -> Result<(), DurableError> {
346        let step_id = self.checked_step_id()?;
347        let timer_id = TimerId::derive(self.execution_id, step_id);
348        let due_ms = system_time_to_millis(due);
349
350        match self.backend.timer_state(timer_id).await? {
351            // Fired during downtime (or earlier in this run) → return immediately (FR-DE-06).
352            Some((_, true)) => return Ok(()),
353            // Already armed in a prior run: re-attach without re-arming.
354            Some((_, false)) => {}
355            // First execution at this position: arm it.
356            None => {
357                self.backend
358                    .arm_timer(timer_id, self.execution_id, due_ms, now_unix_millis())
359                    .await?;
360            }
361        }
362
363        let key = timer_id.as_uuid();
364        wait_on_notify_or_poll(
365            self.backend.timer_waiters(),
366            key,
367            None,
368            || {
369                let remaining =
370                    u64::try_from(due_ms.saturating_sub(now_unix_millis())).unwrap_or(u64::MAX);
371                self.poll_interval.min(Duration::from_millis(remaining))
372            },
373            // Cheap, clock-only pre-register check: no database read (the common case while
374            // parked and not yet due).
375            || self.check_timer_due(timer_id, due_ms),
376            || async move {
377                // Race-closing recheck after registering: a database read catches a resolution
378                // (e.g. by DurableTimerService) that lands in the window between the clock check
379                // above and the wait below, even if our own clock has not yet reached `due_ms`.
380                if let Some(value) = self.check_timer_due(timer_id, due_ms).await? {
381                    return Ok(Some(value));
382                }
383                if matches!(self.backend.timer_state(timer_id).await?, Some((_, true))) {
384                    return Ok(Some(()));
385                }
386                Ok(None)
387            },
388        )
389        .await
390    }
391
392    /// If `due_ms` has passed, fire the timer (idempotent) and report it resolved; otherwise `None`.
393    ///
394    /// A purely clock-based check with no database read on the not-yet-due path — the cheap
395    /// pre-registration check in [`sleep_until`](Self::sleep_until)'s wait loop.
396    async fn check_timer_due(
397        &self,
398        timer_id: TimerId,
399        due_ms: i64,
400    ) -> Result<Option<()>, DurableError> {
401        if now_unix_millis() >= due_ms {
402            // Due: fire it (idempotent — the service may race us; `WHERE fired = 0` dedups).
403            self.backend.mark_timer_fired(timer_id).await?;
404            return Ok(Some(()));
405        }
406        Ok(None)
407    }
408
409    /// Build an out-of-band [`DurableHandle`](crate::DurableHandle) over this context's backend.
410    ///
411    /// The handle is the operator/A2A resolution surface; it MUST NOT be exposed to an LLM tool
412    /// (INV-9). It shares the same backend, so a resolution it commits wakes an
413    /// [`await_promise`](Self::await_promise) parked on the same process at once.
414    #[must_use]
415    pub fn resolver_handle(&self) -> crate::promise::DurableHandle {
416        crate::promise::DurableHandle::new(self.backend.clone())
417    }
418
419    /// Await any in-flight background checkpoint folds — a turn-boundary / test barrier.
420    ///
421    /// The soft step-cap fold runs on a spawned task so it never blocks step dispatch; call this at
422    /// a turn boundary to ensure the journal is compacted before the next phase observes it.
423    #[tracing::instrument(name = "durable.context.drain_background", skip_all, fields(execution_id = %self.execution_id.as_uuid()))]
424    pub async fn drain_background(&self) {
425        let mut set = {
426            let mut guard = self
427                .fold_tasks
428                .lock()
429                .unwrap_or_else(std::sync::PoisonError::into_inner);
430            std::mem::take(&mut *guard)
431        };
432        while set.join_next().await.is_some() {}
433    }
434
435    /// Assign the next deterministic step id (INV-2).
436    fn assign_step_id(&self) -> StepId {
437        StepId::new(self.next_step.fetch_add(1, Ordering::Relaxed))
438    }
439
440    /// Assign the next step id, rejecting it if it exceeds the per-execution step cap.
441    fn checked_step_id(&self) -> Result<StepId, DurableError> {
442        let step_id = self.assign_step_id();
443        if self.max_steps_per_execution != 0 && step_id.value() >= self.max_steps_per_execution {
444            return Err(DurableError::StepCapExceeded {
445                cap: self.max_steps_per_execution,
446            });
447        }
448        Ok(step_id)
449    }
450
451    /// Fold a checkpoint on a background task the first time a step crosses the soft cap.
452    ///
453    /// Compaction NEVER runs on the dispatch hot path (spec NEVER), so the fold is spawned and
454    /// tracked in `fold_tasks` (drainable via [`drain_background`](Self::drain_background), aborted
455    /// on drop). It fires exactly once per execution; the hard cap aborts any execution that keeps
456    /// growing past it.
457    fn maybe_checkpoint(&self, step_id: StepId) {
458        if step_id.value() < self.soft_step_cap {
459            return;
460        }
461        if self.checkpoint_requested.swap(true, Ordering::AcqRel) {
462            return;
463        }
464        let backend = self.backend.clone();
465        let execution_id = self.execution_id;
466        let up_to = self.soft_step_cap;
467        let mut guard = self
468            .fold_tasks
469            .lock()
470            .unwrap_or_else(std::sync::PoisonError::into_inner);
471        guard.spawn(async move {
472            match backend.checkpoint_fold(execution_id, up_to).await {
473                Ok(folded) => {
474                    tracing::info!(
475                        execution_id = %execution_id.as_uuid(),
476                        folded,
477                        "durable checkpoint fold compacted the idempotent prefix"
478                    );
479                }
480                Err(error) => {
481                    tracing::warn!(%error, "durable checkpoint fold failed");
482                }
483            }
484        });
485    }
486
487    /// Whether replay is currently consulted (a resume that has not diverged).
488    fn replay_active(&self) -> bool {
489        self.is_resume && !self.diverged.load(Ordering::Acquire)
490    }
491
492    /// The core step state machine shared by the sequential and parallel entry points.
493    async fn run_step_at<T, F, Fut>(
494        &self,
495        step_id: StepId,
496        desc: StepDescriptor,
497        op: F,
498    ) -> Result<DurableStep<T>, DurableError>
499    where
500        T: Serialize + DeserializeOwned + Send,
501        F: FnOnce(StepHandle) -> Fut + Send,
502        Fut: Future<Output = Result<T, StepError>> + Send,
503    {
504        if self.max_steps_per_execution != 0 && step_id.value() >= self.max_steps_per_execution {
505            return Err(DurableError::StepCapExceeded {
506                cap: self.max_steps_per_execution,
507            });
508        }
509        // Soft cap (90%): fold a checkpoint on a background task, once per execution.
510        self.maybe_checkpoint(step_id);
511        let effect = desc.effect();
512        let idem_key =
513            IdempotencyKey::derive(self.execution_id, step_id, &desc.fingerprint_input());
514
515        let span = tracing::info_span!(
516            "durable.step.run",
517            step_id = step_id.value(),
518            effect_class = effect.as_str(),
519            replayed = tracing::field::Empty,
520        );
521        async move {
522            // 1) Sequential replay: consult the cursor for this position.
523            if self.replay_active() {
524                match self.cursor.lookup(step_id).await? {
525                    StepReplay::Result(entry) => {
526                        self.check_divergence(step_id, idem_key, &entry).await?;
527                        let value = replay_value::<T>(step_id, effect, &entry)?;
528                        tracing::Span::current().record("replayed", true);
529                        return Ok(DurableStep::replayed(step_id, idem_key, value));
530                    }
531                    StepReplay::IntentOnly(entry) => {
532                        self.check_divergence(step_id, idem_key, &entry).await?;
533                        return self.resolve_ambiguous(step_id, idem_key, &desc, op).await;
534                    }
535                    StepReplay::Fresh => {}
536                }
537            }
538
539            // 2) INV-13: a guarded effect that already committed a result must not re-fire, even on a
540            // fresh run that follows a divergence. A point lookup by idempotency key catches it.
541            if effect == EffectClass::ExactlyOnceGuarded
542                && let Some(entry) = self
543                    .backend
544                    .lookup_committed_result(self.execution_id, idem_key)
545                    .await?
546            {
547                let value = replay_value::<T>(step_id, effect, &entry)?;
548                tracing::Span::current().record("replayed", true);
549                return Ok(DurableStep::replayed(step_id, idem_key, value));
550            }
551
552            // 3) Fresh execution of the step.
553            tracing::Span::current().record("replayed", false);
554            if effect == EffectClass::ExactlyOnceGuarded {
555                // FR-DE-04: the intent is committed and ACKed before the effect fires.
556                let intent = self.intent_entry(step_id, idem_key, effect);
557                self.append_acked_degrading(intent, desc.name()).await?;
558            }
559            let value = self.run_op(op, step_id, idem_key, desc.name()).await?;
560            // Serialize before the journal await so no `&T` is held across it (that would force a
561            // `T: Sync` bound the consumer's value need not satisfy); the owned value moves into the
562            // returned record afterward.
563            let payload = serialize_result(&value, desc.name())?;
564            self.journal_result(payload, step_id, idem_key, effect, desc.name())
565                .await?;
566            Ok(DurableStep::live(step_id, idem_key, value))
567        }
568        .instrument(span)
569        .await
570    }
571
572    /// Compare a journaled step's fingerprint against the current descriptor's; abort on mismatch.
573    #[tracing::instrument(name = "durable.context.check_divergence", skip_all, fields(step_id = step_id.value()))]
574    async fn check_divergence(
575        &self,
576        step_id: StepId,
577        expected: IdempotencyKey,
578        entry: &JournalEntry,
579    ) -> Result<(), DurableError> {
580        // The idempotency key folds in the descriptor name, effect, and op fingerprint, so equality
581        // is the one-BLAKE3-compare fingerprint check the divergence guard requires (INV-3).
582        if entry.entry.idempotency_key() == Some(expected) {
583            return Ok(());
584        }
585        self.on_divergence(step_id).await;
586        Err(DurableError::ReplayDivergence { step_id })
587    }
588
589    /// Mark the execution aborted and disable replay so it restarts fresh (FR-DE-03).
590    #[tracing::instrument(name = "durable.context.on_divergence", skip_all, fields(step_id = step_id.value(), execution_id = %self.execution_id.as_uuid()))]
591    async fn on_divergence(&self, step_id: StepId) {
592        self.diverged.store(true, Ordering::Release);
593        tracing::warn!(
594            execution_id = %self.execution_id.as_uuid(),
595            step_id = step_id.value(),
596            "replay divergence detected; marking journal aborted and restarting fresh"
597        );
598        if let Err(error) = self
599            .backend
600            .finalize(self.execution_id, ExecutionStatus::Aborted)
601            .await
602        {
603            tracing::warn!(%error, "failed to mark diverged execution aborted");
604        }
605    }
606
607    /// Apply the [`OnAmbiguous`] policy for a guarded step resumed in the ambiguous window.
608    #[tracing::instrument(name = "durable.context.resolve_ambiguous", skip_all, fields(step_id = step_id.value(), execution_id = %self.execution_id.as_uuid()))]
609    async fn resolve_ambiguous<T, F, Fut>(
610        &self,
611        step_id: StepId,
612        idem_key: IdempotencyKey,
613        desc: &StepDescriptor,
614        op: F,
615    ) -> Result<DurableStep<T>, DurableError>
616    where
617        T: Serialize + DeserializeOwned + Send,
618        F: FnOnce(StepHandle) -> Fut + Send,
619        Fut: Future<Output = Result<T, StepError>> + Send,
620    {
621        let effect = desc.effect();
622        let policy = desc.on_ambiguous().unwrap_or(OnAmbiguous::Fail);
623        self.emit_ambiguous_audit(step_id, effect, idem_key, policy);
624        match policy {
625            // Fail: refuse to guess; surface the irreversible-effect uncertainty to the operator.
626            OnAmbiguous::Fail => Err(DurableError::AmbiguousEffect { step_id }),
627            // Skip re-runs the closure trusting the boundary to deduplicate the re-issued effect by
628            // its idempotency key; Rerun re-runs it assuming the effect never fired. At this layer
629            // both re-execute (the intent already exists, so it is not re-journaled) and the audit
630            // record above distinguishes which policy was applied.
631            OnAmbiguous::Skip | OnAmbiguous::Rerun => {
632                let value = self.run_op(op, step_id, idem_key, desc.name()).await?;
633                let payload = serialize_result(&value, desc.name())?;
634                self.journal_result(payload, step_id, idem_key, effect, desc.name())
635                    .await?;
636                Ok(DurableStep::live(step_id, idem_key, value))
637            }
638        }
639    }
640
641    /// Invoke the operation closure, mapping its failure to [`DurableError::StepFailed`].
642    #[tracing::instrument(name = "durable.context.run_op", skip_all, fields(step_id = step_id.value(), step_name = name))]
643    async fn run_op<T, F, Fut>(
644        &self,
645        op: F,
646        step_id: StepId,
647        idem_key: IdempotencyKey,
648        name: &'static str,
649    ) -> Result<T, DurableError>
650    where
651        F: FnOnce(StepHandle) -> Fut + Send,
652        Fut: Future<Output = Result<T, StepError>> + Send,
653    {
654        let handle = StepHandle::new(step_id, idem_key);
655        op(handle)
656            .await
657            .map_err(|err| DurableError::step_failed(name, err))
658    }
659
660    /// Journal a step's already-serialized result with the durability class its effect requires.
661    #[tracing::instrument(name = "durable.context.journal_result", skip_all, fields(step_id = step_id.value(), effect_class = effect.as_str(), step_name = name))]
662    async fn journal_result(
663        &self,
664        payload: bytes::Bytes,
665        step_id: StepId,
666        idem_key: IdempotencyKey,
667        effect: EffectClass,
668        name: &'static str,
669    ) -> Result<(), DurableError> {
670        // INV-11 write-side guard: reject an oversized payload before it reaches the writer.
671        crate::cipher::ensure_payload_within_limit(payload.len(), self.max_payload_bytes)?;
672        let entry = JournalEntry {
673            seq: None,
674            execution_id: self.execution_id,
675            kind: self.kind,
676            step_id,
677            entry: EntryKind::StepResult {
678                idempotency_key: idem_key,
679                payload,
680                effect,
681                payload_version: PAYLOAD_VERSION,
682            },
683            created_at_ms: now_unix_millis(),
684        };
685        match effect {
686            // Exactly-once results are ACKed so durability-on-return holds (FR-DE-04).
687            EffectClass::ExactlyOnceGuarded => self.append_acked_degrading(entry, name).await,
688            // Buffered results group-commit; a crash before the flush simply re-runs the step.
689            EffectClass::Idempotent | EffectClass::AtLeastOnce => {
690                self.writer.append_buffered(entry);
691                Ok(())
692            }
693        }
694    }
695
696    /// Build an `EffectIntent` entry for a guarded step.
697    fn intent_entry(
698        &self,
699        step_id: StepId,
700        idem_key: IdempotencyKey,
701        effect: EffectClass,
702    ) -> JournalEntry {
703        JournalEntry {
704            seq: None,
705            execution_id: self.execution_id,
706            kind: self.kind,
707            step_id,
708            entry: EntryKind::EffectIntent {
709                idempotency_key: idem_key,
710                effect,
711                // The backend is the HMAC keyholder and stamps the row HMAC itself when configured.
712                hmac: None,
713            },
714            created_at_ms: now_unix_millis(),
715        }
716    }
717
718    /// ACK an append, degrading to non-durable mode on a writer timeout (INV-12) rather than failing.
719    #[tracing::instrument(name = "durable.context.append_acked_degrading", skip_all, fields(step_name = name))]
720    async fn append_acked_degrading(
721        &self,
722        entry: JournalEntry,
723        name: &'static str,
724    ) -> Result<(), DurableError> {
725        match self.writer.append_acked(entry).await {
726            Ok(_) => Ok(()),
727            Err(DurableError::JournalUnavailable) => {
728                tracing::warn!(
729                    step = name,
730                    "journal writer unavailable; this step degrades to non-durable mode"
731                );
732                metrics::counter!("durable.journal.writer.degraded_appends_total").increment(1);
733                Ok(())
734            }
735            Err(error) => Err(error),
736        }
737    }
738
739    /// Emit the mandatory structured audit record for an ambiguous-window resolution (FR-DE-10).
740    fn emit_ambiguous_audit(
741        &self,
742        step_id: StepId,
743        effect: EffectClass,
744        idem_key: IdempotencyKey,
745        policy: OnAmbiguous,
746    ) {
747        tracing::warn!(
748            target: "durable.audit",
749            execution_id = %self.execution_id.as_uuid(),
750            step_id = step_id.value(),
751            effect_class = effect.as_str(),
752            idem_key = %idem_key_hex8(idem_key),
753            on_ambiguous = policy.as_str(),
754            "durable step resumed in the ambiguous window; applying on_ambiguous policy"
755        );
756    }
757}
758
759/// A handle for spawning durable steps with eagerly-assigned, contiguous step ids.
760///
761/// Returned by [`DurableContext::parallel`]. Each [`step`](ParallelScope::step) call assigns its id
762/// synchronously, *before* returning the future, so building a batch of children fixes their ids in
763/// construction order — completion order is then irrelevant (INV-2).
764#[derive(Debug, Clone, Copy)]
765pub struct ParallelScope<'a> {
766    ctx: &'a DurableContext,
767}
768
769impl<'a> ParallelScope<'a> {
770    /// Construct a durable step future with its id assigned eagerly.
771    ///
772    /// The returned future runs (or replays) the step when awaited; its [`StepId`] is already fixed.
773    /// Collect the futures synchronously, then await them concurrently.
774    ///
775    /// # Errors
776    ///
777    /// The awaited future fails for the same reasons as [`DurableContext::step`].
778    pub fn step<T, F, Fut>(
779        &self,
780        desc: StepDescriptor,
781        op: F,
782    ) -> impl Future<Output = Result<DurableStep<T>, DurableError>> + Send + 'a
783    where
784        T: Serialize + DeserializeOwned + Send + 'a,
785        F: FnOnce(StepHandle) -> Fut + Send + 'a,
786        Fut: Future<Output = Result<T, StepError>> + Send + 'a,
787    {
788        let step_id = self.ctx.assign_step_id();
789        let ctx = self.ctx;
790        async move { ctx.run_step_at(step_id, desc, op).await }
791    }
792}
793
794/// Extract and decode a replayed step's value from its journaled `StepResult`.
795fn replay_value<T: DeserializeOwned>(
796    step_id: StepId,
797    effect: EffectClass,
798    entry: &JournalEntry,
799) -> Result<T, DurableError> {
800    let _span = tracing::info_span!(
801        "durable.step.replay",
802        step_id = step_id.value(),
803        effect_class = effect.as_str(),
804    )
805    .entered();
806    match &entry.entry {
807        EntryKind::StepResult { payload, .. } => deserialize_result(payload),
808        _ => Err(DurableError::Decode {
809            context: "replayed entry is not a step result",
810        }),
811    }
812}
813
814/// Convert a [`SystemTime`] to Unix epoch milliseconds, clamped into `i64` and never panicking.
815///
816/// A pre-epoch instant clamps to `0`; an overflowing one clamps to [`i64::MAX`].
817fn system_time_to_millis(time: SystemTime) -> i64 {
818    time.duration_since(SystemTime::UNIX_EPOCH)
819        .map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX))
820}
821
822/// Hex-encode the first 8 bytes of an idempotency key for an audit record.
823///
824/// The key is a BLAKE3 hash, not secret material; the spec redaction rule shows only its first 8
825/// bytes in CLI/audit output (INV-5).
826fn idem_key_hex8(key: IdempotencyKey) -> String {
827    let mut out = String::with_capacity(16);
828    for byte in &key.as_bytes()[..8] {
829        let _ = write!(out, "{byte:02x}");
830    }
831    out
832}
833
834#[cfg(all(test, feature = "sqlite"))]
835mod tests {
836    use std::assert_matches;
837
838    use super::*;
839    use crate::backend::local::LocalBackend;
840    use crate::config::DurableConfig;
841    use crate::effect::EffectIntentSubClass;
842    use crate::timer::DurableTimerService;
843    use crate::writer::JournalWriter;
844    use std::pin::Pin;
845    use std::sync::atomic::AtomicU32;
846    use tokio::task::JoinHandle;
847
848    /// A type-erased durable-step future, so a heterogeneous batch of step closures can share one
849    /// `Vec` for `join_all` (distinct closures otherwise produce distinct opaque future types).
850    type StepFut<'a> =
851        Pin<Box<dyn Future<Output = Result<DurableStep<u32>, DurableError>> + Send + 'a>>;
852
853    fn fast_config() -> DurableConfig {
854        DurableConfig {
855            journal_flush_interval_ms: 5,
856            journal_ack_timeout_ms: 2000,
857            ..DurableConfig::default()
858        }
859    }
860
861    /// A running context over a fresh in-memory backend, with the writer task spawned.
862    struct Harness {
863        ctx: DurableContext,
864        backend: Arc<LocalBackend>,
865        writer_task: JoinHandle<()>,
866        handle: JournalWriterHandle,
867    }
868
869    impl Harness {
870        async fn open(exec: ExecutionId, is_resume: bool) -> Self {
871            let local = Arc::new(LocalBackend::open(":memory:", 1_048_576).await.unwrap());
872            local.init().await.unwrap();
873            local
874                .open_execution(exec, ExecutionKind::AgentTurn)
875                .await
876                .unwrap();
877            let (writer, handle) = JournalWriter::new(local.clone(), &fast_config());
878            let writer_task = tokio::spawn(writer.run());
879            let backend = Arc::new(DurableBackendEnum::Local(local.clone()));
880            let ctx = DurableContext::new(
881                exec,
882                ExecutionKind::AgentTurn,
883                is_resume,
884                backend,
885                handle.clone(),
886                &fast_config(),
887            );
888            Self {
889                ctx,
890                backend: local,
891                writer_task,
892                handle,
893            }
894        }
895
896        /// Reopen over the *same* backing journal to drive a resume run.
897        fn resume(&self) -> DurableContext {
898            let backend = Arc::new(DurableBackendEnum::Local(self.backend.clone()));
899            DurableContext::new(
900                self.ctx.execution_id,
901                ExecutionKind::AgentTurn,
902                true,
903                backend,
904                self.handle.clone(),
905                &fast_config(),
906            )
907        }
908
909        async fn shutdown(self) {
910            // Some tests build a second context (a resume / fresh run) that clones the writer
911            // handle; that clone keeps the channel open, so the writer never stops on its own.
912            // Abort it directly rather than awaiting a graceful drain (data was already flushed
913            // before any assertion that needed it).
914            self.writer_task.abort();
915            let _ = self.writer_task.await;
916        }
917    }
918
919    #[tokio::test]
920    async fn fresh_step_runs_op_and_journals_result() {
921        // FR-DE-01: a fresh step records its result in the journal.
922        let exec = ExecutionId::new();
923        let h = Harness::open(exec, false).await;
924        let value: u32 = h
925            .ctx
926            .step(
927                StepDescriptor::idempotent("count", b"op".to_vec()),
928                |_| async { Ok(7) },
929            )
930            .await
931            .unwrap();
932        assert_eq!(value, 7);
933        h.handle.flush().await.unwrap();
934
935        let entries = h.backend.read_execution(exec).await.unwrap();
936        assert_eq!(entries.len(), 1);
937        assert_matches!(entries[0].entry, EntryKind::StepResult { .. });
938        h.shutdown().await;
939    }
940
941    #[tokio::test]
942    async fn replayed_idempotent_step_skips_op() {
943        // INV-10 / FR-DE-02: a replayed idempotent step returns the journaled value without re-running.
944        let exec = ExecutionId::new();
945        let h = Harness::open(exec, false).await;
946        let desc = || StepDescriptor::idempotent("count", b"op".to_vec());
947        let first: u32 = h.ctx.step(desc(), |_| async { Ok(11) }).await.unwrap();
948        assert_eq!(first, 11);
949        h.handle.flush().await.unwrap();
950
951        let resumed = h.resume();
952        let ran_again = Arc::new(AtomicU32::new(0));
953        let counter = ran_again.clone();
954        let replayed: u32 = resumed
955            .step(desc(), move |_| {
956                let counter = counter.clone();
957                async move {
958                    counter.fetch_add(1, Ordering::SeqCst);
959                    Ok(999)
960                }
961            })
962            .await
963            .unwrap();
964        assert_eq!(
965            replayed, 11,
966            "the journaled value is returned, not the new one"
967        );
968        assert_eq!(
969            ran_again.load(Ordering::SeqCst),
970            0,
971            "the operation closure must not run on replay"
972        );
973        h.shutdown().await;
974    }
975
976    #[tokio::test]
977    async fn guarded_step_commits_intent_before_result() {
978        // FR-DE-04: an EffectIntent is committed before op, a StepResult after.
979        let exec = ExecutionId::new();
980        let h = Harness::open(exec, false).await;
981        let desc = StepDescriptor::exactly_once_guarded(
982            "charge",
983            EffectIntentSubClass::CostBearingOrBoundaryIdempotent,
984            Some(OnAmbiguous::Skip),
985            b"op".to_vec(),
986        )
987        .unwrap();
988        let _: u32 = h.ctx.step(desc, |_| async { Ok(5) }).await.unwrap();
989        h.handle.flush().await.unwrap();
990
991        let entries = h.backend.read_execution(exec).await.unwrap();
992        let kinds: Vec<_> = entries.iter().map(|e| e.entry.tag()).collect();
993        assert_eq!(
994            kinds,
995            vec!["effect_intent", "step_result"],
996            "intent is journaled before the result"
997        );
998        h.shutdown().await;
999    }
1000
1001    #[tokio::test]
1002    async fn replay_divergence_on_fingerprint_mismatch() {
1003        // INV-3 / FR-DE-03: a structurally different step at the same id aborts and restarts fresh.
1004        let exec = ExecutionId::new();
1005        let h = Harness::open(exec, false).await;
1006        let _: u32 = h
1007            .ctx
1008            .step(
1009                StepDescriptor::idempotent("count", b"v1".to_vec()),
1010                |_| async { Ok(1) },
1011            )
1012            .await
1013            .unwrap();
1014        h.handle.flush().await.unwrap();
1015
1016        let resumed = h.resume();
1017        // Same step position, different op fingerprint → different structural fingerprint.
1018        let err = resumed
1019            .step::<u32, _, _>(
1020                StepDescriptor::idempotent("count", b"v2".to_vec()),
1021                |_| async { Ok(2) },
1022            )
1023            .await
1024            .unwrap_err();
1025        assert_matches!(err, DurableError::ReplayDivergence { .. });
1026
1027        let (status,): (String,) = zeph_db::query_as(zeph_db::sql!(
1028            "SELECT status FROM durable_executions WHERE execution_id = ?"
1029        ))
1030        .bind(exec.as_uuid().to_string())
1031        .fetch_one(h.backend.pool())
1032        .await
1033        .unwrap();
1034        assert_eq!(status, "aborted", "the diverged journal is marked aborted");
1035        h.shutdown().await;
1036    }
1037
1038    #[tokio::test]
1039    async fn ambiguous_window_fail_policy_surfaces_error() {
1040        // FR-DE-14 path: an intent without a result, policy = Fail, must not re-fire.
1041        let exec = ExecutionId::new();
1042        let h = Harness::open(exec, false).await;
1043        let step_id = StepId::new(0);
1044        let idem = IdempotencyKey::derive(
1045            exec,
1046            step_id,
1047            &StepDescriptor::exactly_once_guarded(
1048                "delete",
1049                EffectIntentSubClass::Destructive,
1050                Some(OnAmbiguous::Fail),
1051                b"op".to_vec(),
1052            )
1053            .unwrap()
1054            .fingerprint_input(),
1055        );
1056        // Seed only the intent (the crash happened before the result committed).
1057        h.backend
1058            .append(JournalEntry {
1059                seq: None,
1060                execution_id: exec,
1061                kind: ExecutionKind::AgentTurn,
1062                step_id,
1063                entry: EntryKind::EffectIntent {
1064                    idempotency_key: idem,
1065                    effect: EffectClass::ExactlyOnceGuarded,
1066                    hmac: None,
1067                },
1068                created_at_ms: 0,
1069            })
1070            .await
1071            .unwrap();
1072
1073        let resumed = h.resume();
1074        let ran = Arc::new(AtomicU32::new(0));
1075        let counter = ran.clone();
1076        let err = resumed
1077            .step::<u32, _, _>(
1078                StepDescriptor::exactly_once_guarded(
1079                    "delete",
1080                    EffectIntentSubClass::Destructive,
1081                    Some(OnAmbiguous::Fail),
1082                    b"op".to_vec(),
1083                )
1084                .unwrap(),
1085                move |_| {
1086                    let counter = counter.clone();
1087                    async move {
1088                        counter.fetch_add(1, Ordering::SeqCst);
1089                        Ok(1)
1090                    }
1091                },
1092            )
1093            .await
1094            .unwrap_err();
1095        assert_matches!(err, DurableError::AmbiguousEffect { .. });
1096        assert_eq!(
1097            ran.load(Ordering::SeqCst),
1098            0,
1099            "a fail-policy ambiguous step must not re-fire the effect"
1100        );
1101        h.shutdown().await;
1102    }
1103
1104    #[tokio::test]
1105    async fn inv13_committed_guarded_result_is_not_refired() {
1106        // INV-13: on a fresh run after divergence, an already-committed guarded result is returned
1107        // via its idempotency key without re-firing.
1108        let exec = ExecutionId::new();
1109        let h = Harness::open(exec, false).await;
1110        let desc = || {
1111            StepDescriptor::exactly_once_guarded(
1112                "transfer",
1113                EffectIntentSubClass::MoneyMoving,
1114                Some(OnAmbiguous::Fail),
1115                b"op".to_vec(),
1116            )
1117            .unwrap()
1118        };
1119        let first: u32 = h.ctx.step(desc(), |_| async { Ok(500) }).await.unwrap();
1120        assert_eq!(first, 500);
1121        h.handle.flush().await.unwrap();
1122
1123        // A "fresh run after divergence": replay is OFF, but the guarded point lookup must still find
1124        // the committed result. Build a non-resume context over the same journal.
1125        let backend = Arc::new(DurableBackendEnum::Local(h.backend.clone()));
1126        let fresh = DurableContext::new(
1127            exec,
1128            ExecutionKind::AgentTurn,
1129            false,
1130            backend,
1131            h.handle.clone(),
1132            &fast_config(),
1133        );
1134        let ran = Arc::new(AtomicU32::new(0));
1135        let counter = ran.clone();
1136        let value: u32 = fresh
1137            .step(desc(), move |_| {
1138                let counter = counter.clone();
1139                async move {
1140                    counter.fetch_add(1, Ordering::SeqCst);
1141                    Ok(0)
1142                }
1143            })
1144            .await
1145            .unwrap();
1146        assert_eq!(value, 500, "the pre-committed guarded result is returned");
1147        assert_eq!(
1148            ran.load(Ordering::SeqCst),
1149            0,
1150            "the guarded effect must not re-fire"
1151        );
1152        h.shutdown().await;
1153    }
1154
1155    #[tokio::test]
1156    async fn parallel_step_ids_are_completion_order_independent() {
1157        // INV-2: a parallel batch with shuffled completion order yields deterministic step ids.
1158        let exec = ExecutionId::new();
1159        let h = Harness::open(exec, false).await;
1160        let scope = h.ctx.parallel();
1161        // Construct children in argument order; each gets its id eagerly at the `scope.step` call
1162        // (before any future is polled). Box them so the differently-typed closures share one Vec.
1163        let futures: Vec<StepFut> = vec![
1164            Box::pin(scope.step::<u32, _, _>(
1165                StepDescriptor::idempotent("a", b"a".to_vec()),
1166                |handle: StepHandle| async move {
1167                    // Finishes last despite being constructed first.
1168                    tokio::time::sleep(std::time::Duration::from_millis(30)).await;
1169                    Ok(handle.step_id().value())
1170                },
1171            )),
1172            Box::pin(scope.step::<u32, _, _>(
1173                StepDescriptor::idempotent("b", b"b".to_vec()),
1174                |handle: StepHandle| async move {
1175                    tokio::time::sleep(std::time::Duration::from_millis(5)).await;
1176                    Ok(handle.step_id().value())
1177                },
1178            )),
1179            Box::pin(scope.step::<u32, _, _>(
1180                StepDescriptor::idempotent("c", b"c".to_vec()),
1181                |handle: StepHandle| async move { Ok(handle.step_id().value()) },
1182            )),
1183        ];
1184        let results = futures::future::try_join_all(futures).await.unwrap();
1185        let ids: Vec<u32> = results
1186            .iter()
1187            .map(DurableStep::step_id)
1188            .map(StepId::value)
1189            .collect();
1190        // Each child observed the id assigned at construction, regardless of completion order.
1191        assert_eq!(ids, vec![0, 1, 2]);
1192        h.shutdown().await;
1193    }
1194
1195    #[tokio::test]
1196    async fn concurrent_steps_under_shared_ref_are_sound() {
1197        // System-invariants §10: concurrent step() calls under a single &self assign unique ids and
1198        // all journal successfully.
1199        let exec = ExecutionId::new();
1200        let h = Harness::open(exec, false).await;
1201        let scope = h.ctx.parallel();
1202        let futures: Vec<StepFut> = (0..16)
1203            .map(|i| {
1204                Box::pin(scope.step::<u32, _, _>(
1205                    StepDescriptor::idempotent("worker", format!("op:{i}").into_bytes()),
1206                    move |handle: StepHandle| async move { Ok(handle.step_id().value()) },
1207                )) as StepFut
1208            })
1209            .collect();
1210        let results = futures::future::try_join_all(futures).await.unwrap();
1211        let mut ids: Vec<u32> = results
1212            .iter()
1213            .map(DurableStep::step_id)
1214            .map(StepId::value)
1215            .collect();
1216        ids.sort_unstable();
1217        ids.dedup();
1218        assert_eq!(ids.len(), 16, "all 16 concurrent steps got unique ids");
1219        h.handle.flush().await.unwrap();
1220        assert_eq!(h.backend.read_execution(exec).await.unwrap().len(), 16);
1221        h.shutdown().await;
1222    }
1223
1224    #[tokio::test]
1225    async fn op_failure_surfaces_as_step_failed_without_journaling() {
1226        let exec = ExecutionId::new();
1227        let h = Harness::open(exec, false).await;
1228        let err = h
1229            .ctx
1230            .step::<u32, _, _>(
1231                StepDescriptor::idempotent("boom", b"op".to_vec()),
1232                |_| async { Err(StepError::new("op exploded")) },
1233            )
1234            .await
1235            .unwrap_err();
1236        assert_matches!(err, DurableError::StepFailed { step: "boom", .. });
1237        h.handle.flush().await.unwrap();
1238        assert!(
1239            h.backend.read_execution(exec).await.unwrap().is_empty(),
1240            "a failed step journals no result"
1241        );
1242        h.shutdown().await;
1243    }
1244
1245    /// Build a fresh context over a shared in-memory backend with a custom config (e.g. a small cap).
1246    fn context_with(
1247        local: &Arc<LocalBackend>,
1248        handle: &JournalWriterHandle,
1249        exec: ExecutionId,
1250        is_resume: bool,
1251        config: &DurableConfig,
1252    ) -> DurableContext {
1253        let dispatch = Arc::new(DurableBackendEnum::Local(local.clone()));
1254        DurableContext::new(
1255            exec,
1256            ExecutionKind::AgentTurn,
1257            is_resume,
1258            dispatch,
1259            handle.clone(),
1260            config,
1261        )
1262    }
1263
1264    #[tokio::test]
1265    async fn promise_resolves_with_token_and_await_returns_value() {
1266        // FR-DE-05: a promise resolves only via its token; resolution wakes the parked await.
1267        let exec = ExecutionId::new();
1268        let h = Harness::open(exec, false).await;
1269        let promise = h.ctx.promise::<u32>().await.unwrap();
1270        assert!(!promise.is_resumed());
1271        let token = *promise
1272            .resolver_token()
1273            .expect("fresh promise carries a token");
1274        let id = promise.id();
1275        let resolver = h.ctx.resolver_handle();
1276
1277        let (awaited, ack) = tokio::join!(h.ctx.await_promise::<u32>(promise), async {
1278            tokio::time::sleep(Duration::from_millis(25)).await;
1279            resolver.resolve(id, &token, 1234u32).await
1280        });
1281        ack.unwrap();
1282        assert_eq!(
1283            awaited.unwrap(),
1284            1234,
1285            "the awaiter receives the resolved value"
1286        );
1287        h.shutdown().await;
1288    }
1289
1290    #[tokio::test]
1291    async fn wrong_resolver_token_is_rejected_but_correct_one_resolves() {
1292        // INV-9: resolution requires the matching token; a wrong token is rejected (constant-time).
1293        let exec = ExecutionId::new();
1294        let h = Harness::open(exec, false).await;
1295        let promise = h.ctx.promise::<String>().await.unwrap();
1296        let id = promise.id();
1297        let token = *promise.resolver_token().unwrap();
1298        let resolver = h.ctx.resolver_handle();
1299
1300        // The LLM, lacking the token, cannot resolve: a guessed token is rejected and leaves the
1301        // promise pending.
1302        let mut wrong = token;
1303        wrong[0] ^= 0xFF;
1304        assert_matches!(
1305            resolver.resolve(id, &wrong, "forged".to_string()).await,
1306            Err(DurableError::PromiseRejected)
1307        );
1308        assert!(
1309            !h.backend.promise_state(id).await.unwrap().unwrap().resolved,
1310            "a rejected resolution must not resolve the promise"
1311        );
1312
1313        // The genuine token resolves it.
1314        resolver
1315            .resolve(id, &token, "ok".to_string())
1316            .await
1317            .unwrap();
1318        assert!(h.backend.promise_state(id).await.unwrap().unwrap().resolved);
1319
1320        // Resolving an unknown promise fails closed.
1321        assert_matches!(
1322            resolver
1323                .resolve(PromiseId::new(), &token, "x".to_string())
1324                .await,
1325            Err(DurableError::UnknownPromise)
1326        );
1327        h.shutdown().await;
1328    }
1329
1330    #[tokio::test]
1331    async fn resumed_promise_awaits_the_resolved_value() {
1332        // A promise created and resolved before a crash is re-attached on resume and awaited.
1333        let exec = ExecutionId::new();
1334        let h = Harness::open(exec, false).await;
1335        let promise = h.ctx.promise::<u32>().await.unwrap();
1336        let id = promise.id();
1337        let token = *promise.resolver_token().unwrap();
1338        h.ctx
1339            .resolver_handle()
1340            .resolve(id, &token, 77u32)
1341            .await
1342            .unwrap();
1343
1344        // Resume: promise() at the same position returns a token-less, resumed handle.
1345        let resumed = h.resume();
1346        let promise2 = resumed.promise::<u32>().await.unwrap();
1347        assert!(promise2.is_resumed());
1348        assert_eq!(
1349            promise2.id(),
1350            id,
1351            "the resumed promise re-derives the same id"
1352        );
1353        assert_eq!(resumed.await_promise::<u32>(promise2).await.unwrap(), 77);
1354        h.shutdown().await;
1355    }
1356
1357    #[tokio::test]
1358    async fn sleep_until_returns_when_the_instant_passes() {
1359        let exec = ExecutionId::new();
1360        let h = Harness::open(exec, false).await;
1361        // A near-future instant: the context fires its own timer when due (no service needed).
1362        let due = SystemTime::now() + Duration::from_millis(40);
1363        tokio::time::timeout(Duration::from_secs(2), h.ctx.sleep_until(due))
1364            .await
1365            .expect("sleep_until completes before the test timeout")
1366            .expect("sleep_until succeeds");
1367        h.shutdown().await;
1368    }
1369
1370    #[tokio::test]
1371    async fn sleep_until_wakes_on_concurrent_fire_before_due() {
1372        // An external actor (e.g. DurableTimerService, or a concurrent process) marking the timer
1373        // fired while sleep_until is already parked wakes it at once via the in-process notify,
1374        // rather than waiting out the poll interval or the timer's own due instant.
1375        let exec = ExecutionId::new();
1376        let h = Harness::open(exec, false).await;
1377        // Far enough out that only the notify wakes it, not sleep_until's own due-clock check.
1378        let due = SystemTime::now() + Duration::from_secs(30);
1379        let timer_id = TimerId::derive(exec, StepId::new(0));
1380
1381        let (result, marked) = tokio::join!(
1382            tokio::time::timeout(Duration::from_secs(2), h.ctx.sleep_until(due)),
1383            async {
1384                // Give sleep_until time to arm the timer and register on the notify first.
1385                tokio::time::sleep(Duration::from_millis(25)).await;
1386                h.backend.mark_timer_fired(timer_id).await
1387            }
1388        );
1389        assert!(marked.unwrap(), "the timer transitions to fired");
1390        result
1391            .expect("sleep_until wakes on the concurrent fire before the test timeout")
1392            .expect("sleep_until succeeds");
1393        h.shutdown().await;
1394    }
1395
1396    #[tokio::test]
1397    async fn sleep_until_past_due_returns_immediately_on_resume() {
1398        // FR-DE-06: a timer whose instant elapsed during downtime fires at once on resume.
1399        let exec = ExecutionId::new();
1400        let h = Harness::open(exec, false).await;
1401        // Arm a long-past timer at the position sleep_until will re-derive on resume (step 0).
1402        let timer = TimerId::derive(exec, StepId::new(0));
1403        h.backend.arm_timer(timer, exec, 1_000, 0).await.unwrap();
1404
1405        // The timer service fires the past-due timer on its first poll.
1406        let service = DurableTimerService::new(
1407            Arc::new(DurableBackendEnum::Local(h.backend.clone())),
1408            Duration::from_millis(5),
1409        );
1410        service.fire_due().await;
1411        assert_eq!(
1412            h.backend.timer_state(timer).await.unwrap(),
1413            Some((1_000, true))
1414        );
1415
1416        // A resumed sleep_until at the same position returns immediately (already fired).
1417        let resumed = h.resume();
1418        tokio::time::timeout(
1419            Duration::from_millis(200),
1420            resumed.sleep_until(SystemTime::now() + Duration::from_hours(1)),
1421        )
1422        .await
1423        .expect("resumed sleep_until returns immediately")
1424        .unwrap();
1425        h.shutdown().await;
1426    }
1427
1428    #[tokio::test]
1429    async fn soft_cap_triggers_checkpoint_fold_and_replay_skips_folded_steps() {
1430        // Soft cap (90% of 10 = 9): the step at id 9 folds the idempotent prefix [0..9).
1431        let exec = ExecutionId::new();
1432        let local = Arc::new(LocalBackend::open(":memory:", 1_048_576).await.unwrap());
1433        local.init().await.unwrap();
1434        local
1435            .open_execution(exec, ExecutionKind::AgentTurn)
1436            .await
1437            .unwrap();
1438        let (writer, handle) = JournalWriter::new(local.clone(), &fast_config());
1439        let task = tokio::spawn(writer.run());
1440        let config = DurableConfig {
1441            max_steps_per_execution: 10,
1442            ..fast_config()
1443        };
1444        let ctx = context_with(&local, &handle, exec, false, &config);
1445
1446        let desc = |i: u32| StepDescriptor::idempotent("s", format!("op:{i}").into_bytes());
1447        // Steps 0..=8 run and are committed before the soft-cap step triggers the fold.
1448        for i in 0..9 {
1449            let v: u32 = ctx
1450                .step(desc(i), move |_| async move { Ok(i) })
1451                .await
1452                .unwrap();
1453            assert_eq!(v, i);
1454        }
1455        handle.flush().await.unwrap();
1456        // Step id 9 crosses the soft cap and spawns the background fold of [0..9).
1457        ctx.step::<u32, _, _>(desc(9), |_| async { Ok(9) })
1458            .await
1459            .unwrap();
1460        ctx.drain_background().await;
1461        handle.flush().await.unwrap();
1462
1463        // The folded prefix is compacted into a single checkpoint; steps 9 survives as a row.
1464        let entries = local.read_execution(exec).await.unwrap();
1465        let checkpoints = entries
1466            .iter()
1467            .filter(|e| matches!(e.entry, EntryKind::Checkpoint { .. }))
1468            .count();
1469        assert_eq!(checkpoints, 1, "the soft cap folded one checkpoint");
1470        let surviving: Vec<u32> = entries
1471            .iter()
1472            .filter(|e| matches!(e.entry, EntryKind::StepResult { .. }))
1473            .map(|e| e.step_id.value())
1474            .collect();
1475        assert_eq!(surviving, vec![9], "only the post-fold step row survives");
1476
1477        // Resume: the folded steps replay from the checkpoint without re-running their ops.
1478        let resumed = context_with(&local, &handle, exec, true, &config);
1479        let reran = Arc::new(AtomicU32::new(0));
1480        for i in 0..9 {
1481            let counter = reran.clone();
1482            let v: u32 = resumed
1483                .step(desc(i), move |_| {
1484                    let counter = counter.clone();
1485                    async move {
1486                        counter.fetch_add(1, Ordering::SeqCst);
1487                        Ok(999)
1488                    }
1489                })
1490                .await
1491                .unwrap();
1492            assert_eq!(v, i, "folded step {i} replays its journaled value");
1493        }
1494        assert_eq!(
1495            reran.load(Ordering::SeqCst),
1496            0,
1497            "no folded operation closure re-ran on replay"
1498        );
1499
1500        drop(ctx);
1501        drop(resumed);
1502        drop(handle);
1503        task.await.unwrap();
1504    }
1505
1506    #[tokio::test]
1507    async fn step_cap_is_enforced() {
1508        let exec = ExecutionId::new();
1509        let local = Arc::new(LocalBackend::open(":memory:", 1_048_576).await.unwrap());
1510        local.init().await.unwrap();
1511        local
1512            .open_execution(exec, ExecutionKind::AgentTurn)
1513            .await
1514            .unwrap();
1515        let (writer, handle) = JournalWriter::new(local.clone(), &fast_config());
1516        let task = tokio::spawn(writer.run());
1517        let backend = Arc::new(DurableBackendEnum::Local(local.clone()));
1518        let ctx = DurableContext::new(
1519            exec,
1520            ExecutionKind::AgentTurn,
1521            false,
1522            backend,
1523            handle.clone(),
1524            &DurableConfig {
1525                max_steps_per_execution: 1,
1526                ..fast_config()
1527            },
1528        );
1529        // Step id 0 is allowed; step id 1 exceeds the cap of 1.
1530        ctx.step::<u32, _, _>(
1531            StepDescriptor::idempotent("ok", b"op".to_vec()),
1532            |_| async { Ok(0) },
1533        )
1534        .await
1535        .unwrap();
1536        let err = ctx
1537            .step::<u32, _, _>(
1538                StepDescriptor::idempotent("over", b"op".to_vec()),
1539                |_| async { Ok(0) },
1540            )
1541            .await
1542            .unwrap_err();
1543        assert_matches!(err, DurableError::StepCapExceeded { cap: 1 });
1544        drop(ctx);
1545        drop(handle);
1546        task.await.unwrap();
1547    }
1548}