Skip to main content

zeph_durable/backend/
local.rs

1// SPDX-FileCopyrightText: 2026 Andrei G <bug-ops>
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! The always-compiled local journal backend.
5//!
6//! [`LocalBackend`] owns a dedicated [`zeph_db::DbPool`] on its own `durable.db` file (INV-14): a
7//! separate database keeps the high-write journal off the shared application pool, where
8//! `BEGIN IMMEDIATE` contention would otherwise serialize unrelated writers. The schema lives in
9//! `zeph-db/migrations/{sqlite,postgres}/` and is applied via [`zeph_db::run_migrations`]; the
10//! backend owns no `.sql` files of its own.
11//!
12//! # Sealing and integrity
13//!
14//! Payload-bearing entries (currently [`EntryKind::StepResult`]) are AEAD-sealed through the
15//! injected [`PayloadCipher`] before they touch the database, with the entry's location bound as
16//! associated data so a sealed blob cannot be relocated to another step or execution. Control
17//! entries (currently [`EntryKind::EffectIntent`]) carry no payload; when an HMAC key is configured
18//! the backend stamps a keyed BLAKE3 row HMAC over their identity for shared-database deployments,
19//! and every read recomputes and constant-time-verifies that HMAC, failing closed with
20//! [`DurableError::ControlIntegrity`] on a forged or relocated row. When no cipher is injected the
21//! payload is stored verbatim — a development-only posture gated by
22//! [`encryption_gate`](crate::encryption_gate) at startup.
23//!
24//! # Scope
25//!
26//! This revision journals the step-execution entries — [`EntryKind::StepResult`] and
27//! [`EntryKind::EffectIntent`] — that the durable step primitive records, plus execution
28//! lifecycle (open and [`finalize`](Journal::finalize)), the writer's restart anchor (`max_seq`),
29//! and the idempotency-key point lookup
30//! ([`lookup_committed_result`](ExecutionBackend::lookup_committed_result)) that lets a guarded step
31//! recognize an already-committed effect after a replay divergence (INV-13). Promise, timer, and
32//! checkpoint entries are journaled by the promise/timer and retention layers; until then
33//! [`append`](Journal::append) of those kinds fails closed with
34//! [`DurableError::UnsupportedEntryKind`] rather than dropping their state. The retention sweep
35//! ([`prune`](Journal::prune)) is a no-op stub here.
36
37use std::fmt;
38use std::fmt::Write as _;
39use std::path::PathBuf;
40use std::sync::Arc;
41use std::time::{SystemTime, UNIX_EPOCH};
42
43use bytes::Bytes;
44use zeph_db::{DbPool, sql};
45
46use crate::backend::execution_lock::ExecutionLock;
47use crate::backend::{BackendCapabilities, ExecutionBackend, ExecutionSummary, RedactedEntry};
48use crate::cipher::{EntryKindTag, PayloadAad, PayloadCipher, ensure_payload_within_limit};
49use crate::config::RetentionPolicy;
50use crate::error::DurableError;
51use crate::ids::{
52    ExecutionId, ExecutionKind, IdempotencyKey, JournalSeq, PromiseId, StepId, TimerId,
53};
54use crate::journal::{EntryKind, ExecutionStatus, Journal, JournalEntry};
55use crate::promise::PromiseRecord;
56use crate::retention::{CheckpointSnapshot, FoldedStep, decode_checkpoint, encode_checkpoint};
57use crate::waiters::NotifyRegistry;
58use tracing::Instrument as _;
59
60/// Slack added to `max_payload_bytes` for the read-side size guard.
61///
62/// The stored blob carries AEAD framing (key-id, extended nonce, tag) on top of the plaintext, so a
63/// payload accepted at exactly the limit on write is slightly larger on read. The guard exists only
64/// to reject absurdly large rows before allocation/decryption (INV-11), so a small fixed slack
65/// above any real AEAD overhead keeps legitimate near-limit entries readable without weakening the
66/// denial-of-service protection.
67const SEAL_OVERHEAD_SLACK: u64 = 128;
68
69/// Row shape returned by the `list_executions` query.
70type ExecutionRow = (String, String, String, i64, i64, Option<i64>, i64);
71
72/// Row shape returned by the `read_execution_redacted` query.
73type RedactedRow = (
74    i64,
75    i64,
76    String,
77    Option<Vec<u8>>,
78    Option<String>,
79    Option<i64>,
80    i64,
81);
82
83/// Render the first 8 bytes of an idempotency key as a lowercase hex prefix (INV-5).
84fn idem_key_prefix(bytes: &[u8]) -> String {
85    bytes.iter().take(8).fold(String::new(), |mut acc, b| {
86        let _ = write!(acc, "{b:02x}");
87        acc
88    })
89}
90
91/// The always-compiled durable backend that journals to a dedicated `durable.db`.
92///
93/// Construct it from a [`zeph_db::DbPool`] (or open one with [`LocalBackend::open`]), then attach an
94/// optional [`PayloadCipher`] and HMAC key with the builder methods. Call [`LocalBackend::init`]
95/// once before use to apply the schema migrations.
96///
97/// # Examples
98///
99/// ```no_run
100/// # async fn run() -> Result<(), Box<dyn std::error::Error>> {
101/// use zeph_durable::LocalBackend;
102///
103/// // 1 MiB payload ceiling, matching the spec default.
104/// let backend = LocalBackend::open("durable.db", 1_048_576).await?;
105/// backend.init().await?;
106/// # Ok(()) }
107/// ```
108pub struct LocalBackend {
109    pool: DbPool,
110    cipher: Option<Arc<dyn PayloadCipher>>,
111    hmac_key: Option<[u8; 32]>,
112    max_payload_bytes: u64,
113    /// In-process wakeup map for parked promise awaits, shared with the resolver path.
114    promise_waiters: NotifyRegistry,
115    /// In-process wakeup map for parked timers, shared with the timer service.
116    timer_waiters: NotifyRegistry,
117    /// Directory for per-`ExecutionId` advisory lock files (INV-15, #6122), used by
118    /// [`open_execution_exclusive`](Self::open_execution_exclusive). `None` when no on-disk path
119    /// is known for this backend — a `:memory:` database, a backend built via
120    /// [`LocalBackend::new`] from a caller-supplied pool, or a non-SQLite (Postgres) deployment,
121    /// where a filesystem lock file cannot express cross-process exclusivity anyway.
122    lock_dir: Option<PathBuf>,
123    /// Set once [`sweep_orphans`](Self::sweep_orphans) has emitted its warn-once log for a
124    /// `lock_dir = None` backend (#6254), so a background retention tick every
125    /// `prune_interval_secs` does not spam the log for the lifetime of the process.
126    orphan_sweep_warned: std::sync::atomic::AtomicBool,
127}
128
129impl fmt::Debug for LocalBackend {
130    /// Redacts the cipher and HMAC key — never print key material or a cipher handle.
131    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
132        f.debug_struct("LocalBackend")
133            .field("cipher", &self.cipher.as_ref().map(|_| "<cipher>"))
134            .field("hmac_key", &self.hmac_key.as_ref().map(|_| "<redacted>"))
135            .field("max_payload_bytes", &self.max_payload_bytes)
136            .finish_non_exhaustive()
137    }
138}
139
140impl LocalBackend {
141    /// Wrap an existing [`zeph_db::DbPool`] as a local backend with the given payload ceiling.
142    ///
143    /// Call [`LocalBackend::init`] before any journal operation to apply the schema. Attach a
144    /// cipher and HMAC key with [`with_cipher`](Self::with_cipher) and
145    /// [`with_hmac_key`](Self::with_hmac_key).
146    #[must_use]
147    pub fn new(pool: DbPool, max_payload_bytes: u64) -> Self {
148        Self {
149            pool,
150            cipher: None,
151            hmac_key: None,
152            max_payload_bytes,
153            promise_waiters: NotifyRegistry::default(),
154            timer_waiters: NotifyRegistry::default(),
155            lock_dir: None,
156            orphan_sweep_warned: std::sync::atomic::AtomicBool::new(false),
157        }
158    }
159
160    /// Open (or create) a backend on a dedicated `durable.db` file (or `:memory:`).
161    ///
162    /// Connecting also applies the schema migrations, so a freshly opened backend is ready to use;
163    /// [`init`](Self::init) may still be called and is idempotent.
164    ///
165    /// On the `SQLite` backend, also derives the lock directory used by
166    /// [`open_execution_exclusive`](Self::open_execution_exclusive) from `path` (a sibling
167    /// `<path>.locks/` directory), unless `path` is `:memory:`. The Postgres backend never derives
168    /// one — `path` there is a connection URL (which may embed credentials), not a filesystem path.
169    ///
170    /// # Errors
171    ///
172    /// Returns [`DurableError::Storage`] if the pool cannot be opened or migrations fail.
173    pub async fn open(path: &str, max_payload_bytes: u64) -> Result<Self, DurableError> {
174        let pool = zeph_db::DbConfig {
175            url: path.to_string(),
176            pool_size: 5,
177        }
178        .connect()
179        .await
180        .map_err(|e| DurableError::storage("open", e))?;
181        let mut backend = Self::new(pool, max_payload_bytes);
182        backend.lock_dir = lock_dir_for_path(path);
183        Ok(backend)
184    }
185
186    /// Inject the AEAD payload cipher used to seal and open payload-bearing entries.
187    #[must_use]
188    pub fn with_cipher(mut self, cipher: Arc<dyn PayloadCipher>) -> Self {
189        self.cipher = Some(cipher);
190        self
191    }
192
193    /// Configure the keyed-BLAKE3 HMAC key stamped over control entries on shared-database
194    /// deployments, and used to verify them again on every read (INV-8).
195    #[must_use]
196    pub fn with_hmac_key(mut self, key: [u8; 32]) -> Self {
197        self.hmac_key = Some(key);
198        self
199    }
200
201    /// Borrow the underlying pool (for tests and adapters that need direct access).
202    #[must_use]
203    pub fn pool(&self) -> &DbPool {
204        &self.pool
205    }
206
207    /// Apply the durable schema migrations to the backing pool.
208    ///
209    /// Idempotent: safe to call repeatedly. The schema is owned by `zeph-db`, not this crate.
210    ///
211    /// # Errors
212    ///
213    /// Returns [`DurableError::Storage`] if a migration fails.
214    pub async fn init(&self) -> Result<(), DurableError> {
215        zeph_db::run_migrations(&self.pool)
216            .await
217            .map_err(|e| DurableError::storage("init", e))?;
218        Ok(())
219    }
220
221    /// List execution summaries for operability surfaces (the `zeph durable` CLI and TUI).
222    ///
223    /// Returns at most `limit` executions, newest first, optionally filtered by `status` and `kind`
224    /// (each is matched against the raw column tag; `None` disables that filter). Only execution-level
225    /// metadata is read — never payload bytes or resolver tokens (INV-5). The per-execution step
226    /// count is the number of journal entries recorded for it.
227    ///
228    /// Span: `durable.backend.list`.
229    ///
230    /// # Errors
231    ///
232    /// Returns [`DurableError::Storage`] if the query fails, or [`DurableError::Decode`] if a stored
233    /// id or status cannot be reconstructed (schema corruption — the `status` column is
234    /// `CHECK`-constrained, so this is a fail-closed guard rather than a routine path).
235    pub async fn list_executions(
236        &self,
237        status: Option<&str>,
238        kind: Option<&str>,
239        limit: i64,
240    ) -> Result<Vec<ExecutionSummary>, DurableError> {
241        let span = tracing::info_span!(
242            "durable.backend.list",
243            status = status.unwrap_or("*"),
244            kind = kind.unwrap_or("*"),
245            count = tracing::field::Empty,
246        );
247        async move {
248            // `COALESCE(?, col)` keeps a single positional bind per filter and lets the column type
249            // drive the bind type, so the same literal works on both SQLite and Postgres without a
250            // cast on the `?` placeholder.
251            let rows: Vec<ExecutionRow> =
252                zeph_db::query_as(sql!(
253                    "SELECT
254                        e.execution_id,
255                        e.kind,
256                        e.status,
257                        e.created_at,
258                        e.updated_at,
259                        e.finalized_at,
260                        (SELECT COUNT(*) FROM durable_journal j WHERE j.execution_id = e.execution_id)
261                     FROM durable_executions e
262                     WHERE e.status = COALESCE(?, e.status)
263                       AND e.kind = COALESCE(?, e.kind)
264                     ORDER BY e.created_at DESC
265                     LIMIT ?"
266                ))
267                .bind(status)
268                .bind(kind)
269                .bind(limit)
270                .fetch_all(&self.pool)
271                .await
272                .map_err(|e| DurableError::storage("list", e))?;
273            tracing::Span::current().record("count", rows.len());
274            rows.into_iter()
275                .map(|(id, kind, status, created, updated, finalized, steps)| {
276                    Ok(ExecutionSummary {
277                        execution_id: parse_execution_id(&id)?,
278                        kind,
279                        status: ExecutionStatus::from_tag(&status).ok_or(DurableError::Decode {
280                            context: "execution status is not a recognized CHECK-constrained value",
281                        })?,
282                        created_at_ms: created,
283                        updated_at_ms: updated,
284                        finalized_at_ms: finalized,
285                        step_count: steps.max(0).cast_unsigned(),
286                    })
287                })
288                .collect()
289        }
290        .instrument(span)
291        .await
292    }
293
294    /// Read one execution's journal entries as redaction-safe metadata, without decrypting payloads.
295    ///
296    /// Unlike [`read_execution`](Journal::read_execution), this never touches the cipher, so it works
297    /// against a journal whose AEAD key is unavailable and never exposes plaintext (INV-5). It backs
298    /// the default (redacted) `zeph durable show`/`inspect` output. Entries are returned in append
299    /// order.
300    ///
301    /// Span: `durable.backend.read_redacted`.
302    ///
303    /// # Errors
304    ///
305    /// Returns [`DurableError::Storage`] if the query fails.
306    pub async fn read_execution_redacted(
307        &self,
308        id: ExecutionId,
309    ) -> Result<Vec<RedactedEntry>, DurableError> {
310        let exec = id.as_uuid().to_string();
311        let rows: Vec<RedactedRow> = zeph_db::query_as(sql!(
312            "SELECT seq, step_id, entry_kind, idem_key, effect_class, LENGTH(payload), created_at
313                 FROM durable_journal WHERE execution_id = ? ORDER BY seq"
314        ))
315        .bind(&exec)
316        .fetch_all(&self.pool)
317        .await
318        .map_err(|e| DurableError::storage("read_redacted", e))?;
319        Ok(rows
320            .into_iter()
321            .map(
322                |(seq, step, entry_kind, idem, effect_class, payload_len, created)| RedactedEntry {
323                    seq,
324                    step_id: StepId::new(u32::try_from(step).unwrap_or(0)),
325                    entry_kind,
326                    effect_class,
327                    idem_key_prefix: idem.as_deref().map(idem_key_prefix),
328                    payload_len: payload_len.unwrap_or(0).max(0).cast_unsigned(),
329                    created_at_ms: created,
330                },
331            )
332            .collect())
333    }
334
335    /// Count terminal executions a [`prune`](Journal::prune) sweep would delete under `policy`.
336    ///
337    /// Read-only: backs `zeph durable prune --dry-run`. It applies the same TTL cutoffs as the
338    /// delete path, so the count is exactly what a real sweep would remove now.
339    ///
340    /// # Errors
341    ///
342    /// Returns [`DurableError::Storage`] if the query fails.
343    pub async fn count_prunable(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
344        let cutoffs = crate::retention::PruneCutoffs::from_policy(policy, now_unix_millis());
345        let (count,): (i64,) = zeph_db::query_as(sql!(
346            "SELECT COUNT(*) FROM durable_executions
347             WHERE finalized_at IS NOT NULL
348               AND ( (status = 'completed' AND finalized_at <= ?)
349                  OR (status IN ('failed', 'aborted') AND finalized_at <= ?) )"
350        ))
351        .bind(cutoffs.completed_before_ms)
352        .bind(cutoffs.failed_before_ms)
353        .fetch_one(&self.pool)
354        .await
355        .map_err(|e| DurableError::storage("count_prunable", e))?;
356        Ok(count.max(0).cast_unsigned())
357    }
358
359    /// Count crash-orphaned executions a [`sweep_orphans`](Journal::sweep_orphans) sweep would
360    /// abort under `policy` (#6254).
361    ///
362    /// Read-only: backs `zeph durable prune --dry-run`. Mirrors the real sweep's staleness scan
363    /// and INV-15 flock liveness check (acquiring and immediately releasing each candidate's
364    /// `ExecutionLock`, exactly as the real sweep does, so the count reflects genuinely
365    /// unowned rows rather than staleness alone) — but never mutates `status`. Returns `0` when
366    /// the sweep is disabled (`stale_running_after_secs == 0`) or this backend has no `lock_dir`.
367    ///
368    /// # Errors
369    ///
370    /// Returns [`DurableError::Storage`] if the query fails.
371    pub async fn count_orphans(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
372        if policy.stale_running_after_secs == 0 {
373            return Ok(0);
374        }
375        let Some(lock_dir) = self.lock_dir.clone() else {
376            return Ok(0);
377        };
378        let cutoff_ms = orphan_cutoff_ms(policy, now_unix_millis());
379        let candidates: Vec<(String,)> = zeph_db::query_as(sql!(
380            "SELECT execution_id FROM durable_executions WHERE status = 'running' AND updated_at <= ?"
381        ))
382        .bind(cutoff_ms)
383        .fetch_all(&self.pool)
384        .await
385        .map_err(|e| DurableError::storage("count_orphans", e))?;
386        let mut count = 0u64;
387        for (exec_str,) in &candidates {
388            let Ok(execution_id) = parse_execution_id(exec_str) else {
389                continue;
390            };
391            if ExecutionLock::acquire(&lock_dir, execution_id).is_ok() {
392                count += 1;
393            }
394        }
395        Ok(count)
396    }
397
398    /// Ensure a `durable_executions` row exists for `id`, returning whether this is a resume.
399    ///
400    /// Inserts a fresh `running` row for a new execution (returning `false`) or detects an existing
401    /// row for a resumed one (returning `true`). The journal's foreign key requires this row before
402    /// any entry is appended, so callers open the execution first.
403    ///
404    /// Reopening a row previously [`finalize`](Journal::finalize)d as `completed`, `failed`, or
405    /// `aborted` un-finalizes it: status resets to `running` and `finalized_at` clears (INV-16,
406    /// #6254). A caller reopening an execution is, by definition, still using it, so the retention
407    /// sweep (gated on `finalized_at`) must not consider it prunable while it does — without this,
408    /// a long-lived execution finalized at one process's graceful shutdown and legitimately resumed
409    /// by a later process (e.g. a per-conversation `AgentTurn` execution) would keep a stale
410    /// `finalized_at` and could be pruned out from under its still-active journal. `aborted` rows
411    /// are included because the crash-orphan sweep (INV-17) makes `aborted` the common outcome of a
412    /// resumable crash: a resumed execution whose row keeps `finalized_at` set is prunable out from
413    /// under the active resume — the exact hazard this un-finalize prevents for `completed`/`failed`.
414    /// This is also strictly safer for the pre-existing divergence-recovery case, which reopens an
415    /// `aborted` row on purpose: it now also protects that fresh re-drive from prune.
416    ///
417    /// The un-finalize is attempted as a single guarded `UPDATE` (no preceding `SELECT`) so there
418    /// is no read-then-write window against a concurrent prune sweep (#6251 critic S1): if the row
419    /// was deleted by `prune` between an earlier observation and this call, the `UPDATE` simply
420    /// matches zero rows rather than silently resurrecting a half-deleted row. A zero-row `UPDATE`
421    /// falls back to checking whether the row exists at all (already `running`/`aborted`, or
422    /// genuinely gone) before deciding between reporting a resume or inserting a fresh execution —
423    /// so this never reports `is_resume = true` for a row that turned out not to exist.
424    ///
425    /// Span: `durable.backend.open`.
426    ///
427    /// # Errors
428    ///
429    /// Returns [`DurableError::Storage`] if the lookup, reset, or insert fails.
430    pub async fn open_execution(
431        &self,
432        id: ExecutionId,
433        kind: ExecutionKind,
434    ) -> Result<bool, DurableError> {
435        let span = tracing::info_span!(
436            "durable.backend.open",
437            execution_id = %id.as_uuid(),
438            kind = kind.as_str(),
439            is_resume = tracing::field::Empty,
440        );
441        async move {
442            let exec = id.as_uuid().to_string();
443
444            // Attempt the un-finalize directly, with no preceding SELECT: this is the only write
445            // this call needs to make for an existing terminal row, so there is no window between
446            // "observe completed/failed" and "reset to running" for a concurrent prune to act in.
447            let reopened = zeph_db::query(sql!(
448                "UPDATE durable_executions SET status = 'running', updated_at = ?, finalized_at = NULL
449                 WHERE execution_id = ? AND status IN ('completed', 'failed', 'aborted')"
450            ))
451            .bind(now_unix_millis())
452            .bind(&exec)
453            .execute(&self.pool)
454            .await
455            .map_err(|e| DurableError::storage("open", e))?;
456            if reopened.rows_affected() > 0 {
457                tracing::Span::current().record("is_resume", true);
458                return Ok(true);
459            }
460
461            // Zero rows: either the row doesn't exist, or it exists but wasn't terminal (already
462            // `running`, no reset needed — every terminal status is covered by the UPDATE above).
463            // Distinguish the two — if a concurrent prune deleted a terminal row between any
464            // earlier observation and this check, this SELECT sees the authoritative post-delete
465            // state instead of a stale belief that it's there.
466            let existing: Option<(String,)> = zeph_db::query_as(sql!(
467                "SELECT status FROM durable_executions WHERE execution_id = ?"
468            ))
469            .bind(&exec)
470            .fetch_optional(&self.pool)
471            .await
472            .map_err(|e| DurableError::storage("open", e))?;
473            if existing.is_some() {
474                tracing::Span::current().record("is_resume", true);
475                return Ok(true);
476            }
477            let now = now_unix_millis();
478            zeph_db::query(sql!(
479                "INSERT INTO durable_executions
480                    (execution_id, kind, status, created_at, updated_at, finalized_at)
481                 VALUES (?, ?, 'running', ?, ?, NULL)"
482            ))
483            .bind(&exec)
484            .bind(kind.as_str())
485            .bind(now)
486            .bind(now)
487            .execute(&self.pool)
488            .await
489            .map_err(|e| DurableError::storage("open", e))?;
490            tracing::Span::current().record("is_resume", false);
491            Ok(false)
492        }
493        .instrument(span)
494        .await
495    }
496
497    /// Like [`open_execution`](Self::open_execution), but additionally takes a non-blocking,
498    /// exclusive, process-scoped advisory lock on `id` before touching the row (INV-15, #6122).
499    ///
500    /// Closes the race two processes deriving the same `ExecutionId` (e.g. two CLI instances
501    /// pointed at the same `memory.sqlite_path` and the same `ConversationId`) would otherwise hit
502    /// in [`open_execution`](Self::open_execution)'s unsynchronized SELECT-then-INSERT: both could
503    /// observe "no existing row", both insert, and both then drive `next_step` from 0 against the
504    /// same journal, corrupting it. The lock is acquired first, so the loser never reaches the
505    /// row check at all.
506    ///
507    /// Returns `(is_resume, lock)`. The caller MUST hold `lock` for as long as it drives the
508    /// execution — dropping it releases the lock and allows another process to open the same
509    /// `id`. `lock` is `None` when this backend has no on-disk lock directory (a `:memory:`
510    /// database, a backend built via [`LocalBackend::new`], or a Postgres deployment), in which
511    /// case process exclusivity is not enforced — the caller degrades the same way it already does
512    /// for `open_execution`'s other failure modes.
513    ///
514    /// # Errors
515    ///
516    /// Returns [`DurableError::ExecutionLocked`] if another process already holds `id`'s lock, or
517    /// any error [`open_execution`](Self::open_execution) can return.
518    pub async fn open_execution_exclusive(
519        &self,
520        id: ExecutionId,
521        kind: ExecutionKind,
522    ) -> Result<(bool, Option<ExecutionLock>), DurableError> {
523        let lock = self
524            .lock_dir
525            .as_deref()
526            .map(|dir| ExecutionLock::acquire(dir, id))
527            .transpose()?;
528        let is_resume = self.open_execution(id, kind).await?;
529        Ok((is_resume, lock))
530    }
531
532    /// Group-commit a batch of buffered entries in a single write transaction.
533    ///
534    /// Used by the [`JournalWriter`](crate::JournalWriter) to amortize the WAL fsync across all
535    /// entries accumulated within a flush interval. Sealing and HMAC computation run before the
536    /// transaction opens, keeping CPU work off the write lock. The whole batch commits atomically;
537    /// a single malformed entry aborts the batch.
538    ///
539    /// # Errors
540    ///
541    /// Returns [`DurableError::Storage`] on a database failure, or a per-entry error
542    /// ([`DurableError::PayloadTooLarge`], [`DurableError::UnsupportedEntryKind`], or a cipher
543    /// failure) if an entry cannot be prepared.
544    pub(crate) async fn append_batch(&self, entries: &[JournalEntry]) -> Result<(), DurableError> {
545        if entries.is_empty() {
546            return Ok(());
547        }
548        let mut rows = Vec::with_capacity(entries.len());
549        for entry in entries {
550            rows.push(self.prepare_row(entry)?);
551        }
552        // `sql!()` caches its postgres rewrite per call site (see #5431), so hoisting
553        // this out of the loop below is no longer required to avoid a leak — kept
554        // anyway since it reads the intent clearly and costs nothing.
555        let insert = sql!(
556            "INSERT INTO durable_journal
557                (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
558             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
559        );
560        let mut tx = zeph_db::begin_write(&self.pool)
561            .await
562            .map_err(|e| DurableError::storage("append_batch", e))?;
563        for row in rows {
564            zeph_db::query(insert)
565                .bind(row.execution_id)
566                .bind(row.step_id)
567                .bind(row.entry_kind)
568                .bind(row.idem_key)
569                .bind(row.effect_class)
570                .bind(row.payload)
571                .bind(row.payload_version)
572                .bind(row.hmac)
573                .bind(row.created_at)
574                .execute(&mut *tx)
575                .await
576                .map_err(|e| DurableError::storage("append_batch", e))?;
577        }
578        tx.commit()
579            .await
580            .map_err(|e| DurableError::storage("append_batch", e))?;
581        Ok(())
582    }
583
584    /// Look up a committed `StepResult` anywhere in an execution by its [`IdempotencyKey`].
585    ///
586    /// Backs INV-13: a guarded effect that already committed its result must not re-fire after a
587    /// replay divergence restarts the execution fresh. Returns the (opened) `StepResult` entry when
588    /// one exists, or `None`. The `idx_durable_journal_idem_key` partial index makes this an
589    /// `O(log n)` point lookup rather than a scan.
590    ///
591    /// Span: `durable.journal.lookup_idem`.
592    ///
593    /// # Errors
594    ///
595    /// Returns [`DurableError::Storage`] if the query fails, or [`DurableError::Decode`] if the
596    /// located row cannot be reconstructed.
597    pub(crate) async fn lookup_committed_result(
598        &self,
599        id: ExecutionId,
600        idem_key: IdempotencyKey,
601    ) -> Result<Option<JournalEntry>, DurableError> {
602        let span = tracing::info_span!(
603            "durable.journal.lookup_idem",
604            execution_id = %id.as_uuid(),
605            found = tracing::field::Empty,
606        );
607        async move {
608            let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
609                "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
610                 FROM durable_journal
611                 WHERE execution_id = ? AND idem_key = ? AND entry_kind = 'step_result'
612                 ORDER BY seq LIMIT 1"
613            ))
614            .bind(id.as_uuid().to_string())
615            .bind(idem_key.as_bytes().to_vec())
616            .fetch_all(&self.pool)
617            .await
618            .map_err(|e| DurableError::storage("lookup_idem", e))?;
619            let entry = self.rows_to_entries(id, rows).await?.into_iter().next();
620            tracing::Span::current().record("found", entry.is_some());
621            Ok(entry)
622        }
623        .instrument(span)
624        .await
625    }
626
627    /// Read the highest committed [`JournalSeq`], or `None` for an empty journal.
628    ///
629    /// The [`JournalWriter`](crate::JournalWriter) calls this on (re)start to anchor itself at the
630    /// last durably-committed entry (FR-DE-12). Because `seq` is a database-assigned autoincrement,
631    /// resumed appends continue from `MAX(seq) + 1` with neither gap nor duplication.
632    ///
633    /// # Errors
634    ///
635    /// Returns [`DurableError::Storage`] if the query fails.
636    pub(crate) async fn max_seq(&self) -> Result<Option<JournalSeq>, DurableError> {
637        let max: Option<i64> = zeph_db::query_scalar(sql!("SELECT MAX(seq) FROM durable_journal"))
638            .fetch_one(&self.pool)
639            .await
640            .map_err(|e| DurableError::storage("max_seq", e))?;
641        Ok(max.map(JournalSeq::new))
642    }
643
644    /// The in-process wakeup registry for parked promise awaits, shared with the resolver path.
645    pub(crate) fn promise_waiters(&self) -> &NotifyRegistry {
646        &self.promise_waiters
647    }
648
649    /// The in-process wakeup registry for parked timers, shared with the timer service.
650    pub(crate) fn timer_waiters(&self) -> &NotifyRegistry {
651        &self.timer_waiters
652    }
653
654    /// Insert a freshly-created promise row (INV-9: only the resolver-token hash is stored).
655    ///
656    /// Called by `promise()` for a brand-new promise; a resumed execution detects the existing row
657    /// via [`promise_state`](Self::promise_state) and never re-inserts. Span: `durable.promise.create`.
658    ///
659    /// # Errors
660    ///
661    /// Returns [`DurableError::Storage`] if the insert fails.
662    pub(crate) async fn insert_promise(
663        &self,
664        id: PromiseId,
665        execution_id: ExecutionId,
666        resolver_token_hash: [u8; 32],
667        created_at_ms: i64,
668    ) -> Result<(), DurableError> {
669        let span = tracing::info_span!("durable.promise.create", promise_id = %id.as_uuid());
670        async move {
671            zeph_db::query(sql!(
672                "INSERT INTO durable_promises
673                    (promise_id, execution_id, resolver_token_hash, resolved, payload, created_at, resolved_at)
674                 VALUES (?, ?, ?, 0, NULL, ?, NULL)"
675            ))
676            .bind(id.as_uuid().to_string())
677            .bind(execution_id.as_uuid().to_string())
678            .bind(resolver_token_hash.to_vec())
679            .bind(created_at_ms)
680            .execute(&self.pool)
681            .await
682            .map_err(|e| DurableError::storage("insert_promise", e))?;
683            Ok(())
684        }
685        .instrument(span)
686        .await
687    }
688
689    /// Read a promise's persisted state, or `None` if it does not exist.
690    ///
691    /// # Errors
692    ///
693    /// Returns [`DurableError::Storage`] if the query fails, or [`DurableError::Decode`] if a stored
694    /// field cannot be reconstructed.
695    pub(crate) async fn promise_state(
696        &self,
697        id: PromiseId,
698    ) -> Result<Option<PromiseRecord>, DurableError> {
699        let row: Option<PromiseRowRead> = zeph_db::query_as(sql!(
700            "SELECT execution_id, resolver_token_hash, resolved, payload
701             FROM durable_promises WHERE promise_id = ?"
702        ))
703        .bind(id.as_uuid().to_string())
704        .fetch_optional(&self.pool)
705        .await
706        .map_err(|e| DurableError::storage("promise_state", e))?;
707        let Some((exec, hash, resolved, payload)) = row else {
708            return Ok(None);
709        };
710        Ok(Some(PromiseRecord {
711            execution_id: parse_execution_id(&exec)?,
712            resolver_token_hash: slice_to_array32(&hash, "promise resolver_token_hash")?,
713            resolved: resolved != 0,
714            payload,
715        }))
716    }
717
718    /// Commit a resolved value to a pending promise, returning whether it transitioned.
719    ///
720    /// The conditional `WHERE resolved = 0` makes a double-resolve a no-op (returns `false`); the
721    /// caller has already authenticated the resolver token. On a real transition any in-process
722    /// waiter is woken. Span: `durable.promise.resolve`.
723    ///
724    /// # Errors
725    ///
726    /// Returns [`DurableError::PayloadTooLarge`] if the value exceeds the limit, a cipher failure if
727    /// sealing fails, or [`DurableError::Storage`] on a database error.
728    pub(crate) async fn resolve_promise(
729        &self,
730        id: PromiseId,
731        execution_id: ExecutionId,
732        value_plaintext: &[u8],
733        resolved_at_ms: i64,
734    ) -> Result<bool, DurableError> {
735        let span = tracing::info_span!("durable.promise.resolve", promise_id = %id.as_uuid());
736        async move {
737            ensure_payload_within_limit(value_plaintext.len(), self.max_payload_bytes)?;
738            let aad = promise_payload_aad(execution_id, id);
739            let sealed = self.seal_payload(value_plaintext, &aad)?;
740            let affected = zeph_db::query(sql!(
741                "UPDATE durable_promises SET resolved = 1, payload = ?, resolved_at = ?
742                 WHERE promise_id = ? AND resolved = 0"
743            ))
744            .bind(sealed)
745            .bind(resolved_at_ms)
746            .bind(id.as_uuid().to_string())
747            .execute(&self.pool)
748            .await
749            .map_err(|e| DurableError::storage("resolve_promise", e))?
750            .rows_affected();
751            if affected > 0 {
752                self.promise_waiters.wake(id.as_uuid());
753            }
754            Ok(affected > 0)
755        }
756        .instrument(span)
757        .await
758    }
759
760    /// Claim the one-time replay notification for a promise, returning whether this call won.
761    ///
762    /// The conditional `WHERE notified_at IS NULL` makes the claim single-winner: the first caller
763    /// transitions the row (returns `true`); every later caller is a no-op (returns `false`). This
764    /// backs #6027 — a resumed foreground sub-agent's replay notice / TUI completion event must fire
765    /// at most once across repeated parent restarts. Unlike [`resolve_promise`](Self::resolve_promise)
766    /// it touches only the `notified_at` bookkeeping column and carries no payload, so no sealing /
767    /// waiter wakeup is involved. Span: `durable.promise.claim_notify`.
768    ///
769    /// # Errors
770    ///
771    /// Returns [`DurableError::Storage`] on a database error.
772    pub(crate) async fn claim_promise_notification(
773        &self,
774        id: PromiseId,
775        notified_at_ms: i64,
776    ) -> Result<bool, DurableError> {
777        let span = tracing::info_span!("durable.promise.claim_notify", promise_id = %id.as_uuid());
778        async move {
779            let affected = zeph_db::query(sql!(
780                "UPDATE durable_promises SET notified_at = ?
781                 WHERE promise_id = ? AND notified_at IS NULL"
782            ))
783            .bind(notified_at_ms)
784            .bind(id.as_uuid().to_string())
785            .execute(&self.pool)
786            .await
787            .map_err(|e| DurableError::storage("claim_promise_notification", e))?
788            .rows_affected();
789            Ok(affected > 0)
790        }
791        .instrument(span)
792        .await
793    }
794
795    /// Open a promise's sealed resolved payload back to plaintext.
796    ///
797    /// # Errors
798    ///
799    /// Returns [`DurableError::ReplayIntegrity`] if the sealed blob does not authenticate, or
800    /// [`DurableError::PayloadTooLarge`] if it exceeds the read-side limit.
801    pub(crate) fn open_promise_payload(
802        &self,
803        id: PromiseId,
804        execution_id: ExecutionId,
805        sealed: &[u8],
806    ) -> Result<Bytes, DurableError> {
807        ensure_payload_within_limit(
808            sealed.len(),
809            self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
810        )?;
811        let aad = promise_payload_aad(execution_id, id);
812        self.open_payload(sealed, &aad)
813    }
814
815    /// Arm a durable timer to fire at `due_at_ms` (a `durable_timers` row).
816    ///
817    /// Span: `durable.timer.arm`.
818    ///
819    /// # Errors
820    ///
821    /// Returns [`DurableError::Storage`] if the insert fails.
822    pub(crate) async fn arm_timer(
823        &self,
824        id: TimerId,
825        execution_id: ExecutionId,
826        due_at_ms: i64,
827        created_at_ms: i64,
828    ) -> Result<(), DurableError> {
829        let span = tracing::info_span!("durable.timer.arm", timer_id = %id.as_uuid(), due_at_ms);
830        async move {
831            zeph_db::query(sql!(
832                "INSERT INTO durable_timers (timer_id, execution_id, due_at, fired, created_at)
833                 VALUES (?, ?, ?, 0, ?)"
834            ))
835            .bind(id.as_uuid().to_string())
836            .bind(execution_id.as_uuid().to_string())
837            .bind(due_at_ms)
838            .bind(created_at_ms)
839            .execute(&self.pool)
840            .await
841            .map_err(|e| DurableError::storage("arm_timer", e))?;
842            Ok(())
843        }
844        .instrument(span)
845        .await
846    }
847
848    /// Read a timer's `(due_at_ms, fired)` state, or `None` if it does not exist.
849    ///
850    /// # Errors
851    ///
852    /// Returns [`DurableError::Storage`] if the query fails.
853    pub(crate) async fn timer_state(
854        &self,
855        id: TimerId,
856    ) -> Result<Option<(i64, bool)>, DurableError> {
857        let row: Option<(i64, i64)> = zeph_db::query_as(sql!(
858            "SELECT due_at, fired FROM durable_timers WHERE timer_id = ?"
859        ))
860        .bind(id.as_uuid().to_string())
861        .fetch_optional(&self.pool)
862        .await
863        .map_err(|e| DurableError::storage("timer_state", e))?;
864        Ok(row.map(|(due_at, fired)| (due_at, fired != 0)))
865    }
866
867    /// List every unfired timer whose instant is at or before `now_ms`.
868    ///
869    /// The `idx_durable_timers_due(fired, due_at)` index makes this a range scan over due, unfired
870    /// timers rather than a full-table scan.
871    ///
872    /// # Errors
873    ///
874    /// Returns [`DurableError::Storage`] if the query fails, or [`DurableError::Decode`] on a
875    /// malformed id.
876    pub(crate) async fn due_timers(&self, now_ms: i64) -> Result<Vec<TimerId>, DurableError> {
877        let rows: Vec<(String,)> = zeph_db::query_as(sql!(
878            "SELECT timer_id FROM durable_timers WHERE fired = 0 AND due_at <= ? ORDER BY due_at"
879        ))
880        .bind(now_ms)
881        .fetch_all(&self.pool)
882        .await
883        .map_err(|e| DurableError::storage("due_timers", e))?;
884        rows.into_iter().map(|(id,)| parse_timer_id(&id)).collect()
885    }
886
887    /// Mark a timer fired, returning whether it transitioned, and wake its parked waiter.
888    ///
889    /// Span: `durable.timer.fire`.
890    ///
891    /// # Errors
892    ///
893    /// Returns [`DurableError::Storage`] if the update fails.
894    pub(crate) async fn mark_timer_fired(&self, id: TimerId) -> Result<bool, DurableError> {
895        let span = tracing::info_span!("durable.timer.fire", timer_id = %id.as_uuid());
896        async move {
897            let affected = zeph_db::query(sql!(
898                "UPDATE durable_timers SET fired = 1 WHERE timer_id = ? AND fired = 0"
899            ))
900            .bind(id.as_uuid().to_string())
901            .execute(&self.pool)
902            .await
903            .map_err(|e| DurableError::storage("mark_timer_fired", e))?
904            .rows_affected();
905            if affected > 0 {
906                self.timer_waiters.wake(id.as_uuid());
907            }
908            Ok(affected > 0)
909        }
910        .instrument(span)
911        .await
912    }
913
914    /// Open each foldable step result's sealed payload into a [`FoldedStep`], in step order.
915    ///
916    /// The per-step AAD is reconstructed from the row so the opened plaintext authenticates exactly
917    /// as it did at rest; the idempotency key is preserved so the replayed-from-snapshot step still
918    /// satisfies the divergence guard.
919    fn open_foldable_steps(
920        &self,
921        execution_id: ExecutionId,
922        rows: Vec<FoldableRowRead>,
923    ) -> Result<Vec<FoldedStep>, DurableError> {
924        let mut folded = Vec::with_capacity(rows.len());
925        for (step_raw, idem, version, payload) in rows {
926            let step = u32::try_from(step_raw).map_err(|_| DurableError::Decode {
927                context: "checkpoint step_id out of u32 range",
928            })?;
929            let idem_bytes = idem.ok_or(DurableError::Decode {
930                context: "checkpoint step result missing idem_key",
931            })?;
932            let idem_key =
933                IdempotencyKey::from_bytes(slice_to_array32(&idem_bytes, "checkpoint idem_key")?);
934            let sealed = payload.ok_or(DurableError::Decode {
935                context: "checkpoint step result missing payload",
936            })?;
937            let aad = PayloadAad::new(
938                execution_id,
939                StepId::new(step),
940                EntryKindTag::StepResult,
941                Some(idem_key),
942            );
943            let plaintext = self.open_payload(&sealed, &aad)?;
944            let payload_version =
945                u8::try_from(version.unwrap_or(1)).map_err(|_| DurableError::Decode {
946                    context: "checkpoint payload_version out of u8 range",
947                })?;
948            folded.push(FoldedStep {
949                step_id: step,
950                idem_key: *idem_key.as_bytes(),
951                payload_version,
952                payload: plaintext,
953            });
954        }
955        Ok(folded)
956    }
957
958    /// Fold an execution's committed-idempotent prefix below `up_to_step` into one checkpoint entry.
959    ///
960    /// Reads the foldable idempotent step results, packs as many as fit the payload budget into a
961    /// sealed snapshot, writes a single [`EntryKind::Checkpoint`] entry, and deletes the folded rows
962    /// — all in one transaction. A resume replays the folded steps from the snapshot (the snapshot
963    /// preserves each step's idempotency key for the divergence guard) instead of re-running them.
964    /// Returns the number of steps folded. Runs only on a background task (spec NEVER: not the hot
965    /// path). Span: `durable.journal.checkpoint`.
966    ///
967    /// # Errors
968    ///
969    /// Returns [`DurableError::Storage`] on a database error, or a cipher failure if (re)sealing
970    /// fails.
971    pub(crate) async fn checkpoint_fold(
972        &self,
973        execution_id: ExecutionId,
974        up_to_step: u32,
975    ) -> Result<u64, DurableError> {
976        let span = tracing::info_span!(
977            "durable.journal.checkpoint",
978            execution_id = %execution_id.as_uuid(),
979            folded_count = tracing::field::Empty,
980        );
981        async move {
982            let exec = execution_id.as_uuid().to_string();
983            let rows: Vec<FoldableRowRead> = zeph_db::query_as(sql!(
984                "SELECT step_id, idem_key, payload_version, payload FROM durable_journal
985                 WHERE execution_id = ? AND entry_kind = 'step_result'
986                   AND effect_class = 'idempotent' AND step_id < ?
987                 ORDER BY step_id"
988            ))
989            .bind(&exec)
990            .bind(i64::from(up_to_step))
991            .fetch_all(&self.pool)
992            .await
993            .map_err(|e| DurableError::storage("checkpoint", e))?;
994            if rows.is_empty() {
995                return Ok(0);
996            }
997
998            // Open each sealed result, then keep the budget-bounded prefix that fits a checkpoint.
999            let mut folded = self.open_foldable_steps(execution_id, rows)?;
1000            let lens: Vec<usize> = folded.iter().map(|s| s.payload.len()).collect();
1001            let take = crate::retention::fold_prefix_len(
1002                &lens,
1003                crate::retention::checkpoint_budget(self.max_payload_bytes),
1004            );
1005            if take == 0 {
1006                // Not even one result fits the budget; leave the prefix un-folded rather than write
1007                // an over-limit checkpoint.
1008                return Ok(0);
1009            }
1010            folded.truncate(take);
1011            let fold_end = folded.last().map_or(up_to_step, |s| s.step_id.saturating_add(1));
1012
1013            let snapshot = encode_checkpoint(&folded);
1014            let snap_aad =
1015                PayloadAad::new(execution_id, StepId::new(fold_end), EntryKindTag::Checkpoint, None);
1016            let sealed_snapshot = self.seal_payload(&snapshot, &snap_aad)?;
1017
1018            let mut tx = zeph_db::begin_write(&self.pool)
1019                .await
1020                .map_err(|e| DurableError::storage("checkpoint", e))?;
1021            zeph_db::query(sql!(
1022                "INSERT INTO durable_journal
1023                    (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
1024                 VALUES (?, ?, 'checkpoint', NULL, NULL, ?, ?, NULL, ?)"
1025            ))
1026            .bind(&exec)
1027            .bind(i64::from(fold_end))
1028            .bind(sealed_snapshot)
1029            .bind(i32::from(crate::step::PAYLOAD_VERSION))
1030            .bind(now_unix_millis())
1031            .execute(&mut *tx)
1032            .await
1033            .map_err(|e| DurableError::storage("checkpoint", e))?;
1034            zeph_db::query(sql!(
1035                "DELETE FROM durable_journal
1036                 WHERE execution_id = ? AND entry_kind = 'step_result'
1037                   AND effect_class = 'idempotent' AND step_id < ?"
1038            ))
1039            .bind(&exec)
1040            .bind(i64::from(fold_end))
1041            .execute(&mut *tx)
1042            .await
1043            .map_err(|e| DurableError::storage("checkpoint", e))?;
1044            tx.commit()
1045                .await
1046                .map_err(|e| DurableError::storage("checkpoint", e))?;
1047
1048            let count = folded.len() as u64;
1049            tracing::Span::current().record("folded_count", count);
1050            Ok(count)
1051        }
1052        .instrument(span)
1053        .await
1054    }
1055
1056    /// Read every checkpoint snapshot for an execution and reconstruct its folded step results.
1057    ///
1058    /// The replay cursor calls this once on resume to preload folded results before walking the
1059    /// surviving journal rows: each returned [`JournalEntry`] is a `StepResult` whose individual row
1060    /// was deleted by the fold but whose replay value (and idempotency key, for the divergence guard)
1061    /// lives in the snapshot. Each snapshot is AEAD-opened with its checkpoint-bound AAD. Returns an
1062    /// empty vector when the execution has never been folded.
1063    ///
1064    /// # Errors
1065    ///
1066    /// Returns [`DurableError::Storage`] on a database error, or a decode/cipher failure if a
1067    /// snapshot is corrupt.
1068    pub(crate) async fn read_checkpoints(
1069        &self,
1070        execution_id: ExecutionId,
1071    ) -> Result<Vec<JournalEntry>, DurableError> {
1072        let rows: Vec<(i64, Option<Vec<u8>>)> = zeph_db::query_as(sql!(
1073            "SELECT step_id, payload FROM durable_journal
1074             WHERE execution_id = ? AND entry_kind = 'checkpoint' ORDER BY step_id"
1075        ))
1076        .bind(execution_id.as_uuid().to_string())
1077        .fetch_all(&self.pool)
1078        .await
1079        .map_err(|e| DurableError::storage("read_checkpoints", e))?;
1080        if rows.is_empty() {
1081            return Ok(Vec::new());
1082        }
1083        let mut folded: CheckpointSnapshot = Vec::new();
1084        for (up_to, payload) in rows {
1085            let up_to = u32::try_from(up_to).map_err(|_| DurableError::Decode {
1086                context: "checkpoint up_to_step out of u32 range",
1087            })?;
1088            let sealed = payload.ok_or(DurableError::Decode {
1089                context: "checkpoint entry missing snapshot payload",
1090            })?;
1091            ensure_payload_within_limit(
1092                sealed.len(),
1093                self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1094            )?;
1095            let aad = PayloadAad::new(
1096                execution_id,
1097                StepId::new(up_to),
1098                EntryKindTag::Checkpoint,
1099                None,
1100            );
1101            let plaintext = self.open_payload(&sealed, &aad)?;
1102            folded.extend(decode_checkpoint(&plaintext)?);
1103        }
1104        // Reconstruct each folded step as a replayable `StepResult` entry under the real execution
1105        // kind, so the cursor serves it exactly like a surviving row.
1106        let kind = self.lookup_kind(execution_id).await?;
1107        let entries = folded
1108            .into_iter()
1109            .map(|step| JournalEntry {
1110                seq: None,
1111                execution_id,
1112                kind,
1113                step_id: StepId::new(step.step_id),
1114                entry: EntryKind::StepResult {
1115                    idempotency_key: IdempotencyKey::from_bytes(step.idem_key),
1116                    payload: step.payload,
1117                    effect: crate::EffectClass::Idempotent,
1118                    payload_version: step.payload_version,
1119                },
1120                created_at_ms: 0,
1121            })
1122            .collect();
1123        Ok(entries)
1124    }
1125
1126    /// Delete one bounded batch of prunable terminal executions and their child rows.
1127    ///
1128    /// Selects up to `batch` executions past their TTL, then deletes their journal, promise, timer,
1129    /// and execution rows in a single transaction (children first, to respect the foreign keys).
1130    /// Returns the number of executions removed; the retention loop stops once a batch returns fewer
1131    /// than `batch`.
1132    ///
1133    /// The candidate-selection `SELECT` runs *inside* the same `begin_write` transaction as the
1134    /// deletes (not on the autocommit pool beforehand), closing the race where a concurrent
1135    /// `open_execution` reopen (un-finalize, #6251) lands between "select prunable ids" and
1136    /// "delete them" — without this, a legitimately-resumed execution could be deleted out from
1137    /// under its own reopen. `SQLite`'s `BEGIN IMMEDIATE` (via `begin_write`) already serializes
1138    /// writers at the file level, so the `SELECT` alone is enough there; `PostgreSQL` needs an
1139    /// explicit `SELECT ... FOR UPDATE` first to take row locks on the same candidates before
1140    /// they're read, since a plain `BEGIN` does not otherwise block a concurrent `UPDATE` on those
1141    /// rows (mirrors the `BEGIN IMMEDIATE` / `SELECT FOR UPDATE` split in `goal/store.rs`).
1142    async fn delete_prune_batch(
1143        &self,
1144        cutoffs: crate::retention::PruneCutoffs,
1145        batch: u64,
1146    ) -> Result<u64, DurableError> {
1147        let mut tx = zeph_db::begin_write(&self.pool)
1148            .await
1149            .map_err(|e| DurableError::storage("prune", e))?;
1150
1151        // Postgres only: lock the same candidate rows before reading them, so a concurrent
1152        // `open_execution` reopen UPDATE on one of these rows blocks until this transaction
1153        // commits (and then no longer matches, since the SELECT below re-reads post-commit) or
1154        // this transaction rolls back. Bounded by the same ORDER BY/LIMIT as the real read below
1155        // so the lock's blast radius matches the batch, not the whole prunable backlog.
1156        #[cfg(feature = "postgres")]
1157        zeph_db::query(sql!(
1158            "SELECT execution_id FROM durable_executions
1159             WHERE finalized_at IS NOT NULL
1160               AND ( (status = 'completed' AND finalized_at <= ?)
1161                  OR (status IN ('failed', 'aborted') AND finalized_at <= ?) )
1162             ORDER BY finalized_at LIMIT ?
1163             FOR UPDATE"
1164        ))
1165        .bind(cutoffs.completed_before_ms)
1166        .bind(cutoffs.failed_before_ms)
1167        .bind(i64::try_from(batch).unwrap_or(i64::MAX))
1168        .execute(&mut *tx)
1169        .await
1170        .map_err(|e| DurableError::storage("prune", e))?;
1171
1172        let ids: Vec<(String,)> = zeph_db::query_as(sql!(
1173            "SELECT execution_id FROM durable_executions
1174             WHERE finalized_at IS NOT NULL
1175               AND ( (status = 'completed' AND finalized_at <= ?)
1176                  OR (status IN ('failed', 'aborted') AND finalized_at <= ?) )
1177             ORDER BY finalized_at LIMIT ?"
1178        ))
1179        .bind(cutoffs.completed_before_ms)
1180        .bind(cutoffs.failed_before_ms)
1181        .bind(i64::try_from(batch).unwrap_or(i64::MAX))
1182        .fetch_all(&mut *tx)
1183        .await
1184        .map_err(|e| DurableError::storage("prune", e))?;
1185        if ids.is_empty() {
1186            tx.commit()
1187                .await
1188                .map_err(|e| DurableError::storage("prune", e))?;
1189            return Ok(0);
1190        }
1191        let journal = sql!("DELETE FROM durable_journal WHERE execution_id = ?");
1192        let promises = sql!("DELETE FROM durable_promises WHERE execution_id = ?");
1193        let timers = sql!("DELETE FROM durable_timers WHERE execution_id = ?");
1194        // Re-guarded by the same status/finalized_at predicate as the SELECT above (not just
1195        // `execution_id = ?`) — belt and suspenders alongside the transactional read above.
1196        let executions = sql!(
1197            "DELETE FROM durable_executions
1198             WHERE execution_id = ?
1199               AND finalized_at IS NOT NULL
1200               AND ( (status = 'completed' AND finalized_at <= ?)
1201                  OR (status IN ('failed', 'aborted') AND finalized_at <= ?) )"
1202        );
1203        let mut removed = 0u64;
1204        for (id,) in &ids {
1205            for stmt in [journal, promises, timers] {
1206                zeph_db::query(stmt)
1207                    .bind(id)
1208                    .execute(&mut *tx)
1209                    .await
1210                    .map_err(|e| DurableError::storage("prune", e))?;
1211            }
1212            let result = zeph_db::query(executions)
1213                .bind(id)
1214                .bind(cutoffs.completed_before_ms)
1215                .bind(cutoffs.failed_before_ms)
1216                .execute(&mut *tx)
1217                .await
1218                .map_err(|e| DurableError::storage("prune", e))?;
1219            removed += result.rows_affected();
1220        }
1221        tx.commit()
1222            .await
1223            .map_err(|e| DurableError::storage("prune", e))?;
1224        Ok(removed)
1225    }
1226
1227    /// One batch of the crash-orphan sweep (INV-17, #6254).
1228    ///
1229    /// Selects up to `batch` `status='running'` rows whose `updated_at` is at or before
1230    /// `cutoff_ms`, then for each candidate non-blockingly try-acquires its INV-15
1231    /// `ExecutionLock`: `ExecutionLocked` (a live owner holds it) short-circuits to skip —
1232    /// staleness of `updated_at` alone is never sufficient grounds to abort. Only when the lock is
1233    /// acquired does the guarded `UPDATE` run, still holding the lock, so the abort is race-free
1234    /// against a concurrent `open_execution_exclusive` reopen for the same id (both require the
1235    /// same non-reentrant flock). The lock releases when it drops at the end of each loop
1236    /// iteration.
1237    ///
1238    /// `cursor` is the previous batch's [`SweepCursor`](crate::retention::SweepCursor) (`None` for
1239    /// the first batch); the candidate scan is keyset-paginated strictly past it so a skipped
1240    /// (lock-held) row is never re-selected by a later batch — #6254 C1: without this, a batch
1241    /// consisting entirely of lock-held rows would re-select the identical rows on every
1242    /// iteration and the caller's batch loop would never terminate. Returns the number of rows
1243    /// scanned (for the caller's batch-continuation decision), the number actually aborted, and
1244    /// the cursor to resume from on the next call.
1245    async fn sweep_orphan_batch(
1246        &self,
1247        lock_dir: &std::path::Path,
1248        cutoff_ms: i64,
1249        batch: u64,
1250        cursor: Option<crate::retention::SweepCursor>,
1251    ) -> Result<crate::retention::SweepBatchOutcome, DurableError> {
1252        // Sentinel "no lower bound" cursor: every real `updated_at` (Unix ms) is > i64::MIN, so
1253        // this keyset predicate is a no-op on the first batch while still using one static,
1254        // sql!()-cacheable query for both the first and subsequent calls.
1255        let (after_updated_at, after_exec) = cursor.map_or((i64::MIN, String::new()), |c| {
1256            (c.updated_at_ms, c.execution_id)
1257        });
1258
1259        let candidates: Vec<(String, i64)> = zeph_db::query_as(sql!(
1260            "SELECT execution_id, updated_at FROM durable_executions
1261             WHERE status = 'running' AND updated_at <= ?
1262               AND (updated_at > ? OR (updated_at = ? AND execution_id > ?))
1263             ORDER BY updated_at, execution_id LIMIT ?"
1264        ))
1265        .bind(cutoff_ms)
1266        .bind(after_updated_at)
1267        .bind(after_updated_at)
1268        .bind(&after_exec)
1269        .bind(i64::try_from(batch).unwrap_or(i64::MAX))
1270        .fetch_all(&self.pool)
1271        .await
1272        .map_err(|e| DurableError::storage("sweep_orphans", e))?;
1273
1274        let scanned = u64::try_from(candidates.len()).unwrap_or(u64::MAX);
1275        let next_cursor = candidates
1276            .last()
1277            .map(|(id, updated_at)| crate::retention::SweepCursor {
1278                updated_at_ms: *updated_at,
1279                execution_id: id.clone(),
1280            });
1281
1282        let now = now_unix_millis();
1283        let abort = sql!(
1284            "UPDATE durable_executions SET status = 'aborted', finalized_at = ?, updated_at = ?
1285             WHERE execution_id = ? AND status = 'running' AND finalized_at IS NULL"
1286        );
1287        let mut aborted = 0u64;
1288        for (exec_str, _updated_at) in &candidates {
1289            let Ok(execution_id) = parse_execution_id(exec_str) else {
1290                continue;
1291            };
1292            match ExecutionLock::acquire(lock_dir, execution_id) {
1293                Ok(_lock) => {
1294                    let result = zeph_db::query(abort)
1295                        .bind(now)
1296                        .bind(now)
1297                        .bind(exec_str)
1298                        .execute(&self.pool)
1299                        .await
1300                        .map_err(|e| DurableError::storage("sweep_orphans", e))?;
1301                    aborted += result.rows_affected();
1302                    // `_lock` drops here, releasing the flock for the next holder.
1303                }
1304                Err(DurableError::ExecutionLocked { .. }) => {
1305                    // A live owner holds this execution — never abort on staleness alone (INV-17).
1306                }
1307                Err(e) => return Err(e),
1308            }
1309        }
1310        Ok(crate::retention::SweepBatchOutcome {
1311            scanned,
1312            aborted,
1313            next_cursor,
1314        })
1315    }
1316
1317    /// Seal a plaintext payload, or pass it through verbatim when no cipher is configured.
1318    fn seal_payload(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, DurableError> {
1319        match &self.cipher {
1320            Some(cipher) => Ok(cipher.seal(plaintext, aad)?),
1321            None => Ok(plaintext.to_vec()),
1322        }
1323    }
1324
1325    /// Open a sealed payload, or copy it through verbatim when no cipher is configured.
1326    fn open_payload(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Bytes, DurableError> {
1327        match &self.cipher {
1328            Some(cipher) => Ok(Bytes::from(cipher.open(sealed, aad)?)),
1329            None => Ok(Bytes::copy_from_slice(sealed)),
1330        }
1331    }
1332
1333    /// Compute the keyed-BLAKE3 row HMAC over a control entry's identity, when an HMAC key is set.
1334    ///
1335    /// Binds `(execution_id, step_id, entry_kind, idem_key?)` so a control row cannot be forged or
1336    /// relocated on a shared database. Returns `None` when no key is configured (single-user local).
1337    fn control_hmac(
1338        &self,
1339        entry: &JournalEntry,
1340        idem_key: Option<&IdempotencyKey>,
1341    ) -> Option<Vec<u8>> {
1342        self.compute_control_hmac(
1343            entry.execution_id,
1344            entry.step_id,
1345            entry.entry.tag(),
1346            idem_key,
1347        )
1348        .map(|h| h.to_vec())
1349    }
1350
1351    /// Core keyed-BLAKE3 computation shared by [`control_hmac`](Self::control_hmac) (write path,
1352    /// takes a full [`JournalEntry`]) and [`verify_control_hmac`](Self::verify_control_hmac) (read
1353    /// path, which has the row's identity fields but not yet a reconstructed entry). Returns `None`
1354    /// when no HMAC key is configured (single-user local).
1355    fn compute_control_hmac(
1356        &self,
1357        execution_id: ExecutionId,
1358        step_id: StepId,
1359        tag: &'static str,
1360        idem_key: Option<&IdempotencyKey>,
1361    ) -> Option<[u8; 32]> {
1362        let key = self.hmac_key.as_ref()?;
1363        let mut input = Vec::with_capacity(16 + 4 + 16 + 32);
1364        input.extend_from_slice(execution_id.as_bytes());
1365        input.extend_from_slice(&step_id.value().to_le_bytes());
1366        input.extend_from_slice(tag.as_bytes());
1367        if let Some(k) = idem_key {
1368            input.extend_from_slice(k.as_bytes());
1369        }
1370        Some(*blake3::keyed_hash(key, &input).as_bytes())
1371    }
1372
1373    /// Recompute and constant-time-verify a control entry's row HMAC read back from storage
1374    /// (INV-8).
1375    ///
1376    /// A no-op only when no HMAC key is configured **and** the row carries no stored HMAC — the
1377    /// documented single-user local stance where control entries carry no HMAC and none is
1378    /// enforced. If this backend is unkeyed but the row *does* carry a stamped HMAC, that is
1379    /// config drift between the writer and this reader (e.g. `shared_db` toggled, or a reader
1380    /// whose config disagrees with the writer's over the same physical file) and is rejected
1381    /// fail-closed rather than silently trusted, since an `EffectIntent`'s fields are plaintext
1382    /// and an unkeyed reader has no way to tell a genuine stamped row from a forged one. When a
1383    /// key *is* configured, every control row this backend reads must carry a matching HMAC: a
1384    /// missing HMAC or a mismatch both indicate the row was forged, relocated, or written without
1385    /// the configured key, and both fail closed with [`DurableError::ControlIntegrity`].
1386    ///
1387    /// The comparison uses [`blake3::Hash`] equality, which compares in constant time (the same
1388    /// idiom used for the promise resolver-token check in `promise.rs`), so a forged HMAC reveals
1389    /// no timing signal.
1390    fn verify_control_hmac(
1391        &self,
1392        execution_id: ExecutionId,
1393        step_id: StepId,
1394        tag: &'static str,
1395        idem_key: Option<&IdempotencyKey>,
1396        stored: Option<[u8; 32]>,
1397    ) -> Result<(), DurableError> {
1398        let Some(expected) = self.compute_control_hmac(execution_id, step_id, tag, idem_key) else {
1399            return if stored.is_some() {
1400                Err(DurableError::ControlIntegrity)
1401            } else {
1402                Ok(())
1403            };
1404        };
1405        match stored {
1406            Some(stored) if blake3::Hash::from(expected) == blake3::Hash::from(stored) => Ok(()),
1407            _ => Err(DurableError::ControlIntegrity),
1408        }
1409    }
1410
1411    /// Derive the persisted column values for an entry, sealing payloads and stamping HMACs.
1412    fn prepare_row(&self, entry: &JournalEntry) -> Result<JournalRow, DurableError> {
1413        let execution_id = entry.execution_id.as_uuid().to_string();
1414        let step_id = i64::from(entry.step_id.value());
1415        let created_at = entry.created_at_ms;
1416        let entry_kind = entry.entry.tag();
1417        match &entry.entry {
1418            EntryKind::StepResult {
1419                idempotency_key,
1420                payload,
1421                effect,
1422                payload_version,
1423            } => {
1424                ensure_payload_within_limit(payload.len(), self.max_payload_bytes)?;
1425                let aad = PayloadAad::new(
1426                    entry.execution_id,
1427                    entry.step_id,
1428                    EntryKindTag::StepResult,
1429                    Some(*idempotency_key),
1430                );
1431                let sealed = self.seal_payload(payload.as_ref(), &aad)?;
1432                Ok(JournalRow {
1433                    execution_id,
1434                    step_id,
1435                    entry_kind,
1436                    idem_key: Some(idempotency_key.as_bytes().to_vec()),
1437                    effect_class: Some(effect.as_str()),
1438                    payload: Some(sealed),
1439                    payload_version: Some(i32::from(*payload_version)),
1440                    hmac: None,
1441                    created_at,
1442                })
1443            }
1444            EntryKind::EffectIntent {
1445                idempotency_key,
1446                effect,
1447                hmac: _,
1448            } => {
1449                // The backend is the HMAC keyholder; it stamps the row HMAC itself when configured
1450                // and ignores any caller-supplied value.
1451                let hmac = self.control_hmac(entry, Some(idempotency_key));
1452                Ok(JournalRow {
1453                    execution_id,
1454                    step_id,
1455                    entry_kind,
1456                    idem_key: Some(idempotency_key.as_bytes().to_vec()),
1457                    effect_class: Some(effect.as_str()),
1458                    payload: None,
1459                    payload_version: None,
1460                    hmac,
1461                    created_at,
1462                })
1463            }
1464            EntryKind::PromiseCreated { .. }
1465            | EntryKind::PromiseResolved { .. }
1466            | EntryKind::TimerArmed { .. }
1467            | EntryKind::TimerFired { .. }
1468            | EntryKind::Checkpoint { .. } => {
1469                Err(DurableError::UnsupportedEntryKind { kind: entry_kind })
1470            }
1471        }
1472    }
1473
1474    /// Look up the owning execution's kind for read-time entry reconstruction.
1475    async fn lookup_kind(&self, id: ExecutionId) -> Result<ExecutionKind, DurableError> {
1476        let kind: Option<String> = zeph_db::query_scalar(sql!(
1477            "SELECT kind FROM durable_executions WHERE execution_id = ?"
1478        ))
1479        .bind(id.as_uuid().to_string())
1480        .fetch_optional(&self.pool)
1481        .await
1482        .map_err(|e| DurableError::storage("read", e))?;
1483        let kind = kind.ok_or(DurableError::Decode {
1484            context: "journaled entries reference a missing execution row",
1485        })?;
1486        ExecutionKind::from_tag(&kind).ok_or(DurableError::Decode {
1487            context: "execution kind is not reconstructible (custom kind read-back unsupported)",
1488        })
1489    }
1490
1491    /// Reconstruct a [`JournalEntry`] from a stored row, opening sealed payloads.
1492    fn row_to_entry(
1493        &self,
1494        id: ExecutionId,
1495        kind: ExecutionKind,
1496        row: JournalRowRead,
1497    ) -> Result<JournalEntry, DurableError> {
1498        let (
1499            seq,
1500            step_id_raw,
1501            entry_kind,
1502            idem_key,
1503            effect_class,
1504            payload,
1505            payload_version,
1506            hmac,
1507            created_at,
1508        ) = row;
1509        let step_id =
1510            StepId::new(
1511                u32::try_from(step_id_raw).map_err(|_| DurableError::Decode {
1512                    context: "step_id out of u32 range",
1513                })?,
1514            );
1515        let entry = match entry_kind.as_str() {
1516            "step_result" => {
1517                let idem_bytes = idem_key.ok_or(DurableError::Decode {
1518                    context: "step_result idem_key missing",
1519                })?;
1520                let idem_key = IdempotencyKey::from_bytes(slice_to_array32(
1521                    &idem_bytes,
1522                    "step_result idem_key",
1523                )?);
1524                let effect = effect_class
1525                    .as_deref()
1526                    .and_then(crate::EffectClass::from_tag)
1527                    .ok_or(DurableError::Decode {
1528                        context: "step_result effect_class missing or invalid",
1529                    })?;
1530                let sealed = payload.ok_or(DurableError::Decode {
1531                    context: "step_result payload missing",
1532                })?;
1533                ensure_payload_within_limit(
1534                    sealed.len(),
1535                    self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1536                )?;
1537                let aad = PayloadAad::new(id, step_id, EntryKindTag::StepResult, Some(idem_key));
1538                let opened = self.open_payload(&sealed, &aad)?;
1539                let version = u8::try_from(payload_version.unwrap_or(1)).map_err(|_| {
1540                    DurableError::Decode {
1541                        context: "payload_version out of u8 range",
1542                    }
1543                })?;
1544                EntryKind::StepResult {
1545                    idempotency_key: idem_key,
1546                    payload: opened,
1547                    effect,
1548                    payload_version: version,
1549                }
1550            }
1551            "effect_intent" => {
1552                let idem_bytes = idem_key.ok_or(DurableError::Decode {
1553                    context: "effect_intent idem_key missing",
1554                })?;
1555                let idem_key = IdempotencyKey::from_bytes(slice_to_array32(
1556                    &idem_bytes,
1557                    "effect_intent idem_key",
1558                )?);
1559                let effect = effect_class
1560                    .as_deref()
1561                    .and_then(crate::EffectClass::from_tag)
1562                    .ok_or(DurableError::Decode {
1563                        context: "effect_intent effect_class missing or invalid",
1564                    })?;
1565                let hmac = hmac
1566                    .map(|bytes| slice_to_array32(&bytes, "effect_intent hmac"))
1567                    .transpose()?;
1568                self.verify_control_hmac(
1569                    id,
1570                    step_id,
1571                    EntryKindTag::EffectIntent.as_str(),
1572                    Some(&idem_key),
1573                    hmac,
1574                )?;
1575                EntryKind::EffectIntent {
1576                    idempotency_key: idem_key,
1577                    effect,
1578                    hmac,
1579                }
1580            }
1581            "checkpoint" => self.checkpoint_entry(id, step_id, payload)?,
1582            other => {
1583                return Err(DurableError::UnsupportedEntryKind {
1584                    kind: static_entry_tag(other),
1585                });
1586            }
1587        };
1588        Ok(JournalEntry {
1589            seq: Some(JournalSeq::new(seq)),
1590            execution_id: id,
1591            kind,
1592            step_id,
1593            entry,
1594            created_at_ms: created_at,
1595        })
1596    }
1597
1598    /// Reconstruct a [`EntryKind::Checkpoint`] from a stored row, opening its sealed snapshot.
1599    ///
1600    /// `step_id` carries the checkpoint's `up_to_step` (the fold boundary); the snapshot is bound to
1601    /// it in the AAD so a checkpoint blob cannot be relocated to a different fold boundary.
1602    fn checkpoint_entry(
1603        &self,
1604        id: ExecutionId,
1605        step_id: StepId,
1606        payload: Option<Vec<u8>>,
1607    ) -> Result<EntryKind, DurableError> {
1608        let sealed = payload.ok_or(DurableError::Decode {
1609            context: "checkpoint entry missing snapshot payload",
1610        })?;
1611        ensure_payload_within_limit(
1612            sealed.len(),
1613            self.max_payload_bytes.saturating_add(SEAL_OVERHEAD_SLACK),
1614        )?;
1615        let aad = PayloadAad::new(id, step_id, EntryKindTag::Checkpoint, None);
1616        let snapshot = self.open_payload(&sealed, &aad)?;
1617        Ok(EntryKind::Checkpoint {
1618            up_to_step: step_id.value(),
1619            snapshot,
1620        })
1621    }
1622
1623    /// Reconstruct every entry from a fetched row set, sharing one kind lookup.
1624    async fn rows_to_entries(
1625        &self,
1626        id: ExecutionId,
1627        rows: Vec<JournalRowRead>,
1628    ) -> Result<Vec<JournalEntry>, DurableError> {
1629        if rows.is_empty() {
1630            return Ok(Vec::new());
1631        }
1632        let kind = self.lookup_kind(id).await?;
1633        let mut entries = Vec::with_capacity(rows.len());
1634        for row in rows {
1635            entries.push(self.row_to_entry(id, kind, row)?);
1636        }
1637        Ok(entries)
1638    }
1639}
1640
1641impl Journal for LocalBackend {
1642    async fn append(&self, entry: JournalEntry) -> Result<JournalSeq, DurableError> {
1643        let span = tracing::info_span!(
1644            "durable.journal.append",
1645            execution_id = %entry.execution_id.as_uuid(),
1646            step_id = entry.step_id.value(),
1647            entry_kind = entry.entry.tag(),
1648        );
1649        async move {
1650            let row = self.prepare_row(&entry)?;
1651            let (seq,): (i64,) = zeph_db::query_as(sql!(
1652                "INSERT INTO durable_journal
1653                    (execution_id, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)
1654                 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
1655                 RETURNING seq"
1656            ))
1657                .bind(row.execution_id)
1658                .bind(row.step_id)
1659                .bind(row.entry_kind)
1660                .bind(row.idem_key)
1661                .bind(row.effect_class)
1662                .bind(row.payload)
1663                .bind(row.payload_version)
1664                .bind(row.hmac)
1665                .bind(row.created_at)
1666                .fetch_one(&self.pool)
1667                .await
1668                .map_err(|e| DurableError::storage("append", e))?;
1669            Ok(JournalSeq::new(seq))
1670        }
1671        .instrument(span)
1672        .await
1673    }
1674
1675    async fn read_execution(&self, id: ExecutionId) -> Result<Vec<JournalEntry>, DurableError> {
1676        let span = tracing::info_span!(
1677            "durable.journal.read",
1678            execution_id = %id.as_uuid(),
1679            step_count = tracing::field::Empty,
1680        );
1681        async move {
1682            let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
1683                "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
1684                 FROM durable_journal WHERE execution_id = ? ORDER BY seq"
1685            ))
1686            .bind(id.as_uuid().to_string())
1687            .fetch_all(&self.pool)
1688            .await
1689            .map_err(|e| DurableError::storage("read", e))?;
1690            let entries = self.rows_to_entries(id, rows).await?;
1691            tracing::Span::current().record("step_count", entries.len());
1692            Ok(entries)
1693        }
1694        .instrument(span)
1695        .await
1696    }
1697
1698    async fn read_execution_range(
1699        &self,
1700        id: ExecutionId,
1701        from_step_id: u32,
1702        limit: usize,
1703    ) -> Result<Vec<JournalEntry>, DurableError> {
1704        let span = tracing::info_span!(
1705            "durable.journal.read_segment",
1706            execution_id = %id.as_uuid(),
1707            from_step_id,
1708            count = tracing::field::Empty,
1709        );
1710        async move {
1711            let rows: Vec<JournalRowRead> = zeph_db::query_as(sql!(
1712                "SELECT seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at
1713                 FROM durable_journal WHERE execution_id = ? AND step_id >= ? ORDER BY step_id, seq LIMIT ?"
1714            ))
1715            .bind(id.as_uuid().to_string())
1716            .bind(i64::from(from_step_id))
1717            .bind(i64::try_from(limit).unwrap_or(i64::MAX))
1718            .fetch_all(&self.pool)
1719            .await
1720            .map_err(|e| DurableError::storage("read_segment", e))?;
1721            let entries = self.rows_to_entries(id, rows).await?;
1722            tracing::Span::current().record("count", entries.len());
1723            Ok(entries)
1724        }
1725        .instrument(span)
1726        .await
1727    }
1728
1729    async fn finalize(&self, id: ExecutionId, status: ExecutionStatus) -> Result<(), DurableError> {
1730        let span = tracing::info_span!(
1731            "durable.journal.finalize",
1732            execution_id = %id.as_uuid(),
1733            status = status.as_str(),
1734        );
1735        async move {
1736            let now = now_unix_millis();
1737            let finalized_at = (!status.is_running()).then_some(now);
1738            let mut tx = zeph_db::begin_write(&self.pool)
1739                .await
1740                .map_err(|e| DurableError::storage("finalize", e))?;
1741            // `AND status = 'running'` makes this a one-shot transition: whichever of a concurrent
1742            // divergence-triggered `Aborted` or a caller's `Completed`/`Failed` commits first wins,
1743            // and the loser's UPDATE affects zero rows instead of clobbering the winner's terminal
1744            // status (finalize is otherwise safe to call more than once per execution).
1745            zeph_db::query(sql!(
1746                "UPDATE durable_executions SET status = ?, updated_at = ?, finalized_at = ?
1747                 WHERE execution_id = ? AND status = 'running'"
1748            ))
1749            .bind(status.as_str())
1750            .bind(now)
1751            .bind(finalized_at)
1752            .bind(id.as_uuid().to_string())
1753            .execute(&mut *tx)
1754            .await
1755            .map_err(|e| DurableError::storage("finalize", e))?;
1756            tx.commit()
1757                .await
1758                .map_err(|e| DurableError::storage("finalize", e))?;
1759            Ok(())
1760        }
1761        .instrument(span)
1762        .await
1763    }
1764
1765    async fn prune(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
1766        let now = now_unix_millis();
1767        crate::retention::prune_in_batches(policy, now, |cutoffs, batch| {
1768            self.delete_prune_batch(cutoffs, batch)
1769        })
1770        .await
1771    }
1772
1773    /// Crash-orphan reclamation (INV-17, #6254). See [`Journal::sweep_orphans`] for the contract.
1774    async fn sweep_orphans(&self, policy: &RetentionPolicy) -> Result<u64, DurableError> {
1775        if policy.stale_running_after_secs == 0 {
1776            return Ok(0);
1777        }
1778        let Some(lock_dir) = self.lock_dir.clone() else {
1779            if !self
1780                .orphan_sweep_warned
1781                .swap(true, std::sync::atomic::Ordering::Relaxed)
1782            {
1783                tracing::warn!(
1784                    "durable: crash-orphan sweep requires an on-disk advisory-lock dir; orphan \
1785                     reclamation disabled for this backend (Postgres/:memory:/non-Unix)"
1786                );
1787            }
1788            return Ok(0);
1789        };
1790        let cutoff_ms = orphan_cutoff_ms(policy, now_unix_millis());
1791        crate::retention::sweep_orphans_in_batches(
1792            policy.prune_batch_size,
1793            cutoff_ms,
1794            |cutoff, batch, cursor| self.sweep_orphan_batch(&lock_dir, cutoff, batch, cursor),
1795        )
1796        .await
1797    }
1798}
1799
1800impl crate::sealed::Sealed for LocalBackend {}
1801
1802impl ExecutionBackend for LocalBackend {
1803    fn capabilities(&self) -> BackendCapabilities {
1804        BackendCapabilities {
1805            parallel_steps: true,
1806            // The local backend is in-process on SQLite; a Postgres build talks to a shared server.
1807            cross_process: cfg!(feature = "postgres"),
1808            max_payload: usize::try_from(self.max_payload_bytes).unwrap_or(usize::MAX),
1809        }
1810    }
1811
1812    async fn lookup_committed_result(
1813        &self,
1814        id: ExecutionId,
1815        idem_key: IdempotencyKey,
1816    ) -> Result<Option<JournalEntry>, DurableError> {
1817        LocalBackend::lookup_committed_result(self, id, idem_key).await
1818    }
1819}
1820
1821/// Column values for a single `durable_journal` row, ready to bind.
1822struct JournalRow {
1823    execution_id: String,
1824    step_id: i64,
1825    entry_kind: &'static str,
1826    idem_key: Option<Vec<u8>>,
1827    effect_class: Option<&'static str>,
1828    payload: Option<Vec<u8>>,
1829    payload_version: Option<i32>,
1830    hmac: Option<Vec<u8>>,
1831    created_at: i64,
1832}
1833
1834/// A `durable_journal` row read back from storage, decoded dialect-agnostically.
1835///
1836/// Columns are read as a positional tuple (the convention for crates that depend on `zeph-db` but
1837/// not `sqlx` directly, mirroring `zeph-scheduler`): integers decode as `i64`/`i32` and blobs as
1838/// `Vec<u8>`, which both backends satisfy through the same `sql!()`-rewritten query. The
1839/// field order matches the `SELECT` column list:
1840/// `(seq, step_id, entry_kind, idem_key, effect_class, payload, payload_version, hmac, created_at)`.
1841type JournalRowRead = (
1842    i64,
1843    i64,
1844    String,
1845    Option<Vec<u8>>,
1846    Option<String>,
1847    Option<Vec<u8>>,
1848    Option<i32>,
1849    Option<Vec<u8>>,
1850    i64,
1851);
1852
1853/// A `durable_promises` row read back from storage, in `SELECT` column order:
1854/// `(execution_id, resolver_token_hash, resolved, payload)`.
1855type PromiseRowRead = (String, Vec<u8>, i64, Option<Vec<u8>>);
1856
1857/// A foldable `durable_journal` step-result row, in `SELECT` column order:
1858/// `(step_id, idem_key, payload_version, payload)`.
1859type FoldableRowRead = (i64, Option<Vec<u8>>, Option<i32>, Option<Vec<u8>>);
1860
1861/// Derive the per-execution lock directory sibling to a `path` passed to
1862/// [`LocalBackend::open`], `None` for `:memory:`.
1863///
1864/// Feature-gated to the `SQLite` backend only (INV-15): under `postgres`, `path` is a connection
1865/// URL that may embed credentials, and appending a suffix to mint a directory name would risk
1866/// creating a secret-bearing path component on disk.
1867#[cfg(feature = "sqlite")]
1868fn lock_dir_for_path(path: &str) -> Option<std::path::PathBuf> {
1869    (path != ":memory:").then(|| std::path::PathBuf::from(format!("{path}.locks")))
1870}
1871
1872#[cfg(not(feature = "sqlite"))]
1873fn lock_dir_for_path(_path: &str) -> Option<std::path::PathBuf> {
1874    None
1875}
1876
1877/// Regression coverage for the `postgres`-only branch of [`lock_dir_for_path`] (INV-15): a
1878/// connection URL — which may embed credentials — must never be used to mint an on-disk lock
1879/// directory name. The main `mod tests` block below is gated on `feature = "sqlite"` and so never
1880/// exercises this branch; run with `cargo nextest run -p zeph-durable --no-default-features
1881/// --features postgres`.
1882#[cfg(all(test, not(feature = "sqlite")))]
1883mod postgres_lock_dir_tests {
1884    use super::lock_dir_for_path;
1885
1886    #[test]
1887    fn postgres_url_never_derives_a_lock_dir() {
1888        assert_eq!(
1889            lock_dir_for_path("postgres://user:secret@host/db"),
1890            None,
1891            "a Postgres connection URL (which may embed credentials) must never be used to mint \
1892             an on-disk lock directory name"
1893        );
1894        assert_eq!(lock_dir_for_path(":memory:"), None);
1895    }
1896}
1897
1898/// Current Unix time in milliseconds, clamped into `i64` and never panicking.
1899pub(crate) fn now_unix_millis() -> i64 {
1900    SystemTime::now()
1901        .duration_since(UNIX_EPOCH)
1902        .map_or(0, |d| i64::try_from(d.as_millis()).unwrap_or(i64::MAX))
1903}
1904
1905/// The absolute `updated_at` cutoff (Unix ms) at or before which a `status='running'` row becomes
1906/// a crash-orphan sweep candidate (INV-17, #6254).
1907fn orphan_cutoff_ms(policy: &RetentionPolicy, now_ms: i64) -> i64 {
1908    let threshold =
1909        i64::try_from(policy.stale_running_after_secs.saturating_mul(1000)).unwrap_or(i64::MAX);
1910    now_ms.saturating_sub(threshold)
1911}
1912
1913/// Decode a stored blob into a fixed 32-byte array, failing closed on the wrong length.
1914fn slice_to_array32(bytes: &[u8], field: &'static str) -> Result<[u8; 32], DurableError> {
1915    <[u8; 32]>::try_from(bytes).map_err(|_| DurableError::Decode { context: field })
1916}
1917
1918/// Parse a stored `execution_id` TEXT column back into an [`ExecutionId`], failing closed.
1919fn parse_execution_id(text: &str) -> Result<ExecutionId, DurableError> {
1920    uuid::Uuid::parse_str(text)
1921        .map(ExecutionId::from_uuid)
1922        .map_err(|_| DurableError::Decode {
1923            context: "execution_id is not a valid UUID",
1924        })
1925}
1926
1927/// Parse a stored `timer_id` TEXT column back into a [`TimerId`], failing closed.
1928fn parse_timer_id(text: &str) -> Result<TimerId, DurableError> {
1929    uuid::Uuid::parse_str(text)
1930        .map(TimerId::from_uuid)
1931        .map_err(|_| DurableError::Decode {
1932            context: "timer_id is not a valid UUID",
1933        })
1934}
1935
1936/// The AAD binding a promise's resolved payload to `(execution_id, promise_id)`.
1937///
1938/// A promise has no [`StepId`], so the promise id is folded into the AAD's idempotency-key slot:
1939/// a payload sealed for one promise cannot be opened as another's (fail-closed on relocation).
1940fn promise_payload_aad(execution_id: ExecutionId, promise_id: PromiseId) -> PayloadAad {
1941    let binding = IdempotencyKey::derive(
1942        execution_id,
1943        StepId::new(0),
1944        promise_id.as_uuid().as_bytes(),
1945    );
1946    PayloadAad::new(
1947        execution_id,
1948        StepId::new(0),
1949        EntryKindTag::PromiseResolved,
1950        Some(binding),
1951    )
1952}
1953
1954/// Map a database `entry_kind` string to a `'static` tag for [`DurableError::UnsupportedEntryKind`].
1955fn static_entry_tag(tag: &str) -> &'static str {
1956    match tag {
1957        "promise_created" => "promise_created",
1958        "promise_resolved" => "promise_resolved",
1959        "timer_armed" => "timer_armed",
1960        "timer_fired" => "timer_fired",
1961        "checkpoint" => "checkpoint",
1962        _ => "unknown",
1963    }
1964}
1965
1966// Backend tests open a real pool, so they run under the SQLite build (mirroring `zeph-scheduler`,
1967// whose `:memory:` pool is SQLite-specific). The dialect-agnostic `sql!()` SQL and `i64`/`Vec<u8>`
1968// column types are verified to compile under the Postgres feature; live Postgres parity is exercised
1969// by the `#[ignore]`d integration test below.
1970#[cfg(all(test, feature = "sqlite"))]
1971mod tests {
1972    use std::assert_matches;
1973
1974    use super::*;
1975    use crate::cipher::CipherError;
1976    use crate::effect::EffectClass;
1977
1978    /// An AAD-authenticated test cipher: a BLAKE3 tag over the AAD prefixes an XOR-masked payload,
1979    /// so opening with a relocated/forged AAD fails authentication exactly like the real cipher.
1980    struct XorCipher;
1981    const XOR_MASK: u8 = 0x5A;
1982
1983    impl PayloadCipher for XorCipher {
1984        fn seal(&self, plaintext: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
1985            let tag = blake3::hash(&aad.canonical_bytes());
1986            let mut out = tag.as_bytes()[..8].to_vec();
1987            out.extend(plaintext.iter().map(|b| b ^ XOR_MASK));
1988            Ok(out)
1989        }
1990
1991        fn open(&self, sealed: &[u8], aad: &PayloadAad) -> Result<Vec<u8>, CipherError> {
1992            if sealed.len() < 8 {
1993                return Err(CipherError::Malformed {
1994                    context: "sealed blob shorter than the aad tag",
1995                });
1996            }
1997            let expected = blake3::hash(&aad.canonical_bytes());
1998            if sealed[..8] != expected.as_bytes()[..8] {
1999                return Err(CipherError::Authentication);
2000            }
2001            Ok(sealed[8..].iter().map(|b| b ^ XOR_MASK).collect())
2002        }
2003    }
2004
2005    async fn mem_backend(max_payload_bytes: u64) -> LocalBackend {
2006        let backend = LocalBackend::open(":memory:", max_payload_bytes)
2007            .await
2008            .expect("open in-memory backend");
2009        backend.init().await.expect("apply migrations");
2010        backend
2011    }
2012
2013    fn step_result(exec: ExecutionId, step: u32, payload: &[u8]) -> JournalEntry {
2014        let step_id = StepId::new(step);
2015        JournalEntry {
2016            seq: None,
2017            execution_id: exec,
2018            kind: ExecutionKind::AgentTurn,
2019            step_id,
2020            entry: EntryKind::StepResult {
2021                idempotency_key: IdempotencyKey::derive(exec, step_id, b"tool:read"),
2022                payload: Bytes::copy_from_slice(payload),
2023                effect: EffectClass::Idempotent,
2024                payload_version: 1,
2025            },
2026            created_at_ms: 100,
2027        }
2028    }
2029
2030    fn effect_intent(exec: ExecutionId, step: u32) -> JournalEntry {
2031        let step_id = StepId::new(step);
2032        JournalEntry {
2033            seq: None,
2034            execution_id: exec,
2035            kind: ExecutionKind::AgentTurn,
2036            step_id,
2037            entry: EntryKind::EffectIntent {
2038                idempotency_key: IdempotencyKey::derive(exec, step_id, b"transfer"),
2039                effect: EffectClass::ExactlyOnceGuarded,
2040                hmac: None,
2041            },
2042            created_at_ms: 100,
2043        }
2044    }
2045
2046    #[tokio::test]
2047    async fn open_execution_is_fresh_then_resume() {
2048        let backend = mem_backend(1_048_576).await;
2049        let exec = ExecutionId::new();
2050        assert!(
2051            !backend
2052                .open_execution(exec, ExecutionKind::AgentTurn)
2053                .await
2054                .unwrap()
2055        );
2056        assert!(
2057            backend
2058                .open_execution(exec, ExecutionKind::AgentTurn)
2059                .await
2060                .unwrap()
2061        );
2062    }
2063
2064    #[tokio::test]
2065    async fn open_execution_exclusive_is_fresh_then_resume() {
2066        // A file-backed (not `:memory:`) backend is required: only `LocalBackend::open` with a
2067        // real on-disk path derives a `lock_dir` (#6122).
2068        let dir = tempfile::tempdir().unwrap();
2069        let db_path = dir.path().join("durable.db");
2070        let backend = LocalBackend::open(&db_path.to_string_lossy(), 1_048_576)
2071            .await
2072            .unwrap();
2073        backend.init().await.unwrap();
2074
2075        let exec = ExecutionId::new();
2076        let (is_resume, lock) = backend
2077            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
2078            .await
2079            .unwrap();
2080        assert!(!is_resume);
2081        assert!(lock.is_some(), "a file-backed backend must derive a lock");
2082        drop(lock);
2083
2084        let (is_resume, _lock) = backend
2085            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
2086            .await
2087            .unwrap();
2088        assert!(is_resume);
2089    }
2090
2091    /// Regression test for #6122: two `LocalBackend` handles onto the same on-disk journal (as
2092    /// two independent CLI processes sharing `memory.sqlite_path` would each construct) must not
2093    /// both be able to hold `open_execution_exclusive` for the same colliding `ExecutionId`
2094    /// concurrently.
2095    #[tokio::test]
2096    async fn open_execution_exclusive_rejects_concurrent_second_holder() {
2097        let dir = tempfile::tempdir().unwrap();
2098        let db_path = dir.path().join("durable.db");
2099        let url = db_path.to_string_lossy().into_owned();
2100
2101        let backend_a = LocalBackend::open(&url, 1_048_576).await.unwrap();
2102        backend_a.init().await.unwrap();
2103        let backend_b = LocalBackend::open(&url, 1_048_576).await.unwrap();
2104
2105        let exec = ExecutionId::new();
2106        let (_, _lock_a) = backend_a
2107            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
2108            .await
2109            .unwrap();
2110
2111        let err = backend_b
2112            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
2113            .await
2114            .expect_err("a second concurrent holder must be rejected");
2115        assert!(
2116            matches!(err, DurableError::ExecutionLocked { execution_id, .. } if execution_id == exec),
2117            "expected ExecutionLocked, got {err:?}"
2118        );
2119    }
2120
2121    #[tokio::test]
2122    async fn open_execution_exclusive_on_memory_backend_returns_no_lock() {
2123        // `:memory:` has no on-disk directory to lock, so it degrades to unenforced exclusivity —
2124        // consistent with `SessionEventLog::open_exclusive`'s non-Unix degrade.
2125        let backend = mem_backend(1_048_576).await;
2126        let exec = ExecutionId::new();
2127        let (is_resume, lock) = backend
2128            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
2129            .await
2130            .unwrap();
2131        assert!(!is_resume);
2132        assert!(lock.is_none());
2133    }
2134
2135    #[tokio::test]
2136    async fn list_executions_summarizes_and_filters() {
2137        let backend = mem_backend(1_048_576).await;
2138        let turn = ExecutionId::new();
2139        let dag = ExecutionId::new();
2140        backend
2141            .open_execution(turn, ExecutionKind::AgentTurn)
2142            .await
2143            .unwrap();
2144        backend
2145            .open_execution(dag, ExecutionKind::DagRun)
2146            .await
2147            .unwrap();
2148        backend.append(step_result(turn, 0, b"a")).await.unwrap();
2149        backend.append(step_result(turn, 1, b"b")).await.unwrap();
2150        backend.append(step_result(dag, 0, b"c")).await.unwrap();
2151        backend
2152            .finalize(turn, ExecutionStatus::Completed)
2153            .await
2154            .unwrap();
2155
2156        // Unfiltered: both executions with their per-execution step counts.
2157        let all = backend.list_executions(None, None, 10).await.unwrap();
2158        assert_eq!(all.len(), 2);
2159
2160        let turn_row = all
2161            .iter()
2162            .find(|e| e.execution_id == turn)
2163            .expect("turn present");
2164        assert_eq!(turn_row.kind, "agent_turn");
2165        assert_eq!(turn_row.status, ExecutionStatus::Completed);
2166        assert_eq!(turn_row.step_count, 2);
2167        assert!(turn_row.finalized_at_ms.is_some());
2168
2169        let dag_row = all
2170            .iter()
2171            .find(|e| e.execution_id == dag)
2172            .expect("dag present");
2173        assert_eq!(dag_row.status, ExecutionStatus::Running);
2174        assert_eq!(dag_row.step_count, 1);
2175        assert!(dag_row.finalized_at_ms.is_none());
2176
2177        // Status filter narrows to the still-running execution.
2178        let running = backend
2179            .list_executions(Some("running"), None, 10)
2180            .await
2181            .unwrap();
2182        assert_eq!(running.len(), 1);
2183        assert_eq!(running[0].execution_id, dag);
2184
2185        // Kind filter narrows to the DAG execution.
2186        let dags = backend
2187            .list_executions(None, Some("dag_run"), 10)
2188            .await
2189            .unwrap();
2190        assert_eq!(dags.len(), 1);
2191        assert_eq!(dags[0].execution_id, dag);
2192
2193        // Limit caps the result set.
2194        let one = backend.list_executions(None, None, 1).await.unwrap();
2195        assert_eq!(one.len(), 1);
2196    }
2197
2198    #[tokio::test]
2199    async fn append_and_read_round_trips_step_result() {
2200        let backend = mem_backend(1_048_576).await;
2201        let exec = ExecutionId::new();
2202        backend
2203            .open_execution(exec, ExecutionKind::AgentTurn)
2204            .await
2205            .unwrap();
2206
2207        let seq = backend
2208            .append(step_result(exec, 0, b"hello"))
2209            .await
2210            .unwrap();
2211        assert_eq!(seq.value(), 1, "first append takes seq 1");
2212
2213        let entries = backend.read_execution(exec).await.unwrap();
2214        assert_eq!(entries.len(), 1);
2215        match &entries[0].entry {
2216            EntryKind::StepResult {
2217                payload, effect, ..
2218            } => {
2219                assert_eq!(payload.as_ref(), b"hello");
2220                assert_eq!(*effect, EffectClass::Idempotent);
2221            }
2222            other => panic!("unexpected entry kind: {other:?}"),
2223        }
2224        assert_eq!(entries[0].seq, Some(seq));
2225    }
2226
2227    #[tokio::test]
2228    async fn cipher_seals_payload_at_rest_but_round_trips() {
2229        let backend = mem_backend(1_048_576)
2230            .await
2231            .with_cipher(Arc::new(XorCipher));
2232        let exec = ExecutionId::new();
2233        backend
2234            .open_execution(exec, ExecutionKind::AgentTurn)
2235            .await
2236            .unwrap();
2237        backend
2238            .append(step_result(exec, 0, b"secret-payload"))
2239            .await
2240            .unwrap();
2241
2242        // The stored column is sealed, never the plaintext.
2243        let (stored,): (Option<Vec<u8>>,) = zeph_db::query_as(sql!(
2244            "SELECT payload FROM durable_journal WHERE execution_id = ?"
2245        ))
2246        .bind(exec.as_uuid().to_string())
2247        .fetch_one(backend.pool())
2248        .await
2249        .unwrap();
2250        let stored = stored.expect("payload present");
2251        assert_ne!(
2252            stored.as_slice(),
2253            b"secret-payload",
2254            "payload must be sealed at rest"
2255        );
2256
2257        // Reading opens it back to the original plaintext.
2258        let entries = backend.read_execution(exec).await.unwrap();
2259        match &entries[0].entry {
2260            EntryKind::StepResult { payload, .. } => {
2261                assert_eq!(payload.as_ref(), b"secret-payload");
2262            }
2263            other => panic!("unexpected entry kind: {other:?}"),
2264        }
2265    }
2266
2267    #[tokio::test]
2268    async fn control_entry_hmac_is_stamped_only_when_keyed() {
2269        let exec = ExecutionId::new();
2270
2271        let unkeyed = mem_backend(1_048_576).await;
2272        unkeyed
2273            .open_execution(exec, ExecutionKind::AgentTurn)
2274            .await
2275            .unwrap();
2276        unkeyed.append(effect_intent(exec, 0)).await.unwrap();
2277        match &unkeyed.read_execution(exec).await.unwrap()[0].entry {
2278            EntryKind::EffectIntent { hmac, .. } => assert!(hmac.is_none()),
2279            other => panic!("unexpected entry kind: {other:?}"),
2280        }
2281
2282        let keyed = mem_backend(1_048_576).await.with_hmac_key([7u8; 32]);
2283        let exec2 = ExecutionId::new();
2284        keyed
2285            .open_execution(exec2, ExecutionKind::AgentTurn)
2286            .await
2287            .unwrap();
2288        keyed.append(effect_intent(exec2, 0)).await.unwrap();
2289        match &keyed.read_execution(exec2).await.unwrap()[0].entry {
2290            EntryKind::EffectIntent { hmac, .. } => {
2291                assert!(
2292                    hmac.is_some(),
2293                    "keyed backend stamps a row HMAC over control entries"
2294                );
2295            }
2296            other => panic!("unexpected entry kind: {other:?}"),
2297        }
2298    }
2299
2300    /// Regression for #6043/#6044: a control entry written under one HMAC key must fail closed
2301    /// with [`DurableError::ControlIntegrity`] when read back under a *different* key — the
2302    /// forged/relocated-row rejection the row HMAC exists to provide. Both backends share the
2303    /// same underlying pool (a second `LocalBackend` handle over the same connection), so this
2304    /// exercises the read path's recompute-and-compare, not just a difference in whether a key is
2305    /// configured at all.
2306    #[tokio::test]
2307    async fn read_execution_rejects_control_hmac_under_wrong_key() {
2308        let writer = mem_backend(1_048_576).await.with_hmac_key([1u8; 32]);
2309        let exec = ExecutionId::new();
2310        writer
2311            .open_execution(exec, ExecutionKind::AgentTurn)
2312            .await
2313            .unwrap();
2314        writer.append(effect_intent(exec, 0)).await.unwrap();
2315
2316        let wrong_key_reader =
2317            LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([2u8; 32]);
2318        assert_matches!(
2319            wrong_key_reader.read_execution(exec).await,
2320            Err(DurableError::ControlIntegrity)
2321        );
2322
2323        // Reading under the correct key still succeeds.
2324        let right_key_reader =
2325            LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([1u8; 32]);
2326        assert!(right_key_reader.read_execution(exec).await.is_ok());
2327    }
2328
2329    /// Regression for #6043/#6044: a control entry written by an *unkeyed* backend (`hmac =
2330    /// NULL`) must fail closed when later read by a keyed backend — a keyed backend enforces that
2331    /// every control row it reads carries a matching HMAC, so a missing HMAC is treated the same
2332    /// as a mismatched one rather than silently passing through unverified.
2333    #[tokio::test]
2334    async fn read_execution_rejects_missing_hmac_on_keyed_backend() {
2335        let writer = mem_backend(1_048_576).await;
2336        let exec = ExecutionId::new();
2337        writer
2338            .open_execution(exec, ExecutionKind::AgentTurn)
2339            .await
2340            .unwrap();
2341        writer.append(effect_intent(exec, 0)).await.unwrap();
2342
2343        let keyed_reader =
2344            LocalBackend::new(writer.pool().clone(), 1_048_576).with_hmac_key([3u8; 32]);
2345        assert_matches!(
2346            keyed_reader.read_execution(exec).await,
2347            Err(DurableError::ControlIntegrity)
2348        );
2349    }
2350
2351    /// Regression for #6043/#6044 (review S1): a control entry written by a *keyed* backend must
2352    /// fail closed when later read by an *unkeyed* backend, rather than silently trusting the
2353    /// stamped HMAC as an ordinary (unverified) plaintext field. Before this fix,
2354    /// `verify_control_hmac` returned `Ok(())` unconditionally whenever the reader had no HMAC
2355    /// key, regardless of whether the stored row carried one — so config drift between a keyed
2356    /// writer and an unkeyed reader over the same physical file (e.g. `shared_db` toggled, or a
2357    /// reader whose config disagrees with the writer's) let a stamped row through unverified,
2358    /// which is exactly the forgery-acceptance gap #6043 says the row HMAC closes.
2359    #[tokio::test]
2360    async fn read_execution_rejects_stamped_hmac_on_unkeyed_backend() {
2361        let writer = mem_backend(1_048_576).await.with_hmac_key([4u8; 32]);
2362        let exec = ExecutionId::new();
2363        writer
2364            .open_execution(exec, ExecutionKind::AgentTurn)
2365            .await
2366            .unwrap();
2367        writer.append(effect_intent(exec, 0)).await.unwrap();
2368
2369        let unkeyed_reader = LocalBackend::new(writer.pool().clone(), 1_048_576);
2370        assert_matches!(
2371            unkeyed_reader.read_execution(exec).await,
2372            Err(DurableError::ControlIntegrity)
2373        );
2374    }
2375
2376    #[tokio::test]
2377    async fn promise_and_timer_entries_fail_closed() {
2378        let backend = mem_backend(1_048_576).await;
2379        let exec = ExecutionId::new();
2380        backend
2381            .open_execution(exec, ExecutionKind::AgentTurn)
2382            .await
2383            .unwrap();
2384        let timer = JournalEntry {
2385            seq: None,
2386            execution_id: exec,
2387            kind: ExecutionKind::AgentTurn,
2388            step_id: StepId::new(0),
2389            entry: EntryKind::TimerArmed {
2390                timer_id: crate::TimerId::new(),
2391                due_at_ms: 1_000,
2392                hmac: None,
2393            },
2394            created_at_ms: 0,
2395        };
2396        assert_matches!(
2397            backend.append(timer).await,
2398            Err(DurableError::UnsupportedEntryKind {
2399                kind: "timer_armed"
2400            })
2401        );
2402    }
2403
2404    #[tokio::test]
2405    async fn payload_over_limit_is_rejected_fail_closed() {
2406        let backend = mem_backend(8).await;
2407        let exec = ExecutionId::new();
2408        backend
2409            .open_execution(exec, ExecutionKind::AgentTurn)
2410            .await
2411            .unwrap();
2412        let big = vec![0u8; 64];
2413        assert_matches!(
2414            backend.append(step_result(exec, 0, &big)).await,
2415            Err(DurableError::PayloadTooLarge { .. })
2416        );
2417    }
2418
2419    #[tokio::test]
2420    async fn finalize_marks_terminal_status_and_time() {
2421        let backend = mem_backend(1_048_576).await;
2422        let exec = ExecutionId::new();
2423        backend
2424            .open_execution(exec, ExecutionKind::AgentTurn)
2425            .await
2426            .unwrap();
2427        backend
2428            .finalize(exec, ExecutionStatus::Completed)
2429            .await
2430            .unwrap();
2431
2432        let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
2433            "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
2434        ))
2435        .bind(exec.as_uuid().to_string())
2436        .fetch_one(backend.pool())
2437        .await
2438        .unwrap();
2439        assert_eq!(status, "completed");
2440        assert!(finalized.is_some(), "a terminal status stamps finalized_at");
2441    }
2442
2443    #[tokio::test]
2444    async fn finalize_is_a_noop_once_already_terminal() {
2445        // #6251: finalize must be safe to call more than once (e.g. a caller's own `Completed`
2446        // racing the divergence guard's `Aborted`) — whichever status lands first wins.
2447        let backend = mem_backend(1_048_576).await;
2448        let exec = ExecutionId::new();
2449        backend
2450            .open_execution(exec, ExecutionKind::AgentTurn)
2451            .await
2452            .unwrap();
2453        backend
2454            .finalize(exec, ExecutionStatus::Completed)
2455            .await
2456            .unwrap();
2457
2458        // A later call with a different terminal status must not overwrite the first.
2459        backend
2460            .finalize(exec, ExecutionStatus::Failed)
2461            .await
2462            .unwrap();
2463
2464        let (status,): (String,) = zeph_db::query_as(sql!(
2465            "SELECT status FROM durable_executions WHERE execution_id = ?"
2466        ))
2467        .bind(exec.as_uuid().to_string())
2468        .fetch_one(backend.pool())
2469        .await
2470        .unwrap();
2471        assert_eq!(
2472            status, "completed",
2473            "the first terminal status must stick; a later finalize call is a no-op"
2474        );
2475    }
2476
2477    #[tokio::test]
2478    async fn finalize_after_abort_is_a_noop() {
2479        // #6251: the reverse direction of the divergence race — the internal `Aborted` transition
2480        // (replay-divergence guard) commits first, so a consumer's later own `Completed`/`Failed`
2481        // call must be a no-op rather than resurrecting the row out of its aborted state.
2482        let backend = mem_backend(1_048_576).await;
2483        let exec = ExecutionId::new();
2484        backend
2485            .open_execution(exec, ExecutionKind::AgentTurn)
2486            .await
2487            .unwrap();
2488        backend
2489            .finalize(exec, ExecutionStatus::Aborted)
2490            .await
2491            .unwrap();
2492
2493        backend
2494            .finalize(exec, ExecutionStatus::Completed)
2495            .await
2496            .unwrap();
2497
2498        let (status,): (String,) = zeph_db::query_as(sql!(
2499            "SELECT status FROM durable_executions WHERE execution_id = ?"
2500        ))
2501        .bind(exec.as_uuid().to_string())
2502        .fetch_one(backend.pool())
2503        .await
2504        .unwrap();
2505        assert_eq!(
2506            status, "aborted",
2507            "an aborted execution must not be overwritten by a later Completed/Failed call"
2508        );
2509    }
2510
2511    #[tokio::test]
2512    async fn reopening_a_finalized_execution_resets_it_to_running() {
2513        // #6251: a finalized execution that is legitimately reopened (e.g. a resumed conversation)
2514        // must not keep a stale `finalized_at` — otherwise the retention sweep could prune a row
2515        // that is still receiving new journal writes.
2516        let backend = mem_backend(1_048_576).await;
2517        let exec = ExecutionId::new();
2518        backend
2519            .open_execution(exec, ExecutionKind::AgentTurn)
2520            .await
2521            .unwrap();
2522        backend
2523            .finalize(exec, ExecutionStatus::Completed)
2524            .await
2525            .unwrap();
2526
2527        let is_resume = backend
2528            .open_execution(exec, ExecutionKind::AgentTurn)
2529            .await
2530            .unwrap();
2531        assert!(is_resume, "the row already existed, so this is a resume");
2532
2533        let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
2534            "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
2535        ))
2536        .bind(exec.as_uuid().to_string())
2537        .fetch_one(backend.pool())
2538        .await
2539        .unwrap();
2540        assert_eq!(
2541            status, "running",
2542            "reopening a completed execution must un-finalize it"
2543        );
2544        assert!(
2545            finalized.is_none(),
2546            "reopening must clear the stale finalized_at"
2547        );
2548    }
2549
2550    #[tokio::test]
2551    async fn reopening_a_failed_execution_resets_it_to_running() {
2552        // #6251: same guarantee as `reopening_a_finalized_execution_resets_it_to_running`, but for
2553        // the `Failed` terminal status — e.g. a scheduler retry of the same (job_name, slot_ms)
2554        // after the previous fire failed must not orphan a `Failed` row.
2555        let backend = mem_backend(1_048_576).await;
2556        let exec = ExecutionId::new();
2557        backend
2558            .open_execution(exec, ExecutionKind::AgentTurn)
2559            .await
2560            .unwrap();
2561        backend
2562            .finalize(exec, ExecutionStatus::Failed)
2563            .await
2564            .unwrap();
2565
2566        let is_resume = backend
2567            .open_execution(exec, ExecutionKind::AgentTurn)
2568            .await
2569            .unwrap();
2570        assert!(is_resume, "the row already existed, so this is a resume");
2571
2572        let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
2573            "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
2574        ))
2575        .bind(exec.as_uuid().to_string())
2576        .fetch_one(backend.pool())
2577        .await
2578        .unwrap();
2579        assert_eq!(
2580            status, "running",
2581            "reopening a failed execution must un-finalize it"
2582        );
2583        assert!(
2584            finalized.is_none(),
2585            "reopening must clear the stale finalized_at"
2586        );
2587    }
2588
2589    #[tokio::test]
2590    async fn reopening_an_aborted_execution_un_finalizes_it() {
2591        // INV-16 (#6254): reopening a row in ANY terminal status — including `aborted` — must
2592        // un-finalize it back to `running` with `finalized_at` cleared. This covers both the
2593        // pre-existing divergence-recovery reopen (which starts a fresh replay cursor on
2594        // purpose) and the new crash-orphan sweep (INV-17), which makes `aborted` the common
2595        // outcome of a resumable crash: a resumed execution whose row keeps `finalized_at` set
2596        // would otherwise be prunable out from under the active resume.
2597        let backend = mem_backend(1_048_576).await;
2598        let exec = ExecutionId::new();
2599        backend
2600            .open_execution(exec, ExecutionKind::AgentTurn)
2601            .await
2602            .unwrap();
2603        backend
2604            .finalize(exec, ExecutionStatus::Aborted)
2605            .await
2606            .unwrap();
2607
2608        let is_resume = backend
2609            .open_execution(exec, ExecutionKind::AgentTurn)
2610            .await
2611            .unwrap();
2612        assert!(is_resume, "the row already existed, so this is a resume");
2613
2614        let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
2615            "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
2616        ))
2617        .bind(exec.as_uuid().to_string())
2618        .fetch_one(backend.pool())
2619        .await
2620        .unwrap();
2621        assert_eq!(
2622            status, "running",
2623            "reopening an aborted execution must un-finalize it (INV-16)"
2624        );
2625        assert!(
2626            finalized.is_none(),
2627            "reopening must clear the stale finalized_at"
2628        );
2629    }
2630
2631    #[tokio::test]
2632    async fn reopen_of_a_row_deleted_out_from_under_it_starts_fresh() {
2633        // #6251 critic S1: simulates the tail of the prune-vs-reopen race — a concurrent prune
2634        // sweep deletes the row entirely before the reopen's guarded UPDATE runs. The guarded
2635        // UPDATE must match zero rows (not resurrect a half-deleted row), and the existence
2636        // fallback must see the row is genuinely gone and start a fresh execution rather than
2637        // falsely reporting `is_resume = true` for a row that no longer exists.
2638        let backend = mem_backend(1_048_576).await;
2639        let exec = ExecutionId::new();
2640        backend
2641            .open_execution(exec, ExecutionKind::AgentTurn)
2642            .await
2643            .unwrap();
2644        backend
2645            .finalize(exec, ExecutionStatus::Completed)
2646            .await
2647            .unwrap();
2648
2649        // Simulate the prune sweep's delete completing before the reopen runs.
2650        zeph_db::query(sql!(
2651            "DELETE FROM durable_executions WHERE execution_id = ?"
2652        ))
2653        .bind(exec.as_uuid().to_string())
2654        .execute(backend.pool())
2655        .await
2656        .unwrap();
2657
2658        let is_resume = backend
2659            .open_execution(exec, ExecutionKind::AgentTurn)
2660            .await
2661            .unwrap();
2662        assert!(
2663            !is_resume,
2664            "a row deleted by a concurrent prune must be reported as a fresh execution, not a resume"
2665        );
2666
2667        let (status,): (String,) = zeph_db::query_as(sql!(
2668            "SELECT status FROM durable_executions WHERE execution_id = ?"
2669        ))
2670        .bind(exec.as_uuid().to_string())
2671        .fetch_one(backend.pool())
2672        .await
2673        .unwrap();
2674        assert_eq!(status, "running", "the fresh row starts running");
2675    }
2676
2677    #[tokio::test]
2678    async fn prune_does_not_delete_a_row_reopened_since_it_was_finalized() {
2679        // #6251 critic S1: a row finalized, then legitimately reopened (un-finalized back to
2680        // running) before prune runs, must not be deleted even though prune's cutoff would have
2681        // matched its now-stale-if-it-were-still-finalized state.
2682        let backend = mem_backend(1_048_576).await;
2683        let exec = ExecutionId::new();
2684        backend
2685            .open_execution(exec, ExecutionKind::AgentTurn)
2686            .await
2687            .unwrap();
2688        backend.append(step_result(exec, 0, b"x")).await.unwrap();
2689        zeph_db::query(sql!(
2690            "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
2691        ))
2692        .bind(exec.as_uuid().to_string())
2693        .execute(backend.pool())
2694        .await
2695        .unwrap();
2696
2697        // A legitimate resume reopens and un-finalizes it before the prune sweep runs.
2698        let is_resume = backend
2699            .open_execution(exec, ExecutionKind::AgentTurn)
2700            .await
2701            .unwrap();
2702        assert!(is_resume);
2703
2704        let policy = RetentionPolicy {
2705            ttl_completed_secs: 1,
2706            prune_batch_size: 10,
2707            ..RetentionPolicy::default()
2708        };
2709        let deleted = backend.prune(&policy).await.unwrap();
2710        assert_eq!(
2711            deleted, 0,
2712            "a reopened (un-finalized) execution must not be pruned"
2713        );
2714        assert_eq!(
2715            backend.read_execution(exec).await.unwrap().len(),
2716            1,
2717            "the execution's journal must survive"
2718        );
2719    }
2720
2721    #[tokio::test]
2722    async fn concurrent_prune_and_reopen_never_lose_or_corrupt_the_row() {
2723        // #6251 critic S1: the deterministic tests above exercise each ordering of the prune-vs-
2724        // reopen race one step at a time; this test drives the two operations as genuinely
2725        // concurrent tasks against a real multi-connection pool (file-backed — `:memory:` forces
2726        // a single connection, per `zeph-db/src/pool.rs`'s `connect_sqlite`, which would serialize
2727        // the two calls trivially and prove nothing about the locking fix). Runs many trials with
2728        // fresh executions so the two tasks' actual scheduling order varies across iterations,
2729        // covering both "prune's tx starts first" and "reopen's UPDATE starts first" without
2730        // needing artificial delay injection into the DB layer.
2731        //
2732        // Invariant checked every trial, regardless of which task wins: neither operation errors,
2733        // and the row is never lost — it either stays `running` (reopen won, or ran after prune's
2734        // read already excluded it) or is deleted and then reinserted fresh by reopen's
2735        // does-not-exist fallback (prune won). It must never end up half-deleted (FK violation on
2736        // a later journal append) or stuck `completed` with a live journal.
2737        let dir = tempfile::tempdir().unwrap();
2738        let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
2739        let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
2740        backend.init().await.unwrap();
2741
2742        let policy = RetentionPolicy {
2743            ttl_completed_secs: 1,
2744            prune_batch_size: 10,
2745            ..RetentionPolicy::default()
2746        };
2747
2748        for _ in 0..20 {
2749            let exec = ExecutionId::new();
2750            backend
2751                .open_execution(exec, ExecutionKind::AgentTurn)
2752                .await
2753                .unwrap();
2754            backend.append(step_result(exec, 0, b"x")).await.unwrap();
2755            // Backdate finalized_at so this row is immediately prune-eligible.
2756            zeph_db::query(sql!(
2757                "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
2758            ))
2759            .bind(exec.as_uuid().to_string())
2760            .execute(backend.pool())
2761            .await
2762            .unwrap();
2763
2764            let reopen_backend = backend.clone();
2765            let reopen = tokio::spawn(async move {
2766                reopen_backend
2767                    .open_execution(exec, ExecutionKind::AgentTurn)
2768                    .await
2769            });
2770            let prune_backend = backend.clone();
2771            let policy_for_task = policy.clone();
2772            let prune = tokio::spawn(async move { prune_backend.prune(&policy_for_task).await });
2773
2774            let (reopen_result, prune_result) = tokio::join!(reopen, prune);
2775            reopen_result
2776                .expect("reopen task must not panic")
2777                .expect("reopen must not error under concurrent prune");
2778            prune_result
2779                .expect("prune task must not panic")
2780                .expect("prune must not error under a concurrent reopen");
2781
2782            let (status,): (String,) = zeph_db::query_as(sql!(
2783                "SELECT status FROM durable_executions WHERE execution_id = ?"
2784            ))
2785            .bind(exec.as_uuid().to_string())
2786            .fetch_one(backend.pool())
2787            .await
2788            .expect(
2789                "the row must exist under either race outcome — reopened-running, or \
2790                 deleted-then-reinserted-fresh-running by reopen's fallback",
2791            );
2792            assert_eq!(
2793                status, "running",
2794                "whichever task wins, the row must end up running — never left completed \
2795                 (orphaned from a live journal) or absent"
2796            );
2797        }
2798    }
2799
2800    #[tokio::test]
2801    async fn max_seq_reflects_committed_appends() {
2802        let backend = mem_backend(1_048_576).await;
2803        assert_eq!(
2804            backend.max_seq().await.unwrap(),
2805            None,
2806            "empty journal has no max seq"
2807        );
2808
2809        let exec = ExecutionId::new();
2810        backend
2811            .open_execution(exec, ExecutionKind::AgentTurn)
2812            .await
2813            .unwrap();
2814        for step in 0..3 {
2815            backend.append(step_result(exec, step, b"x")).await.unwrap();
2816        }
2817        assert_eq!(backend.max_seq().await.unwrap(), Some(JournalSeq::new(3)));
2818    }
2819
2820    #[tokio::test]
2821    async fn append_batch_group_commits_every_entry() {
2822        let backend = mem_backend(1_048_576).await;
2823        let exec = ExecutionId::new();
2824        backend
2825            .open_execution(exec, ExecutionKind::AgentTurn)
2826            .await
2827            .unwrap();
2828        let batch = vec![
2829            step_result(exec, 0, b"a"),
2830            step_result(exec, 1, b"b"),
2831            step_result(exec, 2, b"c"),
2832        ];
2833        backend.append_batch(&batch).await.unwrap();
2834        assert_eq!(backend.read_execution(exec).await.unwrap().len(), 3);
2835    }
2836
2837    #[tokio::test]
2838    async fn read_execution_range_bounds_the_segment() {
2839        let backend = mem_backend(1_048_576).await;
2840        let exec = ExecutionId::new();
2841        backend
2842            .open_execution(exec, ExecutionKind::AgentTurn)
2843            .await
2844            .unwrap();
2845        for step in 0..5 {
2846            backend.append(step_result(exec, step, b"x")).await.unwrap();
2847        }
2848        let segment = backend.read_execution_range(exec, 2, 2).await.unwrap();
2849        assert_eq!(segment.len(), 2);
2850        assert_eq!(segment[0].step_id, StepId::new(2));
2851        assert_eq!(segment[1].step_id, StepId::new(3));
2852    }
2853
2854    #[tokio::test]
2855    async fn lookup_committed_result_finds_by_idem_key() {
2856        let backend = mem_backend(1_048_576).await;
2857        let exec = ExecutionId::new();
2858        backend
2859            .open_execution(exec, ExecutionKind::AgentTurn)
2860            .await
2861            .unwrap();
2862        let entry = step_result(exec, 0, b"committed");
2863        let idem_key = match &entry.entry {
2864            EntryKind::StepResult {
2865                idempotency_key, ..
2866            } => *idempotency_key,
2867            other => panic!("unexpected entry kind: {other:?}"),
2868        };
2869        backend.append(entry).await.unwrap();
2870
2871        let found = backend
2872            .lookup_committed_result(exec, idem_key)
2873            .await
2874            .unwrap()
2875            .expect("committed result is located by its idempotency key");
2876        match &found.entry {
2877            EntryKind::StepResult { payload, .. } => assert_eq!(payload.as_ref(), b"committed"),
2878            other => panic!("unexpected entry kind: {other:?}"),
2879        }
2880
2881        // A key that was never committed yields nothing rather than erroring.
2882        let absent = IdempotencyKey::derive(exec, StepId::new(99), b"never");
2883        assert!(
2884            backend
2885                .lookup_committed_result(exec, absent)
2886                .await
2887                .unwrap()
2888                .is_none()
2889        );
2890    }
2891
2892    #[tokio::test]
2893    async fn capabilities_describe_the_local_profile() {
2894        let backend = mem_backend(4096).await;
2895        let caps = backend.capabilities();
2896        assert!(caps.parallel_steps);
2897        assert!(
2898            !caps.cross_process,
2899            "the SQLite local backend is in-process"
2900        );
2901        assert_eq!(caps.max_payload, 4096);
2902    }
2903
2904    #[tokio::test]
2905    async fn promise_insert_state_and_resolve_round_trip() {
2906        let backend = mem_backend(1_048_576)
2907            .await
2908            .with_cipher(Arc::new(XorCipher));
2909        let exec = ExecutionId::new();
2910        backend
2911            .open_execution(exec, ExecutionKind::AgentTurn)
2912            .await
2913            .unwrap();
2914        let promise = PromiseId::derive(exec, StepId::new(0));
2915        backend
2916            .insert_promise(promise, exec, [9u8; 32], 100)
2917            .await
2918            .unwrap();
2919
2920        let pending = backend.promise_state(promise).await.unwrap().unwrap();
2921        assert!(!pending.resolved);
2922        assert_eq!(pending.execution_id, exec);
2923        assert_eq!(pending.resolver_token_hash, [9u8; 32]);
2924
2925        // Resolve seals the value at rest; a second resolve is a no-op.
2926        assert!(
2927            backend
2928                .resolve_promise(promise, exec, b"answer", 200)
2929                .await
2930                .unwrap()
2931        );
2932        assert!(
2933            !backend
2934                .resolve_promise(promise, exec, b"again", 300)
2935                .await
2936                .unwrap()
2937        );
2938
2939        let resolved = backend.promise_state(promise).await.unwrap().unwrap();
2940        assert!(resolved.resolved);
2941        let sealed = resolved.payload.expect("resolved payload present");
2942        assert_ne!(sealed.as_slice(), b"answer", "payload is sealed at rest");
2943        let opened = backend
2944            .open_promise_payload(promise, exec, &sealed)
2945            .unwrap();
2946        assert_eq!(opened.as_ref(), b"answer");
2947    }
2948
2949    #[tokio::test]
2950    async fn claim_promise_notification_is_single_winner() {
2951        let backend = mem_backend(1_048_576).await;
2952        let exec = ExecutionId::new();
2953        backend
2954            .open_execution(exec, ExecutionKind::AgentTurn)
2955            .await
2956            .unwrap();
2957        let promise = PromiseId::derive(exec, StepId::new(0));
2958        backend
2959            .insert_promise(promise, exec, [9u8; 32], 100)
2960            .await
2961            .unwrap();
2962
2963        // First claim wins (transitions notified_at from NULL).
2964        assert!(
2965            backend
2966                .claim_promise_notification(promise, 200)
2967                .await
2968                .unwrap()
2969        );
2970        // Every later claim on the same promise is a no-op.
2971        assert!(
2972            !backend
2973                .claim_promise_notification(promise, 300)
2974                .await
2975                .unwrap()
2976        );
2977    }
2978
2979    #[tokio::test]
2980    async fn timer_arm_due_and_fire() {
2981        let backend = mem_backend(1_048_576).await;
2982        let exec = ExecutionId::new();
2983        backend
2984            .open_execution(exec, ExecutionKind::AgentTurn)
2985            .await
2986            .unwrap();
2987        let past = TimerId::derive(exec, StepId::new(0));
2988        let future = TimerId::derive(exec, StepId::new(1));
2989        backend.arm_timer(past, exec, 1_000, 0).await.unwrap();
2990        backend
2991            .arm_timer(future, exec, 9_000_000_000_000, 0)
2992            .await
2993            .unwrap();
2994
2995        // Only the past-due timer is returned at now = 5000.
2996        let due = backend.due_timers(5_000).await.unwrap();
2997        assert_eq!(due, vec![past]);
2998
2999        assert!(backend.mark_timer_fired(past).await.unwrap());
3000        assert!(
3001            !backend.mark_timer_fired(past).await.unwrap(),
3002            "second fire is a no-op"
3003        );
3004        assert_eq!(
3005            backend.timer_state(past).await.unwrap(),
3006            Some((1_000, true))
3007        );
3008        // The fired timer no longer appears as due.
3009        assert!(backend.due_timers(5_000).await.unwrap().is_empty());
3010    }
3011
3012    #[tokio::test]
3013    async fn prune_deletes_terminal_executions_past_ttl() {
3014        let backend = mem_backend(1_048_576).await;
3015        // An old completed execution (finalized long ago) and a fresh running one.
3016        let old = ExecutionId::new();
3017        backend
3018            .open_execution(old, ExecutionKind::AgentTurn)
3019            .await
3020            .unwrap();
3021        backend.append(step_result(old, 0, b"x")).await.unwrap();
3022        // Backdate its finalized_at far into the past.
3023        zeph_db::query(sql!(
3024            "UPDATE durable_executions SET status = 'completed', finalized_at = 1000 WHERE execution_id = ?"
3025        ))
3026        .bind(old.as_uuid().to_string())
3027        .execute(backend.pool())
3028        .await
3029        .unwrap();
3030
3031        let live = ExecutionId::new();
3032        backend
3033            .open_execution(live, ExecutionKind::AgentTurn)
3034            .await
3035            .unwrap();
3036        backend.append(step_result(live, 0, b"y")).await.unwrap();
3037
3038        let policy = RetentionPolicy {
3039            ttl_completed_secs: 1,
3040            prune_batch_size: 10,
3041            ..RetentionPolicy::default()
3042        };
3043        let deleted = backend.prune(&policy).await.unwrap();
3044        assert_eq!(deleted, 1, "only the aged terminal execution is pruned");
3045
3046        // The old execution and its journal are gone; the live one survives.
3047        assert!(backend.read_execution(old).await.unwrap().is_empty());
3048        assert!(
3049            backend
3050                .promise_state(PromiseId::derive(old, StepId::new(0)))
3051                .await
3052                .unwrap()
3053                .is_none()
3054        );
3055        assert_eq!(backend.read_execution(live).await.unwrap().len(), 1);
3056    }
3057
3058    /// Backdate a `durable_executions` row's `updated_at` so it becomes a sweep candidate.
3059    async fn backdate_updated_at(backend: &LocalBackend, id: ExecutionId, updated_at_ms: i64) {
3060        zeph_db::query(sql!(
3061            "UPDATE durable_executions SET updated_at = ? WHERE execution_id = ?"
3062        ))
3063        .bind(updated_at_ms)
3064        .bind(id.as_uuid().to_string())
3065        .execute(backend.pool())
3066        .await
3067        .unwrap();
3068    }
3069
3070    #[tokio::test]
3071    async fn sweep_orphans_disabled_when_threshold_is_zero() {
3072        // A file-backed backend so the sweep would otherwise have a lock_dir to work with;
3073        // stale_running_after_secs = 0 must short-circuit before any scan.
3074        let dir = tempfile::tempdir().unwrap();
3075        let backend =
3076            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
3077                .await
3078                .unwrap();
3079        backend.init().await.unwrap();
3080
3081        let exec = ExecutionId::new();
3082        backend
3083            .open_execution(exec, ExecutionKind::AgentTurn)
3084            .await
3085            .unwrap();
3086        backdate_updated_at(&backend, exec, 0).await;
3087
3088        let policy = RetentionPolicy {
3089            stale_running_after_secs: 0,
3090            ..RetentionPolicy::default()
3091        };
3092        let aborted = backend.sweep_orphans(&policy).await.unwrap();
3093        assert_eq!(
3094            aborted, 0,
3095            "stale_running_after_secs = 0 disables the sweep"
3096        );
3097
3098        let (status,): (String,) = zeph_db::query_as(sql!(
3099            "SELECT status FROM durable_executions WHERE execution_id = ?"
3100        ))
3101        .bind(exec.as_uuid().to_string())
3102        .fetch_one(backend.pool())
3103        .await
3104        .unwrap();
3105        assert_eq!(status, "running");
3106    }
3107
3108    #[tokio::test]
3109    async fn sweep_orphans_is_a_documented_no_op_on_memory_backend() {
3110        // `:memory:` has no on-disk lock_dir (INV-15 degrade), so the sweep must never abort on
3111        // staleness alone — FR-DE-19.
3112        let backend = mem_backend(1_048_576).await;
3113        let exec = ExecutionId::new();
3114        backend
3115            .open_execution(exec, ExecutionKind::AgentTurn)
3116            .await
3117            .unwrap();
3118        backdate_updated_at(&backend, exec, 0).await;
3119
3120        let policy = RetentionPolicy {
3121            stale_running_after_secs: 1,
3122            ..RetentionPolicy::default()
3123        };
3124        let aborted = backend.sweep_orphans(&policy).await.unwrap();
3125        assert_eq!(
3126            aborted, 0,
3127            "a lock_dir=None backend must never abort on staleness alone"
3128        );
3129
3130        let (status,): (String,) = zeph_db::query_as(sql!(
3131            "SELECT status FROM durable_executions WHERE execution_id = ?"
3132        ))
3133        .bind(exec.as_uuid().to_string())
3134        .fetch_one(backend.pool())
3135        .await
3136        .unwrap();
3137        assert_eq!(status, "running");
3138    }
3139
3140    #[tokio::test]
3141    async fn sweep_orphans_aborts_a_stale_running_execution_with_no_live_owner() {
3142        // FR-DE-16/17: a stale `running` row whose lock is free (no live owner) is hard-aborted.
3143        let dir = tempfile::tempdir().unwrap();
3144        let backend =
3145            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
3146                .await
3147                .unwrap();
3148        backend.init().await.unwrap();
3149
3150        let exec = ExecutionId::new();
3151        backend
3152            .open_execution(exec, ExecutionKind::AgentTurn)
3153            .await
3154            .unwrap();
3155        // Nothing holds this execution's ExecutionLock — `open_execution` (not `_exclusive`)
3156        // never acquires one, simulating a crashed owner whose flock released on process exit.
3157        backdate_updated_at(&backend, exec, 0).await;
3158
3159        let policy = RetentionPolicy {
3160            stale_running_after_secs: 1,
3161            ..RetentionPolicy::default()
3162        };
3163        let aborted = backend.sweep_orphans(&policy).await.unwrap();
3164        assert_eq!(aborted, 1);
3165
3166        let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3167            "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3168        ))
3169        .bind(exec.as_uuid().to_string())
3170        .fetch_one(backend.pool())
3171        .await
3172        .unwrap();
3173        assert_eq!(status, "aborted");
3174        assert!(finalized.is_some());
3175    }
3176
3177    #[tokio::test]
3178    async fn sweep_orphans_skips_an_execution_whose_lock_is_held_by_a_live_owner() {
3179        // INV-17: staleness of `updated_at` alone is never sufficient — a stale-but-alive
3180        // execution (long single step, parked HITL promise, multi-hour job) must survive the
3181        // sweep as long as its owner still holds the INV-15 flock.
3182        let dir = tempfile::tempdir().unwrap();
3183        let db_path = dir.path().join("durable.db");
3184        let url = db_path.to_string_lossy().into_owned();
3185
3186        let owner = LocalBackend::open(&url, 1_048_576).await.unwrap();
3187        owner.init().await.unwrap();
3188        let sweeper = LocalBackend::open(&url, 1_048_576).await.unwrap();
3189
3190        let exec = ExecutionId::new();
3191        let (_, _lock) = owner
3192            .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3193            .await
3194            .unwrap();
3195        backdate_updated_at(&owner, exec, 0).await;
3196
3197        let policy = RetentionPolicy {
3198            stale_running_after_secs: 1,
3199            ..RetentionPolicy::default()
3200        };
3201        let aborted = sweeper.sweep_orphans(&policy).await.unwrap();
3202        assert_eq!(aborted, 0, "a live-held lock must never be swept");
3203
3204        let (status,): (String,) = zeph_db::query_as(sql!(
3205            "SELECT status FROM durable_executions WHERE execution_id = ?"
3206        ))
3207        .bind(exec.as_uuid().to_string())
3208        .fetch_one(owner.pool())
3209        .await
3210        .unwrap();
3211        assert_eq!(status, "running");
3212    }
3213
3214    #[tokio::test]
3215    async fn sweep_orphans_leaves_a_fresh_running_execution_untouched() {
3216        // A recently-updated `running` row is not yet a sweep candidate at all.
3217        let dir = tempfile::tempdir().unwrap();
3218        let backend =
3219            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
3220                .await
3221                .unwrap();
3222        backend.init().await.unwrap();
3223
3224        let exec = ExecutionId::new();
3225        backend
3226            .open_execution(exec, ExecutionKind::AgentTurn)
3227            .await
3228            .unwrap();
3229
3230        let policy = RetentionPolicy {
3231            stale_running_after_secs: 3600,
3232            ..RetentionPolicy::default()
3233        };
3234        let aborted = backend.sweep_orphans(&policy).await.unwrap();
3235        assert_eq!(aborted, 0);
3236    }
3237
3238    #[tokio::test]
3239    async fn count_orphans_matches_sweep_without_mutating() {
3240        let dir = tempfile::tempdir().unwrap();
3241        let backend =
3242            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
3243                .await
3244                .unwrap();
3245        backend.init().await.unwrap();
3246
3247        let exec = ExecutionId::new();
3248        backend
3249            .open_execution(exec, ExecutionKind::AgentTurn)
3250            .await
3251            .unwrap();
3252        backdate_updated_at(&backend, exec, 0).await;
3253
3254        let policy = RetentionPolicy {
3255            stale_running_after_secs: 1,
3256            ..RetentionPolicy::default()
3257        };
3258        let counted = backend.count_orphans(&policy).await.unwrap();
3259        assert_eq!(counted, 1);
3260
3261        // count_orphans must not have mutated the row.
3262        let (status,): (String,) = zeph_db::query_as(sql!(
3263            "SELECT status FROM durable_executions WHERE execution_id = ?"
3264        ))
3265        .bind(exec.as_uuid().to_string())
3266        .fetch_one(backend.pool())
3267        .await
3268        .unwrap();
3269        assert_eq!(status, "running");
3270
3271        let aborted = backend.sweep_orphans(&policy).await.unwrap();
3272        assert_eq!(
3273            aborted, counted,
3274            "sweep must abort exactly what count_orphans counted"
3275        );
3276    }
3277
3278    /// Batching-boundary regression: a candidate set straddling `prune_batch_size` (one more row
3279    /// than a single batch) must be fully processed across multiple batches, not just the first
3280    /// one. Exercises the real `sweep_orphan_batch`/`sweep_orphans_in_batches` composition end to
3281    /// end (not the pure-logic unit test in `retention.rs`), so the SQL `LIMIT` and the
3282    /// `scanned`-driven continuation check are both proven against a real DB.
3283    #[tokio::test]
3284    async fn sweep_orphans_processes_every_batch_when_candidates_straddle_the_batch_size() {
3285        let dir = tempfile::tempdir().unwrap();
3286        let backend =
3287            LocalBackend::open(&dir.path().join("durable.db").to_string_lossy(), 1_048_576)
3288                .await
3289                .unwrap();
3290        backend.init().await.unwrap();
3291
3292        let batch_size = 2u64;
3293        let candidate_count = batch_size + 1; // straddles the batch boundary
3294        let mut execs = Vec::new();
3295        for _ in 0..candidate_count {
3296            let exec = ExecutionId::new();
3297            backend
3298                .open_execution(exec, ExecutionKind::AgentTurn)
3299                .await
3300                .unwrap();
3301            backdate_updated_at(&backend, exec, 0).await;
3302            execs.push(exec);
3303        }
3304
3305        let policy = RetentionPolicy {
3306            stale_running_after_secs: 1,
3307            prune_batch_size: batch_size,
3308            ..RetentionPolicy::default()
3309        };
3310        let aborted = backend.sweep_orphans(&policy).await.unwrap();
3311        assert_eq!(
3312            aborted, candidate_count,
3313            "every candidate must be aborted, including the one past the first batch"
3314        );
3315
3316        for exec in execs {
3317            let (status,): (String,) = zeph_db::query_as(sql!(
3318                "SELECT status FROM durable_executions WHERE execution_id = ?"
3319            ))
3320            .bind(exec.as_uuid().to_string())
3321            .fetch_one(backend.pool())
3322            .await
3323            .unwrap();
3324            assert_eq!(status, "aborted");
3325        }
3326    }
3327
3328    /// #6254 C1 regression: when the count of stale-but-live (lock-held) candidates is `>=
3329    /// prune_batch_size`, the sweep must still terminate rather than looping forever re-selecting
3330    /// the same lock-held rows. Before the keyset-pagination fix, `sweep_orphan_batch`'s candidate
3331    /// `SELECT` had no offset/cursor, so a batch consisting entirely of lock-held rows (which the
3332    /// sweep never deletes, mutates, or otherwise removes from the `status='running'` candidate
3333    /// set) would re-select the identical rows on every iteration: `scanned` would stay `==
3334    /// batch` and `aborted` would stay `0` forever, so `sweep_orphans_in_batches`'s `scanned <
3335    /// batch` continuation check would never trip. Exercises the real DB-backed
3336    /// `sweep_orphan_batch`/`sweep_orphans_in_batches` composition (not the pure-logic
3337    /// simulation in `retention.rs`) with more lock-held candidates than `prune_batch_size`, so a
3338    /// naive single-batch-worth-of-locks reproduction would not have caught a bug that only
3339    /// manifests once the candidate set spans multiple batches.
3340    #[tokio::test]
3341    async fn sweep_orphans_terminates_when_lock_held_candidates_exceed_batch_size() {
3342        let dir = tempfile::tempdir().unwrap();
3343        let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
3344
3345        let owner = LocalBackend::open(&db_url, 1_048_576).await.unwrap();
3346        owner.init().await.unwrap();
3347        let sweeper = LocalBackend::open(&db_url, 1_048_576).await.unwrap();
3348
3349        let batch_size = 2u64;
3350        let candidate_count = batch_size * 2 + 1; // spans at least three batches, all lock-held
3351        let mut locks = Vec::new();
3352        for _ in 0..candidate_count {
3353            let exec = ExecutionId::new();
3354            let (_, lock) = owner
3355                .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3356                .await
3357                .unwrap();
3358            backdate_updated_at(&owner, exec, 0).await;
3359            locks.push(lock); // held for the whole test — every candidate stays lock-held
3360        }
3361
3362        let policy = RetentionPolicy {
3363            stale_running_after_secs: 1,
3364            prune_batch_size: batch_size,
3365            ..RetentionPolicy::default()
3366        };
3367
3368        let aborted = tokio::time::timeout(
3369            std::time::Duration::from_secs(10),
3370            sweeper.sweep_orphans(&policy),
3371        )
3372        .await
3373        .expect(
3374            "sweep_orphans must terminate even when lock-held candidates exceed prune_batch_size \
3375             (#6254 C1) — it hung instead of returning",
3376        )
3377        .unwrap();
3378
3379        assert_eq!(aborted, 0, "every candidate's lock is held by a live owner");
3380        drop(locks);
3381    }
3382
3383    /// INV-17: the sweep's guarded abort `UPDATE` runs only while holding the same non-reentrant
3384    /// flock a concurrent `open_execution_exclusive` reopen for the same execution id requires, so
3385    /// the two can never both mutate the row at once. Drives them as genuinely concurrent tasks
3386    /// against a real multi-connection pool (file-backed — `:memory:` forces a single connection,
3387    /// which would serialize the two calls trivially and prove nothing) across many trials so both
3388    /// orderings ("sweep acquires the lock first" and "reopen acquires the lock first") are
3389    /// exercised without artificial delay injection, mirroring the #6251
3390    /// `concurrent_prune_and_reopen_never_lose_or_corrupt_the_row` pattern above.
3391    #[tokio::test]
3392    async fn concurrent_sweep_and_reopen_race_never_corrupts_the_row() {
3393        let dir = tempfile::tempdir().unwrap();
3394        let db_url = dir.path().join("durable.db").to_string_lossy().into_owned();
3395        let backend = Arc::new(LocalBackend::open(&db_url, 1_048_576).await.unwrap());
3396        backend.init().await.unwrap();
3397
3398        let policy = RetentionPolicy {
3399            stale_running_after_secs: 1,
3400            prune_batch_size: 10,
3401            ..RetentionPolicy::default()
3402        };
3403
3404        for _ in 0..20 {
3405            let exec = ExecutionId::new();
3406            backend
3407                .open_execution(exec, ExecutionKind::AgentTurn)
3408                .await
3409                .unwrap();
3410            backdate_updated_at(&backend, exec, 0).await;
3411
3412            let sweep_backend = backend.clone();
3413            let policy_for_task = policy.clone();
3414            let sweep =
3415                tokio::spawn(async move { sweep_backend.sweep_orphans(&policy_for_task).await });
3416
3417            let reopen_backend = backend.clone();
3418            let reopen = tokio::spawn(async move {
3419                reopen_backend
3420                    .open_execution_exclusive(exec, ExecutionKind::AgentTurn)
3421                    .await
3422            });
3423
3424            let (sweep_result, reopen_result) = tokio::join!(sweep, reopen);
3425            let aborted = sweep_result
3426                .expect("sweep task must not panic")
3427                .expect("sweep must not error under a concurrent reopen");
3428            assert!(aborted <= 1, "at most one candidate row exists per trial");
3429
3430            match reopen_result.expect("reopen task must not panic") {
3431                Ok((_is_resume, _lock)) => {
3432                    // reopen won the race for the lock (either before the sweep even tried, or
3433                    // after the sweep aborted the row and released) — the row must be `running`
3434                    // with `finalized_at` cleared either way (INV-16 un-finalizes `aborted` too).
3435                    let (status, finalized): (String, Option<i64>) = zeph_db::query_as(sql!(
3436                        "SELECT status, finalized_at FROM durable_executions WHERE execution_id = ?"
3437                    ))
3438                    .bind(exec.as_uuid().to_string())
3439                    .fetch_one(backend.pool())
3440                    .await
3441                    .unwrap();
3442                    assert_eq!(status, "running");
3443                    assert!(finalized.is_none());
3444                    // Finalize before the next trial: when reopen wins because the row was
3445                    // already `running` (not terminal) at the time it checked, `open_execution`'s
3446                    // existing-row branch never bumps `updated_at` — left alone, this row would
3447                    // stay a stale `running` candidate forever and pollute a later trial's
3448                    // `aborted` count (the `assert!(aborted <= 1, ...)` above would then see more
3449                    // than this trial's own row). Each trial must start with a clean slate of
3450                    // exactly its own candidate.
3451                    backend
3452                        .finalize(exec, ExecutionStatus::Completed)
3453                        .await
3454                        .unwrap();
3455                }
3456                Err(DurableError::ExecutionLocked { .. }) => {
3457                    // The sweep held the lock at the moment reopen tried — expected under the race.
3458                }
3459                Err(e) => panic!(
3460                    "reopen must only ever fail with ExecutionLocked under this race, got {e:?}"
3461                ),
3462            }
3463        }
3464    }
3465
3466    #[tokio::test]
3467    async fn checkpoint_fold_compacts_idempotent_prefix_and_replays() {
3468        let backend = mem_backend(1_048_576)
3469            .await
3470            .with_cipher(Arc::new(XorCipher));
3471        let exec = ExecutionId::new();
3472        backend
3473            .open_execution(exec, ExecutionKind::AgentTurn)
3474            .await
3475            .unwrap();
3476        for step in 0..5 {
3477            backend
3478                .append(step_result(exec, step, format!("v{step}").as_bytes()))
3479                .await
3480                .unwrap();
3481        }
3482
3483        // Fold steps 0..3 into a checkpoint.
3484        let folded = backend.checkpoint_fold(exec, 3).await.unwrap();
3485        assert_eq!(folded, 3);
3486
3487        // The individual rows for the folded steps are gone; steps 3 and 4 remain, plus a checkpoint.
3488        let remaining = backend.read_execution(exec).await.unwrap();
3489        let step_results: Vec<u32> = remaining
3490            .iter()
3491            .filter(|e| matches!(e.entry, EntryKind::StepResult { .. }))
3492            .map(|e| e.step_id.value())
3493            .collect();
3494        assert_eq!(step_results, vec![3, 4], "folded step rows are deleted");
3495        assert!(
3496            remaining
3497                .iter()
3498                .any(|e| matches!(e.entry, EntryKind::Checkpoint { .. })),
3499            "a checkpoint entry replaces the folded prefix"
3500        );
3501
3502        // The reconstructed folded results carry the original values and idempotency keys.
3503        let preloaded = backend.read_checkpoints(exec).await.unwrap();
3504        assert_eq!(preloaded.len(), 3);
3505        for (i, entry) in preloaded.iter().enumerate() {
3506            let step = u32::try_from(i).unwrap();
3507            assert_eq!(entry.step_id, StepId::new(step));
3508            match &entry.entry {
3509                EntryKind::StepResult {
3510                    payload,
3511                    idempotency_key,
3512                    ..
3513                } => {
3514                    assert_eq!(payload.as_ref(), format!("v{step}").as_bytes());
3515                    assert_eq!(
3516                        *idempotency_key,
3517                        IdempotencyKey::derive(exec, StepId::new(step), b"tool:read")
3518                    );
3519                }
3520                other => panic!("unexpected folded entry: {other:?}"),
3521            }
3522        }
3523    }
3524}